support use sips scheme for outbound tls gateway (#332)

* support use sips scheme for outbound tls gateway

* support use sips scheme for outbound tls gateway

* update license

.dockerignore

@@ -0,0 +1 @@
+node_modules

.eslintrc.json

@@ -8,7 +8,7 @@
       "jsx": false,
       "modules": false
     },
-    "ecmaVersion": 2017 
+    "ecmaVersion": 2020
   },
   "plugins": ["promise"],
   "rules": {

.github/workflows/ci.yml

@@ -0,0 +1,17 @@
+name: CI
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: lts/*
+      - run: npm install
+      - run: npm run jslint
+      - run: npm test
+
+ 

.github/workflows/docker-publish-dbcreate.yml

@@ -0,0 +1,55 @@
+name: Docker
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - '*'
+
+jobs:
+  push:
+
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: prepare tag 
+        id: prepare_tag
+        run: |
+            IMAGE_ID=jambonz/db-create
+
+            # Strip git ref prefix from version
+            VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+            # Strip "v" prefix from tag name
+            [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+            # Use Docker `latest` tag convention
+            [ "$VERSION" == "main" ] && VERSION=latest
+
+            echo IMAGE_ID=$IMAGE_ID
+            echo VERSION=$VERSION
+
+            echo "image_id=$IMAGE_ID" >> $GITHUB_OUTPUT
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
+
+      - name: Login to Docker Hub 
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile.db-create
+          push: true
+          tags: ${{ steps.prepare_tag.outputs.image_id }}:${{ steps.prepare_tag.outputs.version }}
+          build-args: |
+            GITHUB_REPOSITORY=$GITHUB_REPOSITORY
+            GITHUB_REF=$GITHUB_REF

.github/workflows/docker-publish.yml

@@ -0,0 +1,54 @@
+name: Docker
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - '*'
+
+jobs:
+  push:
+
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: prepare tag 
+        id: prepare_tag
+        run: |
+            IMAGE_ID=jambonz/api-server
+
+            # Strip git ref prefix from version
+            VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+            # Strip "v" prefix from tag name
+            [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+            # Use Docker `latest` tag convention
+            [ "$VERSION" == "main" ] && VERSION=latest
+
+            echo IMAGE_ID=$IMAGE_ID
+            echo VERSION=$VERSION
+
+            echo "image_id=$IMAGE_ID" >> $GITHUB_OUTPUT
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
+
+      - name: Login to Docker Hub 
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.prepare_tag.outputs.image_id }}:${{ steps.prepare_tag.outputs.version }}
+          build-args: |
+            GITHUB_REPOSITORY=$GITHUB_REPOSITORY
+            GITHUB_REF=$GITHUB_REF

.gitignore

@@ -1,7 +1,7 @@
 # Logs
 logs
 *.log
-package-lock.json
+run-tests.sh
 
 # Runtime data
 pids
@@ -43,4 +43,7 @@ create_db.sql
 .vscode
 
 .env.*
-.env
+.env
+
+test/postgres-data
+db/remove-account.sh

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+npm run jslint

.travis.yml

@@ -1,8 +0,0 @@
-dist: bionic
-language: node_js
-node_js:
-  - "lts/*"
-services:
-  - mysql
-script:
-  - npm test

Dockerfile

@@ -1,13 +1,23 @@
-FROM node:lts-alpine
+FROM --platform=linux/amd64 node:18.15-alpine3.16 as base
 
-RUN mkdir -p /usr/src/app
-WORKDIR /usr/src/app
+RUN apk --update --no-cache add --virtual .builds-deps build-base python3
+
+WORKDIR /opt/app/
+
+FROM base as build
+
+COPY package.json package-lock.json ./
+
+RUN npm ci
+
+COPY . .
+
+FROM base
+
+COPY --from=build /opt/app /opt/app/
 
 ARG NODE_ENV
+
 ENV NODE_ENV $NODE_ENV
 
-COPY package.json /usr/src/app/
-RUN npm install
-COPY . /usr/src/app
-
-CMD [ "npm", "start" ]
+CMD [ "node", "app.js" ]

Dockerfile.db-create

@@ -0,0 +1,23 @@
+FROM --platform=linux/amd64 node:18.15-alpine3.16 as base
+
+RUN apk --update --no-cache add --virtual .builds-deps build-base python3
+
+WORKDIR /opt/app/
+
+FROM base as build
+
+COPY package.json package-lock.json ./
+
+RUN npm ci
+
+COPY . .
+
+FROM base
+
+COPY --from=build /opt/app /opt/app/
+
+ARG NODE_ENV
+
+ENV NODE_ENV $NODE_ENV
+
+CMD [ "npm", "run", "upgrade-db" ]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018-2024 FirstFive8, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,29 +1,46 @@
-# jambones-api-server [![Build Status](https://secure.travis-ci.org/jambonz/jambones-api-server.png)](http://travis-ci.org/jambonz/jambones-api-server)
+# jambonz-api-server ![Build Status](https://github.com/jambonz/jambonz-api-server/workflows/CI/badge.svg)
 
-Jambones REST API server.
+Jambones REST API server of the jambones platform.
 
 ## Configuration
 
-This process requires the following environment variables to be set.
+Configuration is provided via environment variables:
 
-```
-JAMBONES_MYSQL_HOST
-JAMBONES_MYSQL_USER
-JAMBONES_MYSQL_PASSWORD
-JAMBONES_MYSQL_DATABASE
-JAMBONES_MYSQL_CONNECTION_LIMIT   # defaults to 10
-JAMBONES_REDIS_HOST
-JAMBONES_REDIS_PORT
-JAMBONES_LOGLEVEL                 # defaults to info
-JAMBONES_API_VERSION              # defaults to v1
-HTTP_PORT                         # defaults to 3000
-```
+| variable | meaning | required?|
+|----------|----------|---------|
+|JWT_SECRET| secret for signing JWT token |yes|
+|JWT_EXPIRES_IN| expiration time for JWT token(in minutes) |no|
+|ENCRYPTION_SECRET| secret for credential encryption(JWT_SECRET is deprecated) |yes|
+|HTTP_PORT| tcp port to listen on for API requests from jambonz-api-server |no|
+|JAMBONES_LOGLEVEL| log level for application, 'info' or 'debug' |no|
+|JAMBONES_MYSQL_HOST| mysql host |yes|
+|JAMBONES_MYSQL_USER| mysql username |yes|
+|JAMBONES_MYSQL_PASSWORD|  mysql password |yes|
+|JAMBONES_MYSQL_DATABASE| mysql data |yes|
+|JAMBONES_MYSQL_PORT| mysql port |no|
+|JAMBONES_MYSQL_CONNECTION_LIMIT| mysql connection limit |no|
+|JAMBONES_REDIS_HOST| redis host |yes|
+|JAMBONES_REDIS_PORT| redis port |no|
+|RATE_LIMIT_WINDOWS_MINS| rate limit window |no|
+|RATE_LIMIT_MAX_PER_WINDOW| number of requests per window |no|
+|JAMBONES_TRUST_PROXY| trust proxies, must be a number |no|
+|JAMBONES_API_VERSION| api version |no|
+|JAMBONES_TIME_SERIES_HOST| influxdb host |yes|
+|JAMBONES_CLUSTER_ID| cluster id |no|
+|HOMER_BASE_URL| HOMER URL |no|
+|HOMER_USERNAME| HOMER username |no|
+|HOMER_PASSWORD| HOMER password |no|
+|K8S| service running as kubernetes service |no|
+|K8S_FEATURE_SERVER_SERVICE_NAME| feature server name(required for K8S) |no|
+|K8S_FEATURE_SERVER_SERVICE_PORT| feature server port(required for K8S) |no|
+|JAMBONZ_RECORD_WS_USERNAME| recording websocket username|no|
+|JAMBONZ_RECORD_WS_PASSWORD| recording websocket password|no|
 
 #### Database dependency
 A mysql database is used to store long-lived objects such as Accounts, Applications, etc. To create the database schema, use or review the scripts in the 'db' folder, particularly:
-- [create_db.sql](db/create_db.sql), which creates the database and associated user (you may want to edit the username and password),
 - [jambones-sql.sql](db/jambones-sql.sql), which creates the schema,
-- [create-admin-token.sql](db/create-admin-token.sql), which creates an admin-level auth token that can be used for testing/exercising the API.
+- [seed-production-database-open-source.sql](db/seed-production-database-open-source.sql), which seeds the database with initial dataset(accounts, permissions, api keys, applications etc).
+- [create-admin-user.sql](db/create-admin-user.sql), which creates admin user with password set to "admin". The password will be forced to change after the first login.
 
 > Note: due to the dependency on the npmjs [mysql](https://www.npmjs.com/package/mysql) package, the mysql database must be configured to use sql [native authentication](https://medium.com/@crmcmullen/how-to-run-mysql-8-0-with-native-password-authentication-502de5bac661).
 

app.js

@@ -1,64 +1,124 @@
 const assert = require('assert');
-const opts = Object.assign({
-  timestamp: () => {
-    return `, "time": "${new Date().toISOString()}"`;
-  }
-}, {
-  level: process.env.JAMBONES_LOGLEVEL || 'info'
-});
-const logger = require('pino')(opts);
+const logger = require('./lib/logger');
 const express = require('express');
 const app = express();
+const helmet = require('helmet');
+const nocache = require('nocache');
+const rateLimit = require('express-rate-limit');
 const cors = require('cors');
 const passport = require('passport');
-const authStrategy = require('./lib/auth')(logger);
 const routes = require('./lib/routes');
+const Registrar = require('@jambonz/mw-registrar');
 
 assert.ok(process.env.JAMBONES_MYSQL_HOST &&
   process.env.JAMBONES_MYSQL_USER &&
   process.env.JAMBONES_MYSQL_PASSWORD &&
   process.env.JAMBONES_MYSQL_DATABASE, 'missing JAMBONES_MYSQL_XXX env vars');
-assert.ok(process.env.JAMBONES_REDIS_HOST, 'missing JAMBONES_REDIS_HOST env var');
+if (process.env.JAMBONES_REDIS_SENTINELS) {
+  assert.ok(process.env.JAMBONES_REDIS_SENTINEL_MASTER_NAME,
+    'missing JAMBONES_REDIS_SENTINEL_MASTER_NAME env var, JAMBONES_REDIS_SENTINEL_PASSWORD env var is optional');
+} else {
+  assert.ok(process.env.JAMBONES_REDIS_HOST, 'missing JAMBONES_REDIS_HOST env var');
+}
+assert.ok(process.env.JAMBONES_TIME_SERIES_HOST, 'missing JAMBONES_TIME_SERIES_HOST env var');
+assert.ok(process.env.ENCRYPTION_SECRET || process.env.JWT_SECRET, 'missing ENCRYPTION_SECRET env var');
+assert.ok(process.env.JWT_SECRET, 'missing JWT_SECRET env var');
 
 const {
+  queryCdrs,
+  queryCdrsSP,
+  queryAlerts,
+  queryAlertsSP,
+  writeCdrs,
+  writeAlerts,
+  AlertType
+} = require('@jambonz/time-series')(
+  logger, process.env.JAMBONES_TIME_SERIES_HOST
+);
+const {
+  client,
   retrieveCall,
   deleteCall,
   listCalls,
+  listSortedSets,
   purgeCalls,
-  retrieveSet
-} = require('@jambonz/realtimedb-helpers')({
-  host: process.env.JAMBONES_REDIS_HOST || 'localhost',
-  port: process.env.JAMBONES_REDIS_PORT || 6379
-}, logger);
+  retrieveSet,
+  addKey,
+  retrieveKey,
+  deleteKey,
+  incrKey,
+  listConferences
+} = require('./lib/helpers/realtimedb-helpers');
+const {
+  getTtsVoices,
+  getTtsSize,
+  purgeTtsCache,
+  getAwsAuthToken,
+  getVerbioAccessToken,
+  synthAudio
+} = require('@jambonz/speech-utils')({}, logger);
 const {
   lookupAppBySid,
   lookupAccountBySid,
   lookupAccountByPhoneNumber,
-  lookupAppByPhoneNumber
+  lookupAppByPhoneNumber,
+  lookupCarrierBySid,
+  lookupSipGatewayBySid,
+  lookupSmppGatewayBySid,
+  lookupClientByAccountAndUsername
 } = require('@jambonz/db-helpers')({
   host: process.env.JAMBONES_MYSQL_HOST,
   user: process.env.JAMBONES_MYSQL_USER,
+  port: process.env.JAMBONES_MYSQL_PORT || 3306,
   password: process.env.JAMBONES_MYSQL_PASSWORD,
   database: process.env.JAMBONES_MYSQL_DATABASE,
   connectionLimit: process.env.JAMBONES_MYSQL_CONNECTION_LIMIT || 10
 }, logger);
 const PORT = process.env.HTTP_PORT || 3000;
+const authStrategy = require('./lib/auth')(logger, retrieveKey);
+const {delayLoginMiddleware} = require('./lib/middleware');
+const Websocket = require('ws');
 
 passport.use(authStrategy);
 
 app.locals = app.locals || {};
-Object.assign(app.locals, {
+app.locals = {
+  ...app.locals,
+  registrar: new Registrar(logger, client),
   logger,
   retrieveCall,
   deleteCall,
   listCalls,
+  listSortedSets,
+  listConferences,
   purgeCalls,
   retrieveSet,
+  addKey,
+  incrKey,
+  retrieveKey,
+  deleteKey,
+  getTtsVoices,
+  getTtsSize,
+  getAwsAuthToken,
+  getVerbioAccessToken,
+  purgeTtsCache,
+  synthAudio,
   lookupAppBySid,
   lookupAccountBySid,
   lookupAccountByPhoneNumber,
-  lookupAppByPhoneNumber
-});
+  lookupAppByPhoneNumber,
+  lookupCarrierBySid,
+  lookupSipGatewayBySid,
+  lookupSmppGatewayBySid,
+  lookupClientByAccountAndUsername,
+  queryCdrs,
+  queryCdrsSP,
+  queryAlerts,
+  queryAlertsSP,
+  writeCdrs,
+  writeAlerts,
+  AlertType
+};
 
 const unless = (paths, middleware) => {
   return (req, res, next) => {
@@ -67,14 +127,51 @@ const unless = (paths, middleware) => {
   };
 };
 
+const limiter = rateLimit({
+  windowMs: (process.env.RATE_LIMIT_WINDOWS_MINS || 5) * 60 * 1000, // 5 minutes
+  max: process.env.RATE_LIMIT_MAX_PER_WINDOW || 600, // Limit each IP to 600 requests per `window`
+  standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers
+  legacyHeaders: false, // Disable the `X-RateLimit-*` headers
+});
+
+// Setup websocket for recording audio
+const recordWsServer = require('./lib/record');
+const wsServer = new Websocket.Server({ noServer: true });
+wsServer.setMaxListeners(0);
+wsServer.on('connection', recordWsServer.bind(null, logger));
+
+if (process.env.JAMBONES_TRUST_PROXY) {
+  const proxyCount = parseInt(process.env.JAMBONES_TRUST_PROXY);
+  if (!isNaN(proxyCount) && proxyCount > 0) {
+    logger.info(`setting trust proxy to ${proxyCount} and mounting endpoint /ip`);
+    app.set('trust proxy', proxyCount);
+    app.get('/ip', (req, res) => {
+      logger.info({headers: req.headers}, 'received GET /ip');
+      res.send(req.ip);
+    });
+  }
+}
+app.use(limiter);
+app.use(helmet());
+app.use(helmet.hidePoweredBy());
+app.use(nocache());
+app.use(passport.initialize());
 app.use(cors());
-app.use(express.urlencoded({
-  extended: true
-}));
-app.use(express.json());
-app.use('/v1', unless(['/login', '/Users', '/messaging', '/outboundSMS'], passport.authenticate('bearer', {
-  session: false
-})));
+app.use(express.urlencoded({extended: true}));
+app.use(delayLoginMiddleware);
+app.use(unless(['/stripe'], express.json()));
+app.use('/v1', unless(
+  [
+    '/register',
+    '/forgot-password',
+    '/signin',
+    '/login',
+    '/messaging',
+    '/outboundSMS',
+    '/AccountTest',
+    '/InviteCodes',
+    '/PredefinedCarriers'
+  ], passport.authenticate('bearer', {session: false})));
 app.use('/', routes);
 app.use((err, req, res, next) => {
   logger.error(err, 'burped error');
@@ -83,7 +180,52 @@ app.use((err, req, res, next) => {
   });
 });
 logger.info(`listening for HTTP traffic on port ${PORT}`);
-app.listen(PORT);
+const server = app.listen(PORT);
+
+
+const isValidWsKey = (hdr) => {
+  const username = process.env.JAMBONZ_RECORD_WS_USERNAME || process.env.JAMBONES_RECORD_WS_USERNAME;
+  const password = process.env.JAMBONZ_RECORD_WS_PASSWORD || process.env.JAMBONES_RECORD_WS_PASSWORD;
+  if (username && password) {
+    if (!hdr) {
+      // auth header is missing
+      return false;
+    }
+    const token = Buffer.from(`${username}:${password}`).toString('base64');
+    const arr = /^Basic (.*)$/.exec(hdr);
+    if (!Array.isArray(arr)) {
+      // malformed auth header
+      return false;
+    }
+    return arr[1] === token;
+  }
+  return true;
+};
+
+server.on('upgrade', (request, socket, head) => {
+  logger.debug({
+    url: request.url,
+    headers: request.headers,
+  }, 'received upgrade request');
+
+  /* verify the path starts with /transcribe */
+  if (!request.url.includes('/record/')) {
+    logger.info(`unhandled path: ${request.url}`);
+    return socket.write('HTTP/1.1 404 Not Found \r\n\r\n', () => socket.destroy());
+  }
+
+  /* verify the api key */
+  if (!isValidWsKey(request.headers['authorization'])) {
+    logger.info(`invalid auth header: ${request.headers['authorization'] || 'authorization header missing'}`);
+    return socket.write('HTTP/1.1 403 Forbidden \r\n\r\n', () => socket.destroy());
+  }
+
+  /* complete the upgrade */
+  wsServer.handleUpgrade(request, socket, head, (ws) => {
+    logger.info(`upgraded to websocket, url: ${request.url}`);
+    wsServer.emit('connection', ws, request.url);
+  });
+});
 
 // purge old calls from active call set every 10 mins
 async function purge() {

db/add-predefined-carriers.sql

@@ -0,0 +1,50 @@
+
+-- create predefined carriers
+insert into predefined_carriers (predefined_carrier_sid, name, requires_static_ip, e164_leading_plus, 
+requires_register, register_username, register_password, 
+register_sip_realm, tech_prefix, inbound_auth_username, inbound_auth_password, diversion)
+VALUES
+('7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', 'Twilio', 0, 1, 0, '<your-twilio-credential-username>', '<your-twilio-credential-password>', NULL, NULL, NULL, NULL, NULL),
+('032d90d5-39e8-41c0-b807-9c88cffba65c', 'Voxbone', 0, 1, 0, '<your-voxbone-outbound-username>', '<your-voxbone-outbound-password>', NULL, NULL, NULL, NULL, '<your-voxbone-DID>'),
+('17479288-bb9f-421a-89d1-f4ac57af1dca', 'TelecomsXChange', 0, 0, 0, NULL, NULL, NULL, 'your-tech-prefix', NULL, NULL, NULL),
+('e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'Simwood', 0, 1, 0, '<your-simwood-auth-trunk-username>',  '<your-simwood-auth-trunk-password>', NULL, NULL, NULL, NULL, NULL);
+
+-- TelecomXchange gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('c9c3643e-9a83-4b78-b172-9c09d911bef5', '17479288-bb9f-421a-89d1-f4ac57af1dca', '174.136.44.213', 32, 5060, 1, 0),
+('3b5b7fa5-4e61-4423-b921-05c3283b2101', '17479288-bb9f-421a-89d1-f4ac57af1dca', 'sip01.TelecomsXChange.com', 32, 5060, 0, 1);
+
+-- twilio gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('d2ccfcb1-9198-4fe9-a0ca-6e49395837c4', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.172.60.0', 30, 5060, 1, 0),
+('6b1d0032-4430-41f1-87c6-f22233d394ef', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.244.51.0', 30, 5060, 1, 0),
+('0de40217-8bd5-4aa8-a9fd-1994282953c6', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.192', 30, 5060, 1, 0),
+('37bc0b20-b53c-4c31-95a6-f82b1c3713e3', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '35.156.191.128', 30, 5060, 1, 0),
+('39791f4e-b612-4882-a37e-e92711a39f3f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.65.63.192', 30, 5060, 1, 0),
+('81a0c8cb-a33e-42da-8f20-99083da6f02f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.252.254.64', 30, 5060, 1, 0),
+('eeeef07a-46b8-4ffe-a4f2-04eb32ca889e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.169.127.128', 30, 5060, 1, 0),
+('fbb6c194-4b68-4dff-9b42-52412be1c39e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '177.71.206.192', 30, 5060, 1, 0),
+('3ed1dd12-e1a7-44ff-811a-3cc5dc13dc72', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '<your-domain>.pstn.twilio.com', 32, 5060, 0, 1);
+
+-- voxbone gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('d531c582-2103-42a0-b9f0-f80c215b3ec5', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.83.45', 32, 5060, 1, 0),
+('95c888e5-c959-4d92-82c4-8597dddff75e', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.86.45', 32, 5060, 1, 0),
+('1de3b2a1-96f0-407a-bcc4-ce371d823a8d', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.82.45', 32, 5060, 1, 0),
+('50c1f91a-6080-4495-a241-6bba6e9d9688', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.85.45', 32, 5060, 1, 0),
+('e6ebad33-80d5-4dbb-bc6f-a7ae08160cc6', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.84.45', 32, 5060, 1, 0),
+('7bae60b3-4237-4baa-a711-30ea3bce19d8', '032d90d5-39e8-41c0-b807-9c88cffba65c', '185.47.148.45', 32, 5060, 1, 0),
+('bc933522-18a2-47d8-9ae4-9faa8de4e927', '032d90d5-39e8-41c0-b807-9c88cffba65c', 'outbound.voxbone.com', 32, 5060, 0, 1);
+
+-- simwood gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('91cb050f-9826-4ac9-b736-84a10372a9fe', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.139.77', 32, 5060, 1, 0),
+('58700fad-98bf-4d31-b61e-888c54911b35', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.140.77', 32, 5060, 1, 0),
+('d020fd9e-7fdb-4bca-ae0d-e61b38142873', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.142.77', 32, 5060, 1, 0),
+('38d8520a-527f-4f8e-8456-f9dfca742561', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.77', 32, 5060, 1, 0),
+('834f8b0c-d4c2-4f3e-93d9-cf307995eedd', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.88', 32, 5060, 1, 0),
+('5f431d42-48e4-44ce-a311-d946f0b475b6', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'out.simwood.com', 32, 5060, 0, 1);

db/create-admin-user.sql

@@ -0,0 +1,10 @@
+/* hashed password is "admin" */
+insert into users (user_sid, name, email, hashed_password, force_change, provider, email_validated)
+values ('12c80508-edf9-4b22-8d09-55abd02648eb', 'admin', 'joe@foo.bar', '$argon2i$v=19$m=65536,t=3,p=4$c2FsdHNhbHRzYWx0c2FsdA$x5OO6gXFXS25oqUU2JvbYqrSgRxBujNUJBq6xv9EgjM', 1, 'local', 1);
+
+insert into user_permissions (user_permissions_sid, user_sid, permission_sid) 
+values ('8919e0dc-4d69-4de5-be56-a121598d9093', '12c80508-edf9-4b22-8d09-55abd02648eb', 'ffbc342a-546a-11ed-bdc3-0242ac120002');
+insert into user_permissions (user_permissions_sid, user_sid, permission_sid) 
+values ('d6fdf064-0a65-4b17-8b10-5500e956a159', '12c80508-edf9-4b22-8d09-55abd02648eb', 'ffbc3a10-546a-11ed-bdc3-0242ac120002');
+insert into user_permissions (user_permissions_sid, user_sid, permission_sid) 
+values ('f68185dd-0486-4767-a77d-a0b84c1b236e' ,'12c80508-edf9-4b22-8d09-55abd02648eb', 'ffbc3c5e-546a-11ed-bdc3-0242ac120002');

db/create-default-account.sql

@@ -1,4 +1,4 @@
 insert into service_providers (service_provider_sid, name) 
 values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'default service provider');
-insert into accounts (account_sid, service_provider_sid, name) 
-values ('9351f46a-678c-43f5-b8a6-d4eb58d131af','2708b1b3-2736-40ea-b502-c53d8396247f', 'default account');
+insert into accounts (account_sid, service_provider_sid, name, webhook_secret) 
+values ('9351f46a-678c-43f5-b8a6-d4eb58d131af','2708b1b3-2736-40ea-b502-c53d8396247f', 'default account', 'foobar');

db/create_test_db.sql

@@ -1,3 +1,3 @@
 create database jambones_test;
-create user jambones_test@localhost IDENTIFIED WITH mysql_native_password by 'jambones_test';
-grant all on jambones_test.* to jambones_test@localhost;
+create user jambones_test@'%' IDENTIFIED WITH mysql_native_password by 'jambones_test';
+grant all on jambones_test.* to jambones_test@'%';

db/generate-beta-invite-codes.js

@@ -0,0 +1,40 @@
+#!/usr/bin/env node
+const crypto = require('crypto');
+const {promisePool} = require('../lib/db');
+const sql = 'INSERT INTO beta_invite_codes (invite_code) VALUES (?);';
+
+const rand_string = (n) => {
+  if (n <= 0) {
+    return '';
+  }
+  var rs = '';
+  try {
+    rs = crypto.randomBytes(Math.ceil(n/2)).toString('hex').slice(0,n);
+    /* note: could do this non-blocking, but still might fail */
+  }
+  catch (ex) {
+    /* known exception cause: depletion of entropy info for randomBytes */
+    console.error('Exception generating random string: ' + ex);
+    /* weaker random fallback */
+    rs = '';
+    var r = n % 8, q = (n - r) / 8, i;
+    for (i = 0; i < q; i++) {
+      rs += Math.random().toString(16).slice(2);
+    }
+    if (r > 0) {
+      rs += Math.random().toString(16).slice(2, i);
+    }
+  }
+  return rs;
+};
+
+const doIt = async(len) => {
+  for (let i = 0; i < 50; i++) {
+    const val = rand_string(len).toUpperCase();
+    await promisePool.execute(sql, [val]);
+  }
+  process.exit(0);
+};
+
+doIt(6);
+

db/jambones-sql.sql

@@ -2,20 +2,68 @@
 
 SET FOREIGN_KEY_CHECKS=0;
 
+DROP TABLE IF EXISTS account_static_ips;
+
+DROP TABLE IF EXISTS account_limits;
+
+DROP TABLE IF EXISTS account_products;
+
+DROP TABLE IF EXISTS account_subscriptions;
+
+DROP TABLE IF EXISTS beta_invite_codes;
+
 DROP TABLE IF EXISTS call_routes;
 
+DROP TABLE IF EXISTS clients;
+
+DROP TABLE IF EXISTS dns_records;
+
+DROP TABLE IF EXISTS lcr;
+
 DROP TABLE IF EXISTS lcr_carrier_set_entry;
 
 DROP TABLE IF EXISTS lcr_routes;
 
-DROP TABLE IF EXISTS api_keys;
+DROP TABLE IF EXISTS password_settings;
 
-DROP TABLE IF EXISTS ms_teams_tenants;
+DROP TABLE IF EXISTS user_permissions;
+
+DROP TABLE IF EXISTS permissions;
+
+DROP TABLE IF EXISTS predefined_sip_gateways;
+
+DROP TABLE IF EXISTS predefined_smpp_gateways;
+
+DROP TABLE IF EXISTS predefined_carriers;
+
+DROP TABLE IF EXISTS account_offers;
+
+DROP TABLE IF EXISTS products;
+
+DROP TABLE IF EXISTS schema_version;
+
+DROP TABLE IF EXISTS api_keys;
 
 DROP TABLE IF EXISTS sbc_addresses;
 
+DROP TABLE IF EXISTS ms_teams_tenants;
+
+DROP TABLE IF EXISTS service_provider_limits;
+
+DROP TABLE IF EXISTS signup_history;
+
+DROP TABLE IF EXISTS smpp_addresses;
+
+DROP TABLE IF EXISTS google_custom_voices;
+
+DROP TABLE IF EXISTS speech_credentials;
+
+DROP TABLE IF EXISTS system_information;
+
 DROP TABLE IF EXISTS users;
 
+DROP TABLE IF EXISTS smpp_gateways;
+
 DROP TABLE IF EXISTS phone_numbers;
 
 DROP TABLE IF EXISTS sip_gateways;
@@ -30,6 +78,50 @@ DROP TABLE IF EXISTS service_providers;
 
 DROP TABLE IF EXISTS webhooks;
 
+CREATE TABLE account_static_ips
+(
+account_static_ip_sid CHAR(36) NOT NULL UNIQUE ,
+account_sid CHAR(36) NOT NULL,
+public_ipv4 VARCHAR(16) NOT NULL UNIQUE ,
+private_ipv4 VARBINARY(16) NOT NULL UNIQUE ,
+PRIMARY KEY (account_static_ip_sid)
+);
+
+CREATE TABLE account_limits
+(
+account_limits_sid CHAR(36) NOT NULL UNIQUE ,
+account_sid CHAR(36) NOT NULL,
+category ENUM('api_rate','voice_call_session', 'device','voice_call_minutes','voice_call_session_license', 'voice_call_minutes_license') NOT NULL,
+quantity INTEGER NOT NULL,
+PRIMARY KEY (account_limits_sid)
+);
+
+CREATE TABLE account_subscriptions
+(
+account_subscription_sid CHAR(36) NOT NULL UNIQUE ,
+account_sid CHAR(36) NOT NULL,
+pending BOOLEAN NOT NULL DEFAULT false,
+effective_start_date DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+effective_end_date DATETIME,
+change_reason VARCHAR(255),
+stripe_subscription_id VARCHAR(56),
+stripe_payment_method_id VARCHAR(56),
+stripe_statement_descriptor VARCHAR(255),
+last4 VARCHAR(512),
+exp_month INTEGER,
+exp_year INTEGER,
+card_type VARCHAR(16),
+pending_reason VARBINARY(52),
+PRIMARY KEY (account_subscription_sid)
+);
+
+CREATE TABLE beta_invite_codes
+(
+invite_code CHAR(6) NOT NULL UNIQUE ,
+in_use BOOLEAN NOT NULL DEFAULT false,
+PRIMARY KEY (invite_code)
+);
+
 CREATE TABLE call_routes
 (
 call_route_sid CHAR(36) NOT NULL UNIQUE ,
@@ -38,16 +130,139 @@ account_sid CHAR(36) NOT NULL,
 regex VARCHAR(255) NOT NULL,
 application_sid CHAR(36) NOT NULL,
 PRIMARY KEY (call_route_sid)
-) ENGINE=InnoDB COMMENT='a regex-based pattern match for call routing';
+) COMMENT='a regex-based pattern match for call routing';
+
+CREATE TABLE clients
+(
+client_sid CHAR(36) NOT NULL UNIQUE ,
+account_sid CHAR(36) NOT NULL,
+is_active BOOLEAN NOT NULL DEFAULT 1,
+username VARCHAR(64),
+password VARCHAR(1024),
+allow_direct_app_calling BOOLEAN NOT NULL DEFAULT 1,
+allow_direct_queue_calling BOOLEAN NOT NULL DEFAULT 1,
+allow_direct_user_calling BOOLEAN NOT NULL DEFAULT 1,
+PRIMARY KEY (client_sid)
+);
+
+CREATE TABLE dns_records
+(
+dns_record_sid CHAR(36) NOT NULL UNIQUE ,
+account_sid CHAR(36) NOT NULL,
+record_type VARCHAR(6) NOT NULL,
+record_id INTEGER NOT NULL,
+PRIMARY KEY (dns_record_sid)
+);
 
 CREATE TABLE lcr_routes
 (
 lcr_route_sid CHAR(36),
+lcr_sid CHAR(36) NOT NULL,
 regex VARCHAR(32) NOT NULL COMMENT 'regex-based pattern match against dialed number, used for LCR routing of PSTN calls',
 description VARCHAR(1024),
-priority INTEGER NOT NULL UNIQUE  COMMENT 'lower priority routes are attempted first',
+priority INTEGER NOT NULL COMMENT 'lower priority routes are attempted first',
 PRIMARY KEY (lcr_route_sid)
-) COMMENT='Least cost routing table';
+) COMMENT='An ordered list of  digit patterns in an LCR table.  The patterns are tested in sequence until one matches';
+
+CREATE TABLE lcr
+(
+lcr_sid CHAR(36) NOT NULL UNIQUE ,
+name VARCHAR(64) COMMENT 'User-assigned name for this LCR table',
+is_active BOOLEAN NOT NULL DEFAULT 1,
+default_carrier_set_entry_sid CHAR(36) COMMENT 'default carrier/route to use when no digit match based results are found.',
+service_provider_sid CHAR(36),
+account_sid CHAR(36),
+PRIMARY KEY (lcr_sid)
+) COMMENT='An LCR (least cost routing) table that is used by a service provider or account to make decisions about routing outbound calls when multiple carriers are available.';
+
+CREATE TABLE password_settings
+(
+min_password_length INTEGER NOT NULL DEFAULT 8,
+require_digit BOOLEAN NOT NULL DEFAULT false,
+require_special_character BOOLEAN NOT NULL DEFAULT false
+);
+
+CREATE TABLE permissions
+(
+permission_sid CHAR(36) NOT NULL UNIQUE ,
+name VARCHAR(32) NOT NULL UNIQUE ,
+description VARCHAR(255),
+PRIMARY KEY (permission_sid)
+);
+
+CREATE TABLE predefined_carriers
+(
+predefined_carrier_sid CHAR(36) NOT NULL UNIQUE ,
+name VARCHAR(64) NOT NULL,
+requires_static_ip BOOLEAN NOT NULL DEFAULT false,
+e164_leading_plus BOOLEAN NOT NULL DEFAULT false COMMENT 'if true, a leading plus should be prepended to outbound phone numbers',
+requires_register BOOLEAN NOT NULL DEFAULT false,
+register_username VARCHAR(64),
+register_sip_realm VARCHAR(64),
+register_password VARCHAR(64),
+tech_prefix VARCHAR(16) COMMENT 'tech prefix to prepend to outbound calls to this carrier',
+inbound_auth_username VARCHAR(64),
+inbound_auth_password VARCHAR(64),
+diversion VARCHAR(32),
+PRIMARY KEY (predefined_carrier_sid)
+);
+
+CREATE TABLE predefined_sip_gateways
+(
+predefined_sip_gateway_sid CHAR(36) NOT NULL UNIQUE ,
+ipv4 VARCHAR(128) NOT NULL COMMENT 'ip address or DNS name of the gateway.  For gateways providing inbound calling service, ip address is required.',
+port INTEGER NOT NULL DEFAULT 5060 COMMENT 'sip signaling port',
+inbound BOOLEAN NOT NULL COMMENT 'if true, whitelist this IP to allow inbound calls from the gateway',
+outbound BOOLEAN NOT NULL COMMENT 'if true, include in least-cost routing when placing calls to the PSTN',
+netmask INTEGER NOT NULL DEFAULT 32,
+predefined_carrier_sid CHAR(36) NOT NULL,
+PRIMARY KEY (predefined_sip_gateway_sid)
+);
+
+CREATE TABLE predefined_smpp_gateways
+(
+predefined_smpp_gateway_sid CHAR(36) NOT NULL UNIQUE ,
+ipv4 VARCHAR(128) NOT NULL COMMENT 'ip address or DNS name of the gateway. ',
+port INTEGER NOT NULL DEFAULT 2775 COMMENT 'smpp signaling port',
+inbound BOOLEAN NOT NULL COMMENT 'if true, whitelist this IP to allow inbound SMS from the gateway',
+outbound BOOLEAN NOT NULL COMMENT 'i',
+netmask INTEGER NOT NULL DEFAULT 32,
+is_primary BOOLEAN NOT NULL DEFAULT 1,
+use_tls BOOLEAN DEFAULT 0,
+predefined_carrier_sid CHAR(36) NOT NULL,
+PRIMARY KEY (predefined_smpp_gateway_sid)
+);
+
+CREATE TABLE products
+(
+product_sid CHAR(36) NOT NULL UNIQUE ,
+name VARCHAR(32) NOT NULL,
+category ENUM('api_rate','voice_call_session', 'device') NOT NULL,
+PRIMARY KEY (product_sid)
+);
+
+CREATE TABLE account_products
+(
+account_product_sid CHAR(36) NOT NULL UNIQUE ,
+account_subscription_sid CHAR(36) NOT NULL,
+product_sid CHAR(36) NOT NULL,
+quantity INTEGER NOT NULL,
+PRIMARY KEY (account_product_sid)
+);
+
+CREATE TABLE account_offers
+(
+account_offer_sid CHAR(36) NOT NULL UNIQUE ,
+account_sid CHAR(36) NOT NULL,
+product_sid CHAR(36) NOT NULL,
+stripe_product_id VARCHAR(56) NOT NULL,
+PRIMARY KEY (account_offer_sid)
+);
+
+CREATE TABLE schema_version
+(
+version VARCHAR(16)
+);
 
 CREATE TABLE api_keys
 (
@@ -55,11 +270,23 @@ api_key_sid CHAR(36) NOT NULL UNIQUE ,
 token CHAR(36) NOT NULL UNIQUE ,
 account_sid CHAR(36),
 service_provider_sid CHAR(36),
-expires_at TIMESTAMP NULL,
-last_used TIMESTAMP NULL,
+expires_at TIMESTAMP NULL  DEFAULT NULL,
+last_used TIMESTAMP NULL  DEFAULT NULL,
 created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
 PRIMARY KEY (api_key_sid)
-) ENGINE=InnoDB COMMENT='An authorization token that is used to access the REST api';
+) COMMENT='An authorization token that is used to access the REST api';
+
+CREATE TABLE sbc_addresses
+(
+sbc_address_sid CHAR(36) NOT NULL UNIQUE ,
+ipv4 VARCHAR(255) NOT NULL,
+port INTEGER NOT NULL DEFAULT 5060,
+tls_port INTEGER,
+wss_port INTEGER,
+service_provider_sid CHAR(36),
+last_updated DATETIME,
+PRIMARY KEY (sbc_address_sid)
+);
 
 CREATE TABLE ms_teams_tenants
 (
@@ -71,65 +298,170 @@ tenant_fqdn VARCHAR(255) NOT NULL UNIQUE ,
 PRIMARY KEY (ms_teams_tenant_sid)
 ) COMMENT='A Microsoft Teams customer tenant';
 
-CREATE TABLE sbc_addresses
+CREATE TABLE service_provider_limits
 (
-sbc_address_sid CHAR(36) NOT NULL UNIQUE ,
+service_provider_limits_sid CHAR(36) NOT NULL UNIQUE ,
+service_provider_sid CHAR(36) NOT NULL,
+category ENUM('api_rate','voice_call_session', 'device','voice_call_minutes','voice_call_session_license', 'voice_call_minutes_license') NOT NULL,
+quantity INTEGER NOT NULL,
+PRIMARY KEY (service_provider_limits_sid)
+);
+
+CREATE TABLE signup_history
+(
+email VARCHAR(255) NOT NULL,
+name VARCHAR(255),
+signed_up_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+PRIMARY KEY (email)
+);
+
+CREATE TABLE smpp_addresses
+(
+smpp_address_sid CHAR(36) NOT NULL UNIQUE ,
 ipv4 VARCHAR(255) NOT NULL,
 port INTEGER NOT NULL DEFAULT 5060,
+use_tls BOOLEAN NOT NULL DEFAULT 0,
+is_primary BOOLEAN NOT NULL DEFAULT 1,
 service_provider_sid CHAR(36),
-PRIMARY KEY (sbc_address_sid)
+PRIMARY KEY (smpp_address_sid)
+);
+
+CREATE TABLE speech_credentials
+(
+speech_credential_sid CHAR(36) NOT NULL UNIQUE ,
+service_provider_sid CHAR(36),
+account_sid CHAR(36),
+vendor VARCHAR(32) NOT NULL,
+credential VARCHAR(8192) NOT NULL,
+use_for_tts BOOLEAN DEFAULT true,
+use_for_stt BOOLEAN DEFAULT true,
+last_used DATETIME,
+last_tested DATETIME,
+tts_tested_ok BOOLEAN,
+stt_tested_ok BOOLEAN,
+created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+label VARCHAR(64),
+PRIMARY KEY (speech_credential_sid)
+);
+
+CREATE TABLE google_custom_voices
+(
+google_custom_voice_sid CHAR(36) NOT NULL UNIQUE ,
+speech_credential_sid CHAR(36) NOT NULL,
+model VARCHAR(512) NOT NULL,
+reported_usage ENUM('REPORTED_USAGE_UNSPECIFIED','REALTIME','OFFLINE') DEFAULT 'REALTIME',
+name VARCHAR(64) NOT NULL,
+PRIMARY KEY (google_custom_voice_sid)
+);
+
+CREATE TABLE system_information
+(
+domain_name VARCHAR(255),
+sip_domain_name VARCHAR(255),
+monitoring_domain_name VARCHAR(255)
 );
 
 CREATE TABLE users
 (
 user_sid CHAR(36) NOT NULL UNIQUE ,
-name CHAR(36) NOT NULL UNIQUE ,
-hashed_password VARCHAR(1024) NOT NULL,
-salt CHAR(16) NOT NULL,
-force_change BOOLEAN NOT NULL DEFAULT TRUE,
+name VARCHAR(255) NOT NULL,
+email VARCHAR(255) NOT NULL,
+pending_email VARCHAR(255),
+phone VARCHAR(20) UNIQUE ,
+hashed_password VARCHAR(1024),
+account_sid CHAR(36),
+service_provider_sid CHAR(36),
+force_change BOOLEAN NOT NULL DEFAULT FALSE,
+provider VARCHAR(255) NOT NULL,
+provider_userid VARCHAR(255),
+scope VARCHAR(16) NOT NULL DEFAULT 'read-write',
+phone_activation_code VARCHAR(16),
+email_activation_code VARCHAR(16),
+email_validated BOOLEAN NOT NULL DEFAULT false,
+phone_validated BOOLEAN NOT NULL DEFAULT false,
+email_content_opt_out BOOLEAN NOT NULL DEFAULT false,
+is_active BOOLEAN NOT NULL DEFAULT true,
 PRIMARY KEY (user_sid)
 );
 
 CREATE TABLE voip_carriers
 (
 voip_carrier_sid CHAR(36) NOT NULL UNIQUE ,
-name VARCHAR(64) NOT NULL UNIQUE ,
+name VARCHAR(64) NOT NULL,
 description VARCHAR(255),
-account_sid CHAR(36) COMMENT 'if provided, indicates this entity represents a customer PBX that is associated with a specific account',
+account_sid CHAR(36) COMMENT 'if provided, indicates this entity represents a sip trunk that is associated with a specific account',
+service_provider_sid CHAR(36),
 application_sid CHAR(36) COMMENT 'If provided, all incoming calls from this source will be routed to the associated application',
-e164_leading_plus BOOLEAN NOT NULL DEFAULT false,
+e164_leading_plus BOOLEAN NOT NULL DEFAULT false COMMENT 'if true, a leading plus should be prepended to outbound phone numbers',
+requires_register BOOLEAN NOT NULL DEFAULT false,
+register_username VARCHAR(64),
+register_sip_realm VARCHAR(64),
+register_password VARCHAR(64),
+tech_prefix VARCHAR(16) COMMENT 'tech prefix to prepend to outbound calls to this carrier',
+inbound_auth_username VARCHAR(64),
+inbound_auth_password VARCHAR(64),
+diversion VARCHAR(32),
+is_active BOOLEAN NOT NULL DEFAULT true,
+created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+smpp_system_id VARCHAR(255),
+smpp_password VARCHAR(64),
+smpp_enquire_link_interval INTEGER DEFAULT 0,
+smpp_inbound_system_id VARCHAR(255),
+smpp_inbound_password VARCHAR(64),
+register_from_user VARCHAR(128),
+register_from_domain VARCHAR(255),
+register_public_ip_in_contact BOOLEAN NOT NULL DEFAULT false,
+register_status VARCHAR(4096),
 PRIMARY KEY (voip_carrier_sid)
-) ENGINE=InnoDB COMMENT='A Carrier or customer PBX that can send or receive calls';
+) COMMENT='A Carrier or customer PBX that can send or receive calls';
+
+CREATE TABLE user_permissions
+(
+user_permissions_sid CHAR(36) NOT NULL UNIQUE ,
+user_sid CHAR(36) NOT NULL,
+permission_sid CHAR(36) NOT NULL,
+PRIMARY KEY (user_permissions_sid)
+);
+
+CREATE TABLE smpp_gateways
+(
+smpp_gateway_sid CHAR(36) NOT NULL UNIQUE ,
+ipv4 VARCHAR(128) NOT NULL,
+port INTEGER NOT NULL DEFAULT 2775,
+netmask INTEGER NOT NULL DEFAULT 32,
+is_primary BOOLEAN NOT NULL DEFAULT 1,
+inbound BOOLEAN NOT NULL DEFAULT 0 COMMENT 'if true, whitelist this IP to allow inbound calls from the gateway',
+outbound BOOLEAN NOT NULL DEFAULT 1 COMMENT 'if true, include in least-cost routing when placing calls to the PSTN',
+use_tls BOOLEAN DEFAULT 0,
+voip_carrier_sid CHAR(36) NOT NULL,
+PRIMARY KEY (smpp_gateway_sid)
+);
 
 CREATE TABLE phone_numbers
 (
 phone_number_sid CHAR(36) UNIQUE ,
-number VARCHAR(32) NOT NULL UNIQUE ,
-voip_carrier_sid CHAR(36) NOT NULL,
+number VARCHAR(132) NOT NULL,
+voip_carrier_sid CHAR(36),
 account_sid CHAR(36),
 application_sid CHAR(36),
+service_provider_sid CHAR(36) COMMENT 'if not null, this number is a test number for the associated service provider',
 PRIMARY KEY (phone_number_sid)
-) ENGINE=InnoDB COMMENT='A phone number that has been assigned to an account';
-
-CREATE TABLE webhooks
-(
-webhook_sid CHAR(36) NOT NULL UNIQUE ,
-url VARCHAR(1024) NOT NULL,
-method ENUM("GET","POST") NOT NULL DEFAULT 'POST',
-username VARCHAR(255),
-password VARCHAR(255),
-PRIMARY KEY (webhook_sid)
-) COMMENT='An HTTP callback';
+) COMMENT='A phone number that has been assigned to an account';
 
 CREATE TABLE sip_gateways
 (
 sip_gateway_sid CHAR(36),
 ipv4 VARCHAR(128) NOT NULL COMMENT 'ip address or DNS name of the gateway.  For gateways providing inbound calling service, ip address is required.',
-port INTEGER NOT NULL DEFAULT 5060 COMMENT 'sip signaling port',
+netmask INTEGER NOT NULL DEFAULT 32,
+port INTEGER COMMENT 'sip signaling port',
 inbound BOOLEAN NOT NULL COMMENT 'if true, whitelist this IP to allow inbound calls from the gateway',
 outbound BOOLEAN NOT NULL COMMENT 'if true, include in least-cost routing when placing calls to the PSTN',
 voip_carrier_sid CHAR(36) NOT NULL,
 is_active BOOLEAN NOT NULL DEFAULT 1,
+send_options_ping BOOLEAN NOT NULL DEFAULT 0,
+use_sips_scheme BOOLEAN NOT NULL DEFAULT 0,
+pad_crypto BOOLEAN NOT NULL DEFAULT 0,
+protocol ENUM('udp','tcp','tls', 'tls/srtp') DEFAULT 'udp' COMMENT 'Outbound call protocol',
 PRIMARY KEY (sip_gateway_sid)
 ) COMMENT='A whitelisted sip gateway used for origination/termination';
 
@@ -143,21 +475,45 @@ priority INTEGER NOT NULL DEFAULT 0 COMMENT 'lower priority carriers are attempt
 PRIMARY KEY (lcr_carrier_set_entry_sid)
 ) COMMENT='An entry in the LCR routing list';
 
+CREATE TABLE webhooks
+(
+webhook_sid CHAR(36) NOT NULL UNIQUE ,
+url VARCHAR(1024) NOT NULL,
+method ENUM("GET","POST") NOT NULL DEFAULT 'POST',
+username VARCHAR(255),
+password VARCHAR(255),
+PRIMARY KEY (webhook_sid)
+) COMMENT='An HTTP callback';
+
 CREATE TABLE applications
 (
 application_sid CHAR(36) NOT NULL UNIQUE ,
 name VARCHAR(64) NOT NULL,
-account_sid CHAR(36) NOT NULL COMMENT 'account that this application belongs to',
+service_provider_sid CHAR(36) COMMENT 'if non-null, this application is a test application that can be used by any account under the associated service provider',
+account_sid CHAR(36) COMMENT 'account that this application belongs to (if null, this is a service provider test application)',
 call_hook_sid CHAR(36) COMMENT 'webhook to call for inbound calls ',
 call_status_hook_sid CHAR(36) COMMENT 'webhook to call for call status events',
 messaging_hook_sid CHAR(36) COMMENT 'webhook to call for inbound SMS/MMS ',
+app_json TEXT,
 speech_synthesis_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
 speech_synthesis_language VARCHAR(12) NOT NULL DEFAULT 'en-US',
-speech_synthesis_voice VARCHAR(64),
+speech_synthesis_voice VARCHAR(256),
+speech_synthesis_label VARCHAR(64),
 speech_recognizer_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
 speech_recognizer_language VARCHAR(64) NOT NULL DEFAULT 'en-US',
+speech_recognizer_label VARCHAR(64),
+use_for_fallback_speech BOOLEAN DEFAULT false,
+fallback_speech_synthesis_vendor VARCHAR(64),
+fallback_speech_synthesis_language VARCHAR(12),
+fallback_speech_synthesis_voice VARCHAR(256),
+fallback_speech_synthesis_label VARCHAR(64),
+fallback_speech_recognizer_vendor VARCHAR(64),
+fallback_speech_recognizer_language VARCHAR(64),
+fallback_speech_recognizer_label VARCHAR(64),
+created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+record_all_calls BOOLEAN NOT NULL DEFAULT false,
 PRIMARY KEY (application_sid)
-) ENGINE=InnoDB COMMENT='A defined set of behaviors to be applied to phone calls ';
+) COMMENT='A defined set of behaviors to be applied to phone calls ';
 
 CREATE TABLE service_providers
 (
@@ -168,7 +524,7 @@ root_domain VARCHAR(128) UNIQUE ,
 registration_hook_sid CHAR(36),
 ms_teams_fqdn VARCHAR(255),
 PRIMARY KEY (service_provider_sid)
-) ENGINE=InnoDB COMMENT='A partition of the platform used by one service provider';
+) COMMENT='A partition of the platform used by one service provider';
 
 CREATE TABLE accounts
 (
@@ -177,67 +533,184 @@ name VARCHAR(64) NOT NULL,
 sip_realm VARCHAR(132) UNIQUE  COMMENT 'sip domain that will be used for devices registering under this account',
 service_provider_sid CHAR(36) NOT NULL COMMENT 'service provider that owns the customer relationship with this account',
 registration_hook_sid CHAR(36) COMMENT 'webhook to call when devices underr this account attempt to register',
+queue_event_hook_sid CHAR(36),
 device_calling_application_sid CHAR(36) COMMENT 'application to use for outbound calling from an account',
 is_active BOOLEAN NOT NULL DEFAULT true,
+created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+plan_type ENUM('trial','free','paid') NOT NULL DEFAULT 'trial',
+stripe_customer_id VARCHAR(56),
+webhook_secret VARCHAR(36) NOT NULL,
+disable_cdrs BOOLEAN NOT NULL DEFAULT 0,
+trial_end_date DATETIME,
+deactivated_reason VARCHAR(255),
+device_to_call_ratio INTEGER NOT NULL DEFAULT 5,
+subspace_client_id VARCHAR(255),
+subspace_client_secret VARCHAR(255),
+subspace_sip_teleport_id VARCHAR(255),
+subspace_sip_teleport_destinations VARCHAR(255),
+siprec_hook_sid CHAR(36),
+record_all_calls BOOLEAN NOT NULL DEFAULT false,
+record_format VARCHAR(16) NOT NULL DEFAULT 'mp3',
+bucket_credential VARCHAR(8192) COMMENT 'credential used to authenticate with storage service',
 PRIMARY KEY (account_sid)
-) ENGINE=InnoDB COMMENT='An enterprise that uses the platform for comm services';
+) COMMENT='An enterprise that uses the platform for comm services';
 
+CREATE INDEX account_static_ip_sid_idx ON account_static_ips (account_static_ip_sid);
+CREATE INDEX account_sid_idx ON account_static_ips (account_sid);
+ALTER TABLE account_static_ips ADD FOREIGN KEY account_sid_idxfk (account_sid) REFERENCES accounts (account_sid);
+
+CREATE INDEX account_sid_idx ON account_limits (account_sid);
+ALTER TABLE account_limits ADD FOREIGN KEY account_sid_idxfk_1 (account_sid) REFERENCES accounts (account_sid) ON DELETE CASCADE;
+
+CREATE INDEX account_subscription_sid_idx ON account_subscriptions (account_subscription_sid);
+CREATE INDEX account_sid_idx ON account_subscriptions (account_sid);
+ALTER TABLE account_subscriptions ADD FOREIGN KEY account_sid_idxfk_2 (account_sid) REFERENCES accounts (account_sid);
+
+CREATE INDEX invite_code_idx ON beta_invite_codes (invite_code);
 CREATE INDEX call_route_sid_idx ON call_routes (call_route_sid);
-ALTER TABLE call_routes ADD FOREIGN KEY account_sid_idxfk (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE call_routes ADD FOREIGN KEY account_sid_idxfk_3 (account_sid) REFERENCES accounts (account_sid);
 
 ALTER TABLE call_routes ADD FOREIGN KEY application_sid_idxfk (application_sid) REFERENCES applications (application_sid);
 
+CREATE INDEX client_sid_idx ON clients (client_sid);
+ALTER TABLE clients ADD CONSTRAINT account_sid_idxfk_13 FOREIGN KEY account_sid_idxfk_13 (account_sid) REFERENCES accounts (account_sid);
+
+CREATE INDEX dns_record_sid_idx ON dns_records (dns_record_sid);
+ALTER TABLE dns_records ADD FOREIGN KEY account_sid_idxfk_4 (account_sid) REFERENCES accounts (account_sid);
+
+CREATE INDEX lcr_sid_idx ON lcr_routes (lcr_sid);
+ALTER TABLE lcr_routes ADD FOREIGN KEY lcr_sid_idxfk (lcr_sid) REFERENCES lcr (lcr_sid);
+
+CREATE INDEX lcr_sid_idx ON lcr (lcr_sid);
+ALTER TABLE lcr ADD FOREIGN KEY default_carrier_set_entry_sid_idxfk (default_carrier_set_entry_sid) REFERENCES lcr_carrier_set_entry (lcr_carrier_set_entry_sid);
+
+CREATE INDEX service_provider_sid_idx ON lcr (service_provider_sid);
+CREATE INDEX account_sid_idx ON lcr (account_sid);
+CREATE INDEX permission_sid_idx ON permissions (permission_sid);
+CREATE INDEX predefined_carrier_sid_idx ON predefined_carriers (predefined_carrier_sid);
+CREATE INDEX predefined_sip_gateway_sid_idx ON predefined_sip_gateways (predefined_sip_gateway_sid);
+CREATE INDEX predefined_carrier_sid_idx ON predefined_sip_gateways (predefined_carrier_sid);
+ALTER TABLE predefined_sip_gateways ADD FOREIGN KEY predefined_carrier_sid_idxfk (predefined_carrier_sid) REFERENCES predefined_carriers (predefined_carrier_sid);
+
+CREATE INDEX predefined_smpp_gateway_sid_idx ON predefined_smpp_gateways (predefined_smpp_gateway_sid);
+CREATE INDEX predefined_carrier_sid_idx ON predefined_smpp_gateways (predefined_carrier_sid);
+ALTER TABLE predefined_smpp_gateways ADD FOREIGN KEY predefined_carrier_sid_idxfk_1 (predefined_carrier_sid) REFERENCES predefined_carriers (predefined_carrier_sid);
+
+CREATE INDEX product_sid_idx ON products (product_sid);
+CREATE INDEX account_product_sid_idx ON account_products (account_product_sid);
+CREATE INDEX account_subscription_sid_idx ON account_products (account_subscription_sid);
+ALTER TABLE account_products ADD FOREIGN KEY account_subscription_sid_idxfk (account_subscription_sid) REFERENCES account_subscriptions (account_subscription_sid);
+
+ALTER TABLE account_products ADD FOREIGN KEY product_sid_idxfk (product_sid) REFERENCES products (product_sid);
+
+CREATE INDEX account_offer_sid_idx ON account_offers (account_offer_sid);
+CREATE INDEX account_sid_idx ON account_offers (account_sid);
+ALTER TABLE account_offers ADD FOREIGN KEY account_sid_idxfk_5 (account_sid) REFERENCES accounts (account_sid);
+
+CREATE INDEX product_sid_idx ON account_offers (product_sid);
+ALTER TABLE account_offers ADD FOREIGN KEY product_sid_idxfk_1 (product_sid) REFERENCES products (product_sid);
+
 CREATE INDEX api_key_sid_idx ON api_keys (api_key_sid);
 CREATE INDEX account_sid_idx ON api_keys (account_sid);
-ALTER TABLE api_keys ADD FOREIGN KEY account_sid_idxfk_1 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE api_keys ADD FOREIGN KEY account_sid_idxfk_6 (account_sid) REFERENCES accounts (account_sid);
 
 CREATE INDEX service_provider_sid_idx ON api_keys (service_provider_sid);
 ALTER TABLE api_keys ADD FOREIGN KEY service_provider_sid_idxfk (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
-CREATE INDEX ms_teams_tenant_sid_idx ON ms_teams_tenants (ms_teams_tenant_sid);
-ALTER TABLE ms_teams_tenants ADD FOREIGN KEY service_provider_sid_idxfk_1 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
-
-ALTER TABLE ms_teams_tenants ADD FOREIGN KEY account_sid_idxfk_2 (account_sid) REFERENCES accounts (account_sid);
-
-ALTER TABLE ms_teams_tenants ADD FOREIGN KEY application_sid_idxfk_1 (application_sid) REFERENCES applications (application_sid);
-
-CREATE INDEX tenant_fqdn_idx ON ms_teams_tenants (tenant_fqdn);
 CREATE INDEX sbc_addresses_idx_host_port ON sbc_addresses (ipv4,port);
 
 CREATE INDEX sbc_address_sid_idx ON sbc_addresses (sbc_address_sid);
 CREATE INDEX service_provider_sid_idx ON sbc_addresses (service_provider_sid);
-ALTER TABLE sbc_addresses ADD FOREIGN KEY service_provider_sid_idxfk_2 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+ALTER TABLE sbc_addresses ADD FOREIGN KEY service_provider_sid_idxfk_1 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+
+CREATE INDEX ms_teams_tenant_sid_idx ON ms_teams_tenants (ms_teams_tenant_sid);
+ALTER TABLE ms_teams_tenants ADD FOREIGN KEY service_provider_sid_idxfk_2 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+
+ALTER TABLE ms_teams_tenants ADD FOREIGN KEY account_sid_idxfk_7 (account_sid) REFERENCES accounts (account_sid);
+
+ALTER TABLE ms_teams_tenants ADD FOREIGN KEY application_sid_idxfk_1 (application_sid) REFERENCES applications (application_sid);
+
+CREATE INDEX tenant_fqdn_idx ON ms_teams_tenants (tenant_fqdn);
+CREATE INDEX service_provider_sid_idx ON service_provider_limits (service_provider_sid);
+ALTER TABLE service_provider_limits ADD FOREIGN KEY service_provider_sid_idxfk_3 (service_provider_sid) REFERENCES service_providers (service_provider_sid) ON DELETE CASCADE;
+
+CREATE INDEX email_idx ON signup_history (email);
+CREATE INDEX smpp_address_sid_idx ON smpp_addresses (smpp_address_sid);
+CREATE INDEX service_provider_sid_idx ON smpp_addresses (service_provider_sid);
+ALTER TABLE smpp_addresses ADD FOREIGN KEY service_provider_sid_idxfk_4 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+
+CREATE INDEX speech_credential_sid_idx ON speech_credentials (speech_credential_sid);
+CREATE INDEX service_provider_sid_idx ON speech_credentials (service_provider_sid);
+ALTER TABLE speech_credentials ADD FOREIGN KEY service_provider_sid_idxfk_5 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+
+CREATE INDEX account_sid_idx ON speech_credentials (account_sid);
+ALTER TABLE speech_credentials ADD FOREIGN KEY account_sid_idxfk_8 (account_sid) REFERENCES accounts (account_sid);
+
+CREATE INDEX google_custom_voice_sid_idx ON google_custom_voices (google_custom_voice_sid);
+CREATE INDEX speech_credential_sid_idx ON google_custom_voices (speech_credential_sid);
+ALTER TABLE google_custom_voices ADD FOREIGN KEY speech_credential_sid_idxfk (speech_credential_sid) REFERENCES speech_credentials (speech_credential_sid) ON DELETE CASCADE;
 
 CREATE INDEX user_sid_idx ON users (user_sid);
-CREATE INDEX name_idx ON users (name);
+CREATE INDEX email_idx ON users (email);
+CREATE INDEX phone_idx ON users (phone);
+CREATE INDEX account_sid_idx ON users (account_sid);
+ALTER TABLE users ADD FOREIGN KEY account_sid_idxfk_9 (account_sid) REFERENCES accounts (account_sid);
+
+CREATE INDEX service_provider_sid_idx ON users (service_provider_sid);
+ALTER TABLE users ADD FOREIGN KEY service_provider_sid_idxfk_6 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+
+CREATE INDEX email_activation_code_idx ON users (email_activation_code);
 CREATE INDEX voip_carrier_sid_idx ON voip_carriers (voip_carrier_sid);
-CREATE INDEX name_idx ON voip_carriers (name);
-ALTER TABLE voip_carriers ADD FOREIGN KEY account_sid_idxfk_3 (account_sid) REFERENCES accounts (account_sid);
+CREATE INDEX account_sid_idx ON voip_carriers (account_sid);
+ALTER TABLE voip_carriers ADD FOREIGN KEY account_sid_idxfk_10 (account_sid) REFERENCES accounts (account_sid);
+
+CREATE INDEX service_provider_sid_idx ON voip_carriers (service_provider_sid);
+ALTER TABLE voip_carriers ADD FOREIGN KEY service_provider_sid_idxfk_7 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
 ALTER TABLE voip_carriers ADD FOREIGN KEY application_sid_idxfk_2 (application_sid) REFERENCES applications (application_sid);
 
-CREATE INDEX phone_number_sid_idx ON phone_numbers (phone_number_sid);
-CREATE INDEX voip_carrier_sid_idx ON phone_numbers (voip_carrier_sid);
-ALTER TABLE phone_numbers ADD FOREIGN KEY voip_carrier_sid_idxfk (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid);
+CREATE INDEX user_permissions_sid_idx ON user_permissions (user_permissions_sid);
+CREATE INDEX user_sid_idx ON user_permissions (user_sid);
+ALTER TABLE user_permissions ADD FOREIGN KEY user_sid_idxfk (user_sid) REFERENCES users (user_sid) ON DELETE CASCADE;
 
-ALTER TABLE phone_numbers ADD FOREIGN KEY account_sid_idxfk_4 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE user_permissions ADD FOREIGN KEY permission_sid_idxfk (permission_sid) REFERENCES permissions (permission_sid);
+
+CREATE INDEX smpp_gateway_sid_idx ON smpp_gateways (smpp_gateway_sid);
+CREATE INDEX voip_carrier_sid_idx ON smpp_gateways (voip_carrier_sid);
+ALTER TABLE smpp_gateways ADD FOREIGN KEY voip_carrier_sid_idxfk (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid);
+
+CREATE UNIQUE INDEX phone_numbers_unique_idx_voip_carrier_number ON phone_numbers (number,voip_carrier_sid);
+
+CREATE INDEX phone_number_sid_idx ON phone_numbers (phone_number_sid);
+CREATE INDEX number_idx ON phone_numbers (number);
+CREATE INDEX voip_carrier_sid_idx ON phone_numbers (voip_carrier_sid);
+ALTER TABLE phone_numbers ADD FOREIGN KEY voip_carrier_sid_idxfk_1 (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid);
+
+ALTER TABLE phone_numbers ADD FOREIGN KEY account_sid_idxfk_11 (account_sid) REFERENCES accounts (account_sid);
 
 ALTER TABLE phone_numbers ADD FOREIGN KEY application_sid_idxfk_3 (application_sid) REFERENCES applications (application_sid);
 
-CREATE INDEX webhook_sid_idx ON webhooks (webhook_sid);
-CREATE UNIQUE INDEX sip_gateway_idx_hostport ON sip_gateways (ipv4,port);
+CREATE INDEX service_provider_sid_idx ON phone_numbers (service_provider_sid);
+ALTER TABLE phone_numbers ADD FOREIGN KEY service_provider_sid_idxfk_8 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
-ALTER TABLE sip_gateways ADD FOREIGN KEY voip_carrier_sid_idxfk_1 (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid);
+CREATE INDEX sip_gateway_idx_hostport ON sip_gateways (ipv4,port);
+
+CREATE INDEX voip_carrier_sid_idx ON sip_gateways (voip_carrier_sid);
+ALTER TABLE sip_gateways ADD FOREIGN KEY voip_carrier_sid_idxfk_2 (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid);
 
 ALTER TABLE lcr_carrier_set_entry ADD FOREIGN KEY lcr_route_sid_idxfk (lcr_route_sid) REFERENCES lcr_routes (lcr_route_sid);
 
-ALTER TABLE lcr_carrier_set_entry ADD FOREIGN KEY voip_carrier_sid_idxfk_2 (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid);
+ALTER TABLE lcr_carrier_set_entry ADD FOREIGN KEY voip_carrier_sid_idxfk_3 (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid);
 
+CREATE INDEX webhook_sid_idx ON webhooks (webhook_sid);
 CREATE UNIQUE INDEX applications_idx_name ON applications (account_sid,name);
 
 CREATE INDEX application_sid_idx ON applications (application_sid);
+CREATE INDEX service_provider_sid_idx ON applications (service_provider_sid);
+ALTER TABLE applications ADD FOREIGN KEY service_provider_sid_idxfk_9 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+
 CREATE INDEX account_sid_idx ON applications (account_sid);
-ALTER TABLE applications ADD FOREIGN KEY account_sid_idxfk_5 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE applications ADD FOREIGN KEY account_sid_idxfk_12 (account_sid) REFERENCES accounts (account_sid);
 
 ALTER TABLE applications ADD FOREIGN KEY call_hook_sid_idxfk (call_hook_sid) REFERENCES webhooks (webhook_sid);
 
@@ -253,10 +726,14 @@ ALTER TABLE service_providers ADD FOREIGN KEY registration_hook_sid_idxfk (regis
 CREATE INDEX account_sid_idx ON accounts (account_sid);
 CREATE INDEX sip_realm_idx ON accounts (sip_realm);
 CREATE INDEX service_provider_sid_idx ON accounts (service_provider_sid);
-ALTER TABLE accounts ADD FOREIGN KEY service_provider_sid_idxfk_3 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+ALTER TABLE accounts ADD FOREIGN KEY service_provider_sid_idxfk_10 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
 ALTER TABLE accounts ADD FOREIGN KEY registration_hook_sid_idxfk_1 (registration_hook_sid) REFERENCES webhooks (webhook_sid);
 
+ALTER TABLE accounts ADD FOREIGN KEY queue_event_hook_sid_idxfk (queue_event_hook_sid) REFERENCES webhooks (webhook_sid);
+
 ALTER TABLE accounts ADD FOREIGN KEY device_calling_application_sid_idxfk (device_calling_application_sid) REFERENCES applications (application_sid);
 
+ALTER TABLE accounts ADD FOREIGN KEY siprec_hook_sid_idxfk (siprec_hook_sid) REFERENCES applications (application_sid);
+
 SET FOREIGN_KEY_CHECKS=1;

db/reset_admin_password.js

@@ -1,71 +1,59 @@
 #!/usr/bin/env node
-const {getMysqlConnection} = require('../lib/db');
-const crypto = require('crypto');
-const uuidv4 = require('uuid/v4');
+const {promisePool} = require('../lib/db');
+const { v4: uuidv4 } = require('uuid');
+const {generateHashedPassword} = require('../lib/utils/password-utils');
 const sqlInsert = `INSERT into users 
-(user_sid, name, hashed_password, salt) 
-values (?, ?, ?, ?)
+(user_sid, name, email, hashed_password, force_change, provider, email_validated) 
+values (?, ?, ?, ?, ?, ?, ?)
 `;
-const sqlChangeAdminToken = `UPDATE api_keys set token = ? 
-WHERE account_sid IS NULL 
-AND service_provider_sid IS NULL`;
+const sqlInsertAdminToken = `INSERT into api_keys 
+(api_key_sid, token) 
+values (?, ?)`;
+const sqlQueryAccount = 'SELECT * from accounts LEFT JOIN api_keys ON api_keys.account_sid = accounts.account_sid';
+const sqlAddAccountToken = `INSERT into api_keys (api_key_sid, token, account_sid) 
+VALUES (?, ?, ?)`;
+const sqlInsertPermissions = `
+INSERT into user_permissions (user_permissions_sid, user_sid, permission_sid) 
+VALUES (?,?,?)`;
 
-/**
- * generates random string of characters i.e salt
- * @function
- * @param {number} length - Length of the random string.
- */
-const genRandomString = (len) => {
-  return crypto.randomBytes(Math.ceil(len / 2))
-    .toString('hex') /** convert to hexadecimal format */
-    .slice(0, len);   /** return required number of characters */
-};
+const password = process.env.JAMBONES_ADMIN_INITIAL_PASSWORD || 'admin';
+console.log(`reset_admin_password, initial admin password is ${password}`);
 
-/**
- * hash password with sha512.
- * @function
- * @param {string} password - List of required fields.
- * @param {string} salt - Data to be validated.
- */
-const sha512 = function(password, salt) {
-  const hash = crypto.createHmac('sha512', salt); /** Hashing algorithm sha512 */
-  hash.update(password);
-  var value = hash.digest('hex');
-  return {
-    salt:salt,
-    passwordHash:value
-  };
-};
+const doIt = async() => {
+  const passwordHash = await generateHashedPassword(password);
+  const sid = uuidv4();
+  await promisePool.execute('DELETE from users where name = "admin"');
+  await promisePool.execute('DELETE from api_keys where account_sid is null and service_provider_sid is null');
 
-const saltHashPassword = (userpassword) => {
-  var salt = genRandomString(16); /** Gives us salt of length 16 */
-  return sha512(userpassword, salt);
-};
-
-/* reset admin password */
-getMysqlConnection((err, conn) => {
-  if (err) return console.log(err, 'Error connecting to database');
-
-  /* delete admin user if it exists */
-  conn.query('DELETE from users where name = "admin"', (err) => {
-    if (err) return console.log(err, 'Error removing admin user');
-    const {salt, passwordHash} = saltHashPassword('admin');
-    const sid = uuidv4();
-    conn.query(sqlInsert, [
+  await promisePool.execute(sqlInsert,
+    [
       sid,
       'admin',
+      'joe@foo.bar',
       passwordHash,
-      salt
-    ], (err) => {
-      if (err) return console.log(err, 'Error inserting admin user');
-      console.log('successfully reset admin password');
-      const uuid = uuidv4();
-      conn.query(sqlChangeAdminToken, [uuid], (err) => {
-        if (err) return console.log(err, 'Error updating admin token');
-        console.log('successfully changed admin tokens');
-        conn.release();
-        process.exit(0);
-      });
-    });
-  });
-});
+      1,
+      'local',
+      1
+    ]
+  );
+  await promisePool.execute(sqlInsertAdminToken, [uuidv4(), uuidv4()]);
+
+  /* assign all permissions to the admin user */
+  const [p] = await promisePool.query('SELECT * from permissions');
+  for (const perm of p) {
+    await promisePool.execute(sqlInsertPermissions, [uuidv4(), sid, perm.permission_sid]);
+  }
+
+  /* create admin token for single account */
+  const [r] = await promisePool.query({sql: sqlQueryAccount, nestTables: true});
+  if (1 === r.length && r[0].api_keys.api_key_sid === null) {
+    const api_key_sid = uuidv4();
+    const token = uuidv4();
+    const {account_sid} = r[0].accounts;
+    await promisePool.execute(sqlAddAccountToken, [api_key_sid, token, account_sid]);
+  }
+
+  process.exit(0);
+};
+
+doIt();

db/seed-integration-test.sql

@@ -0,0 +1,115 @@
+SET FOREIGN_KEY_CHECKS=0;
+
+-- create standard permissions 
+insert into permissions (permission_sid, name, description)
+values 
+('ffbc342a-546a-11ed-bdc3-0242ac120002', 'VIEW_ONLY', 'Can view data but not make changes'),
+('ffbc3a10-546a-11ed-bdc3-0242ac120002', 'PROVISION_SERVICES', 'Can provision services'),
+('ffbc3c5e-546a-11ed-bdc3-0242ac120002', 'PROVISION_USERS', 'Can provision users');
+
+insert into sbc_addresses (sbc_address_sid, ipv4, port) 
+values('f6567ae1-bf97-49af-8931-ca014b689995', '52.55.111.178', 5060);
+insert into sbc_addresses (sbc_address_sid, ipv4, port) 
+values('de5ed2f1-bccd-4600-a95e-cef46e9a3a4f', '3.34.102.122', 5060);
+insert into smpp_addresses (smpp_address_sid, ipv4, port, use_tls, is_primary) 
+values('de5ed2f1-bccd-4600-a95e-cef46e9a3a4f', '34.197.99.29', 2775, 0, 1);
+insert into smpp_addresses (smpp_address_sid, ipv4, port, use_tls, is_primary) 
+values('049078a0', '3.209.58.102', 3550, 1, 1);
+
+-- create one service provider and account
+insert into api_keys (api_key_sid, token) 
+values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36f0', '38700987-c7a4-4685-a5bb-af378f9734de');
+
+-- create one service provider and one account
+insert into service_providers (service_provider_sid, name, root_domain) 
+values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'default service provider', 'sip.jambonz.cloud');
+
+insert into accounts (account_sid, service_provider_sid, name, webhook_secret) 
+values ('9351f46a-678c-43f5-b8a6-d4eb58d131af','2708b1b3-2736-40ea-b502-c53d8396247f', 'default account', 'wh_secret_cJqgtMDPzDhhnjmaJH6Mtk');
+
+-- create account level api key
+insert into api_keys (api_key_sid, token, service_provider_sid) 
+values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36fa', '38700987-c7a4-4685-a5bb-af378f9734da', '9351f46a-678c-43f5-b8a6-d4eb58d131af');
+
+-- create SP level api key
+insert into api_keys (api_key_sid, token, account_sid) 
+values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36fs', '38700987-c7a4-4685-a5bb-af378f9734ds', '2708b1b3-2736-40ea-b502-c53d8396247f');
+
+-- create two applications
+insert into webhooks(webhook_sid, url, method)
+values 
+('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.cloud/call-status', 'POST'),
+('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.cloud/hello-world', 'POST'),
+('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.cloud/dial-time', 'POST');
+
+insert into applications (application_sid, account_sid, name, call_hook_sid, call_status_hook_sid, speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice, speech_recognizer_vendor, speech_recognizer_language)
+VALUES
+('7087fe50-8acb-4f3b-b820-97b573723aab', '9351f46a-678c-43f5-b8a6-d4eb58d131af', 'hello world', 'd31568d0-b193-4a05-8ff6-778369bc6efe', '84e3db00-b172-4e46-b54b-a503fdb19e4a', 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US'),
+('4ca2fb6a-8636-4f2e-96ff-8966c5e26f8e', '9351f46a-678c-43f5-b8a6-d4eb58d131af', 'dial time', '81844b05-714d-4295-8bf3-3b0640a4bf02', '84e3db00-b172-4e46-b54b-a503fdb19e4a', 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US');
+
+-- create our products
+insert into products (product_sid, name, category)
+values
+('c4403cdb-8e75-4b27-9726-7d8315e3216d', 'concurrent call session', 'voice_call_session'),
+('2c815913-5c26-4004-b748-183b459329df', 'registered device', 'device'),
+('35a9fb10-233d-4eb9-aada-78de5814d680', 'api call', 'api_rate');
+
+-- create predefined carriers
+insert into predefined_carriers (predefined_carrier_sid, name, requires_static_ip, e164_leading_plus, 
+requires_register, register_username, register_password, 
+register_sip_realm, tech_prefix, inbound_auth_username, inbound_auth_password, diversion)
+VALUES
+('17479288-bb9f-421a-89d1-f4ac57af1dca', 'TelecomsXChange', 0, 0, 0, NULL, NULL, NULL, 'your-tech-prefix', NULL, NULL, NULL),
+('7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', 'Twilio', 0, 1, 0, '<your-twilio-credential-username>', '<your-twilio-credential-password>', NULL, NULL, NULL, NULL, NULL),
+('032d90d5-39e8-41c0-b807-9c88cffba65c', 'Voxbone', 0, 1, 0, '<your-voxbone-outbound-username>', '<your-voxbone-outbound-password>', NULL, NULL, NULL, NULL, '<your-voxbone-DID>'),
+('e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'Simwood', 0, 1, 0, '<your-simwood-auth-trunk-username>',  '<your-simwood-auth-trunk-password>', NULL, NULL, NULL, NULL, NULL);
+
+-- TelecomXchange gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('c9c3643e-9a83-4b78-b172-9c09d911bef5', '17479288-bb9f-421a-89d1-f4ac57af1dca', '174.136.44.213', 32, 5060, 1, 0),
+('3b5b7fa5-4e61-4423-b921-05c3283b2101', '17479288-bb9f-421a-89d1-f4ac57af1dca', 'sip01.TelecomsXChange.com', 32, 5060, 0, 1);
+
+insert into predefined_smpp_gateways (predefined_smpp_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('9b72467a-cfe3-491f-80bf-652c38e666b9', '17479288-bb9f-421a-89d1-f4ac57af1dca', 'smpp01.telecomsxchange.com', 32, 2776, 0, 1),
+('d22883b9-f124-4a89-bab2-4487cf783f64', '17479288-bb9f-421a-89d1-f4ac57af1dca', '174.136.44.11', 32, 2775, 1, 0),
+('fdcf7f1e-1f5f-487b-afb3-c0f75ed0aa3d', '17479288-bb9f-421a-89d1-f4ac57af1dca', '174.136.44.213', 32, 2775, 1, 0);
+
+-- twilio gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('d2ccfcb1-9198-4fe9-a0ca-6e49395837c4', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.172.60.0', 30, 5060, 1, 0),
+('6b1d0032-4430-41f1-87c6-f22233d394ef', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.244.51.0', 30, 5060, 1, 0),
+('0de40217-8bd5-4aa8-a9fd-1994282953c6', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.192', 30, 5060, 1, 0),
+('37bc0b20-b53c-4c31-95a6-f82b1c3713e3', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '35.156.191.128', 30, 5060, 1, 0),
+('39791f4e-b612-4882-a37e-e92711a39f3f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.65.63.192', 30, 5060, 1, 0),
+('81a0c8cb-a33e-42da-8f20-99083da6f02f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.252.254.64', 30, 5060, 1, 0),
+('eeeef07a-46b8-4ffe-a4f2-04eb32ca889e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.169.127.128', 30, 5060, 1, 0),
+('fbb6c194-4b68-4dff-9b42-52412be1c39e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '177.71.206.192', 30, 5060, 1, 0),
+('973e7824-0cf3-4645-88e4-d2460ddb8577', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '168.86.128.0', 18, 5060, 1, 0),
+('3ed1dd12-e1a7-44ff-811a-3cc5dc13dc72', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '<your-domain>.pstn.twilio.com', 32, 5060, 0, 1);
+
+-- voxbone gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('d531c582-2103-42a0-b9f0-f80c215b3ec5', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.83.45', 32, 5060, 1, 0),
+('95c888e5-c959-4d92-82c4-8597dddff75e', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.86.45', 32, 5060, 1, 0),
+('1de3b2a1-96f0-407a-bcc4-ce371d823a8d', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.82.45', 32, 5060, 1, 0),
+('50c1f91a-6080-4495-a241-6bba6e9d9688', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.85.45', 32, 5060, 1, 0),
+('e6ebad33-80d5-4dbb-bc6f-a7ae08160cc6', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.84.45', 32, 5060, 1, 0),
+('7bae60b3-4237-4baa-a711-30ea3bce19d8', '032d90d5-39e8-41c0-b807-9c88cffba65c', '185.47.148.45', 32, 5060, 1, 0),
+('bc933522-18a2-47d8-9ae4-9faa8de4e927', '032d90d5-39e8-41c0-b807-9c88cffba65c', 'outbound.voxbone.com', 32, 5060, 0, 1);
+
+-- simwood gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('91cb050f-9826-4ac9-b736-84a10372a9fe', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.139.77', 32, 5060, 1, 0),
+('58700fad-98bf-4d31-b61e-888c54911b35', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.140.77', 32, 5060, 1, 0),
+('d020fd9e-7fdb-4bca-ae0d-e61b38142873', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.142.77', 32, 5060, 1, 0),
+('38d8520a-527f-4f8e-8456-f9dfca742561', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.77', 32, 5060, 1, 0),
+('834f8b0c-d4c2-4f3e-93d9-cf307995eedd', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.88', 32, 5060, 1, 0),
+('5f431d42-48e4-44ce-a311-d946f0b475b6', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'out.simwood.com', 32, 5060, 0, 1);
+
+
+SET FOREIGN_KEY_CHECKS=1;

db/seed-production-database-open-source.sql

@@ -0,0 +1,99 @@
+SET FOREIGN_KEY_CHECKS=0;
+
+-- create standard permissions 
+insert into permissions (permission_sid, name, description)
+values 
+('ffbc342a-546a-11ed-bdc3-0242ac120002', 'VIEW_ONLY', 'Can view data but not make changes'),
+('ffbc3a10-546a-11ed-bdc3-0242ac120002', 'PROVISION_SERVICES', 'Can provision services'),
+('ffbc3c5e-546a-11ed-bdc3-0242ac120002', 'PROVISION_USERS', 'Can provision users');
+
+-- create one service provider and account
+insert into api_keys (api_key_sid, token) 
+values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36f0', '38700987-c7a4-4685-a5bb-af378f9734de');
+
+-- create one service provider and one account
+insert into service_providers (service_provider_sid, name) 
+values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'default service provider');
+
+insert into accounts (account_sid, service_provider_sid, name, webhook_secret) 
+values ('9351f46a-678c-43f5-b8a6-d4eb58d131af','2708b1b3-2736-40ea-b502-c53d8396247f', 'default account', 'wh_secret_cJqgtMDPzDhhnjmaJH6Mtk');
+
+insert into api_keys (api_key_sid, token, account_sid) 
+values ('09e92f3c-9d73-4303-b63f-3668574862ce', '1cf2f4f4-64c4-4249-9a3e-5bb4cb597c2a', '9351f46a-678c-43f5-b8a6-d4eb58d131af');
+
+-- create two applications
+insert into webhooks(webhook_sid, url, method)
+values 
+('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.cloud/call-status', 'POST'),
+('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.cloud/hello-world', 'POST'),
+('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.cloud/dial-time', 'POST');
+insert into applications (application_sid, account_sid, name, call_hook_sid, call_status_hook_sid, speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice, speech_recognizer_vendor, speech_recognizer_language)
+VALUES
+('7087fe50-8acb-4f3b-b820-97b573723aab', '9351f46a-678c-43f5-b8a6-d4eb58d131af', 'hello world', 'd31568d0-b193-4a05-8ff6-778369bc6efe', '84e3db00-b172-4e46-b54b-a503fdb19e4a', 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US'),
+('4ca2fb6a-8636-4f2e-96ff-8966c5e26f8e', '9351f46a-678c-43f5-b8a6-d4eb58d131af', 'dial time', '81844b05-714d-4295-8bf3-3b0640a4bf02', '84e3db00-b172-4e46-b54b-a503fdb19e4a', 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US');
+
+-- create our products
+insert into products (product_sid, name, category)
+values
+('c4403cdb-8e75-4b27-9726-7d8315e3216d', 'concurrent call session', 'voice_call_session'),
+('2c815913-5c26-4004-b748-183b459329df', 'registered device', 'device'),
+('35a9fb10-233d-4eb9-aada-78de5814d680', 'api call', 'api_rate');
+
+-- create predefined carriers
+insert into predefined_carriers (predefined_carrier_sid, name, requires_static_ip, e164_leading_plus, 
+requires_register, register_username, register_password, 
+register_sip_realm, tech_prefix, inbound_auth_username, inbound_auth_password, diversion)
+VALUES
+('17479288-bb9f-421a-89d1-f4ac57af1dca', 'TelecomsXChange', 0, 0, 0, NULL, NULL, NULL, 'your-tech-prefix', NULL, NULL, NULL),
+('7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', 'Twilio', 0, 1, 0, '<your-twilio-credential-username>', '<your-twilio-credential-password>', NULL, NULL, NULL, NULL, NULL),
+('032d90d5-39e8-41c0-b807-9c88cffba65c', 'Voxbone', 0, 1, 0, '<your-voxbone-outbound-username>', '<your-voxbone-outbound-password>', NULL, NULL, NULL, NULL, '<your-voxbone-DID>'),
+('e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'Simwood', 0, 1, 0, '<your-simwood-auth-trunk-username>',  '<your-simwood-auth-trunk-password>', NULL, NULL, NULL, NULL, NULL);
+
+-- TelecomXchange gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('c9c3643e-9a83-4b78-b172-9c09d911bef5', '17479288-bb9f-421a-89d1-f4ac57af1dca', '174.136.44.213', 32, 5060, 1, 0),
+('3b5b7fa5-4e61-4423-b921-05c3283b2101', '17479288-bb9f-421a-89d1-f4ac57af1dca', 'sip01.TelecomsXChange.com', 32, 5060, 0, 1);
+
+insert into predefined_smpp_gateways (predefined_smpp_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('9b72467a-cfe3-491f-80bf-652c38e666b9', '17479288-bb9f-421a-89d1-f4ac57af1dca', 'smpp01.telecomsxchange.com', 32, 2776, 0, 1),
+('d22883b9-f124-4a89-bab2-4487cf783f64', '17479288-bb9f-421a-89d1-f4ac57af1dca', '174.136.44.11', 32, 2775, 1, 0),
+('fdcf7f1e-1f5f-487b-afb3-c0f75ed0aa3d', '17479288-bb9f-421a-89d1-f4ac57af1dca', '174.136.44.213', 32, 2775, 1, 0);
+
+-- twilio gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('d2ccfcb1-9198-4fe9-a0ca-6e49395837c4', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.172.60.0', 30, 5060, 1, 0),
+('6b1d0032-4430-41f1-87c6-f22233d394ef', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.244.51.0', 30, 5060, 1, 0),
+('0de40217-8bd5-4aa8-a9fd-1994282953c6', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.192', 30, 5060, 1, 0),
+('37bc0b20-b53c-4c31-95a6-f82b1c3713e3', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '35.156.191.128', 30, 5060, 1, 0),
+('39791f4e-b612-4882-a37e-e92711a39f3f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.65.63.192', 30, 5060, 1, 0),
+('81a0c8cb-a33e-42da-8f20-99083da6f02f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.252.254.64', 30, 5060, 1, 0),
+('eeeef07a-46b8-4ffe-a4f2-04eb32ca889e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.169.127.128', 30, 5060, 1, 0),
+('fbb6c194-4b68-4dff-9b42-52412be1c39e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '177.71.206.192', 30, 5060, 1, 0),
+('973e7824-0cf3-4645-88e4-d2460ddb8577', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '168.86.128.0', 18, 5060, 1, 0),
+('3ed1dd12-e1a7-44ff-811a-3cc5dc13dc72', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '<your-domain>.pstn.twilio.com', 32, 5060, 0, 1);
+
+-- voxbone gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('d531c582-2103-42a0-b9f0-f80c215b3ec5', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.83.45', 32, 5060, 1, 0),
+('95c888e5-c959-4d92-82c4-8597dddff75e', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.86.45', 32, 5060, 1, 0),
+('1de3b2a1-96f0-407a-bcc4-ce371d823a8d', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.82.45', 32, 5060, 1, 0),
+('50c1f91a-6080-4495-a241-6bba6e9d9688', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.85.45', 32, 5060, 1, 0),
+('e6ebad33-80d5-4dbb-bc6f-a7ae08160cc6', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.84.45', 32, 5060, 1, 0),
+('7bae60b3-4237-4baa-a711-30ea3bce19d8', '032d90d5-39e8-41c0-b807-9c88cffba65c', '185.47.148.45', 32, 5060, 1, 0),
+('bc933522-18a2-47d8-9ae4-9faa8de4e927', '032d90d5-39e8-41c0-b807-9c88cffba65c', 'outbound.voxbone.com', 32, 5060, 0, 1);
+
+-- simwood gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('91cb050f-9826-4ac9-b736-84a10372a9fe', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.139.77', 32, 5060, 1, 0),
+('58700fad-98bf-4d31-b61e-888c54911b35', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.140.77', 32, 5060, 1, 0),
+('d020fd9e-7fdb-4bca-ae0d-e61b38142873', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.142.77', 32, 5060, 1, 0),
+('38d8520a-527f-4f8e-8456-f9dfca742561', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.77', 32, 5060, 1, 0),
+('834f8b0c-d4c2-4f3e-93d9-cf307995eedd', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.88', 32, 5060, 1, 0),
+('5f431d42-48e4-44ce-a311-d946f0b475b6', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'out.simwood.com', 32, 5060, 0, 1);
+
+SET FOREIGN_KEY_CHECKS=1;

db/seed-production-database.sql

@@ -0,0 +1,80 @@
+SET FOREIGN_KEY_CHECKS=0;
+
+-- create standard permissions 
+insert into permissions (permission_sid, name, description)
+values 
+('ffbc342a-546a-11ed-bdc3-0242ac120002', 'VIEW_ONLY', 'Can view data but not make changes'),
+('ffbc3a10-546a-11ed-bdc3-0242ac120002', 'PROVISION_SERVICES', 'Can provision services'),
+('ffbc3c5e-546a-11ed-bdc3-0242ac120002', 'PROVISION_USERS', 'Can provision users');
+
+-- create one service provider
+insert into service_providers (service_provider_sid, name, description, root_domain) 
+values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'sip.jambonz.xyz', 'jambonz.xyz service provider', 'sip.jambonz.xyz');
+insert into api_keys (api_key_sid, token) 
+values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36f0', '38700987-c7a4-4685-a5bb-af378f9734de');
+
+-- create our products
+insert into products (product_sid, name, category)
+values
+('c4403cdb-8e75-4b27-9726-7d8315e3216d', 'concurrent call session', 'voice_call_session'),
+('2c815913-5c26-4004-b748-183b459329df', 'registered device', 'device'),
+('35a9fb10-233d-4eb9-aada-78de5814d680', 'api call', 'api_rate');
+
+-- create predefined carriers
+insert into predefined_carriers (predefined_carrier_sid, name, requires_static_ip, e164_leading_plus, 
+requires_register, register_username, register_password, 
+register_sip_realm, tech_prefix, inbound_auth_username, inbound_auth_password, diversion)
+VALUES
+('17479288-bb9f-421a-89d1-f4ac57af1dca', 'TelecomsXChange', 0, 0, 0, NULL, NULL, NULL, 'your-tech-prefix', NULL, NULL, NULL),
+('7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', 'Twilio', 0, 1, 0, '<your-twilio-credential-username>', '<your-twilio-credential-password>', NULL, NULL, NULL, NULL, NULL),
+('032d90d5-39e8-41c0-b807-9c88cffba65c', 'Voxbone', 0, 1, 0, '<your-voxbone-outbound-username>', '<your-voxbone-outbound-password>', NULL, NULL, NULL, NULL, '<your-voxbone-DID>'),
+('e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'Simwood', 0, 1, 0, '<your-simwood-auth-trunk-username>',  '<your-simwood-auth-trunk-password>', NULL, NULL, NULL, NULL, NULL);
+
+-- TelecomXchange gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('c9c3643e-9a83-4b78-b172-9c09d911bef5', '17479288-bb9f-421a-89d1-f4ac57af1dca', '174.136.44.213', 32, 5060, 1, 0),
+('3b5b7fa5-4e61-4423-b921-05c3283b2101', '17479288-bb9f-421a-89d1-f4ac57af1dca', 'sip01.TelecomsXChange.com', 32, 5060, 0, 1);
+
+insert into predefined_smpp_gateways (predefined_smpp_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('9b72467a-cfe3-491f-80bf-652c38e666b9', '17479288-bb9f-421a-89d1-f4ac57af1dca', 'smpp01.telecomsxchange.com', 32, 2776, 0, 1),
+('d22883b9-f124-4a89-bab2-4487cf783f64', '17479288-bb9f-421a-89d1-f4ac57af1dca', '174.136.44.11', 32, 2775, 1, 0),
+('fdcf7f1e-1f5f-487b-afb3-c0f75ed0aa3d', '17479288-bb9f-421a-89d1-f4ac57af1dca', '174.136.44.213', 32, 2775, 1, 0);
+
+-- twilio gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('d2ccfcb1-9198-4fe9-a0ca-6e49395837c4', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.172.60.0', 30, 5060, 1, 0),
+('6b1d0032-4430-41f1-87c6-f22233d394ef', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.244.51.0', 30, 5060, 1, 0),
+('0de40217-8bd5-4aa8-a9fd-1994282953c6', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.192', 30, 5060, 1, 0),
+('37bc0b20-b53c-4c31-95a6-f82b1c3713e3', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '35.156.191.128', 30, 5060, 1, 0),
+('39791f4e-b612-4882-a37e-e92711a39f3f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.65.63.192', 30, 5060, 1, 0),
+('81a0c8cb-a33e-42da-8f20-99083da6f02f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.252.254.64', 30, 5060, 1, 0),
+('eeeef07a-46b8-4ffe-a4f2-04eb32ca889e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.169.127.128', 30, 5060, 1, 0),
+('fbb6c194-4b68-4dff-9b42-52412be1c39e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '177.71.206.192', 30, 5060, 1, 0),
+('973e7824-0cf3-4645-88e4-d2460ddb8577', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '168.86.128.0', 18, 5060, 1, 0),
+('3ed1dd12-e1a7-44ff-811a-3cc5dc13dc72', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '<your-domain>.pstn.twilio.com', 32, 5060, 0, 1);
+
+-- voxbone gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('d531c582-2103-42a0-b9f0-f80c215b3ec5', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.83.45', 32, 5060, 1, 0),
+('95c888e5-c959-4d92-82c4-8597dddff75e', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.86.45', 32, 5060, 1, 0),
+('1de3b2a1-96f0-407a-bcc4-ce371d823a8d', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.82.45', 32, 5060, 1, 0),
+('50c1f91a-6080-4495-a241-6bba6e9d9688', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.85.45', 32, 5060, 1, 0),
+('e6ebad33-80d5-4dbb-bc6f-a7ae08160cc6', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.84.45', 32, 5060, 1, 0),
+('7bae60b3-4237-4baa-a711-30ea3bce19d8', '032d90d5-39e8-41c0-b807-9c88cffba65c', '185.47.148.45', 32, 5060, 1, 0),
+('bc933522-18a2-47d8-9ae4-9faa8de4e927', '032d90d5-39e8-41c0-b807-9c88cffba65c', 'outbound.voxbone.com', 32, 5060, 0, 1);
+
+-- simwood gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('91cb050f-9826-4ac9-b736-84a10372a9fe', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.139.77', 32, 5060, 1, 0),
+('58700fad-98bf-4d31-b61e-888c54911b35', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.140.77', 32, 5060, 1, 0),
+('d020fd9e-7fdb-4bca-ae0d-e61b38142873', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.142.77', 32, 5060, 1, 0),
+('38d8520a-527f-4f8e-8456-f9dfca742561', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.77', 32, 5060, 1, 0),
+('834f8b0c-d4c2-4f3e-93d9-cf307995eedd', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.88', 32, 5060, 1, 0),
+('5f431d42-48e4-44ce-a311-d946f0b475b6', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'out.simwood.com', 32, 5060, 0, 1);
+
+SET FOREIGN_KEY_CHECKS=1;

db/upgrade-jambonz-db.js

@@ -0,0 +1,281 @@
+#!/usr/bin/env node
+/* eslint-disable max-len */
+const assert = require('assert');
+const mysql = require('mysql2/promise');
+const {readFile} = require('fs/promises');
+const {execSync} = require('child_process');
+const {version:desiredVersion} = require('../package.json');
+const logger = require('pino')();
+
+logger.info(`upgrade-jambonz-db: desired version ${desiredVersion}`);
+
+assert.ok(process.env.JAMBONES_MYSQL_HOST, 'missing env JAMBONES_MYSQL_HOST');
+assert.ok(process.env.JAMBONES_MYSQL_DATABASE, 'missing env JAMBONES_MYSQL_DATABASE');
+assert.ok(process.env.JAMBONES_MYSQL_PASSWORD, 'missing env JAMBONES_MYSQL_PASSWORD');
+assert.ok(process.env.JAMBONES_MYSQL_USER, 'missing env JAMBONES_MYSQL_USER');
+
+const opts = {
+  host: process.env.JAMBONES_MYSQL_HOST,
+  user: process.env.JAMBONES_MYSQL_USER,
+  password: process.env.JAMBONES_MYSQL_PASSWORD,
+  database: process.env.JAMBONES_MYSQL_DATABASE,
+  port: process.env.JAMBONES_MYSQL_PORT || 3306,
+  multipleStatements: true
+};
+
+const sql = {
+  '7006': [
+    'ALTER TABLE `accounts` ADD COLUMN `siprec_hook_sid` CHAR(36)',
+    'ALTER TABLE accounts ADD FOREIGN KEY siprec_hook_sid_idxfk (siprec_hook_sid) REFERENCES applications (application_sid)'
+  ],
+  '7007': [
+    `CREATE TABLE service_provider_limits 
+    (service_provider_limits_sid CHAR(36) NOT NULL UNIQUE,
+    service_provider_sid CHAR(36) NOT NULL,
+    category ENUM('api_rate','voice_call_session', 'device') NOT NULL,
+    quantity INTEGER NOT NULL,
+    PRIMARY KEY (service_provider_limits_sid)
+    )`,
+    `CREATE TABLE account_limits
+    (
+    account_limits_sid CHAR(36) NOT NULL UNIQUE ,
+    account_sid CHAR(36) NOT NULL,
+    category ENUM('api_rate','voice_call_session', 'device') NOT NULL,
+    quantity INTEGER NOT NULL,
+    PRIMARY KEY (account_limits_sid)
+    )`,
+    'CREATE INDEX service_provider_sid_idx ON service_provider_limits (service_provider_sid)',
+    `ALTER TABLE service_provider_limits 
+    ADD FOREIGN KEY service_provider_sid_idxfk_3 (service_provider_sid) 
+    REFERENCES service_providers (service_provider_sid) 
+    ON DELETE CASCADE`,
+    'CREATE INDEX account_sid_idx ON account_limits (account_sid)',
+    `ALTER TABLE account_limits 
+    ADD FOREIGN KEY account_sid_idxfk_2 (account_sid) 
+    REFERENCES accounts (account_sid) 
+    ON DELETE CASCADE`,
+    'ALTER TABLE `voip_carriers` ADD COLUMN `register_from_user` VARCHAR(128)',
+    'ALTER TABLE `voip_carriers` ADD COLUMN `register_from_domain` VARCHAR(256)',
+    'ALTER TABLE `voip_carriers` ADD COLUMN `register_public_ip_in_contact` BOOLEAN NOT NULL DEFAULT false'
+  ],
+  '8000': [
+    'ALTER TABLE `applications` ADD COLUMN `app_json` TEXT',
+    'ALTER TABLE voip_carriers CHANGE register_public_domain_in_contact register_public_ip_in_contact BOOLEAN',
+    'alter table phone_numbers modify number varchar(132) NOT NULL UNIQUE',
+    `CREATE TABLE permissions
+    (
+    permission_sid CHAR(36) NOT NULL UNIQUE ,
+    name VARCHAR(32) NOT NULL UNIQUE ,
+    description VARCHAR(255),
+    PRIMARY KEY (permission_sid)
+    )`,
+    `CREATE TABLE user_permissions
+    (
+    user_permissions_sid CHAR(36) NOT NULL UNIQUE ,
+    user_sid CHAR(36) NOT NULL,
+    permission_sid CHAR(36) NOT NULL,
+    PRIMARY KEY (user_permissions_sid)
+    )`,
+    `CREATE TABLE password_settings
+    (
+    min_password_length INTEGER NOT NULL DEFAULT 8,
+    require_digit BOOLEAN NOT NULL DEFAULT false,
+    require_special_character BOOLEAN NOT NULL DEFAULT false
+    )`,
+    'CREATE INDEX user_permissions_sid_idx ON user_permissions (user_permissions_sid)',
+    'CREATE INDEX user_sid_idx ON user_permissions (user_sid)',
+    'ALTER TABLE user_permissions ADD FOREIGN KEY user_sid_idxfk (user_sid) REFERENCES users (user_sid) ON DELETE CASCADE',
+    'ALTER TABLE user_permissions ADD FOREIGN KEY permission_sid_idxfk (permission_sid) REFERENCES permissions (permission_sid)',
+    'ALTER TABLE `users` ADD COLUMN `is_active` BOOLEAN NOT NULL default true',
+  ],
+  '8003': [
+    'SET FOREIGN_KEY_CHECKS=0',
+    'ALTER TABLE `voip_carriers` ADD COLUMN `register_status` VARCHAR(4096)',
+    'ALTER TABLE `sbc_addresses` ADD COLUMN `last_updated` DATETIME',
+    'ALTER TABLE `sbc_addresses` ADD COLUMN `tls_port` INTEGER',
+    'ALTER TABLE `sbc_addresses` ADD COLUMN `wss_port` INTEGER',
+    `CREATE TABLE system_information
+    (
+    domain_name VARCHAR(255),
+    sip_domain_name VARCHAR(255),
+    monitoring_domain_name VARCHAR(255)
+    )`,
+    'DROP TABLE IF EXISTS `lcr_routes`',
+    'DROP TABLE IF EXISTS `lcr_carrier_set_entry`',
+    `CREATE TABLE lcr_routes
+    (
+    lcr_route_sid CHAR(36),
+    lcr_sid CHAR(36) NOT NULL,
+    regex VARCHAR(32) NOT NULL COMMENT 'regex-based pattern match against dialed number, used for LCR routing of PSTN calls',
+    description VARCHAR(1024),
+    priority INTEGER NOT NULL COMMENT 'lower priority routes are attempted first',
+    PRIMARY KEY (lcr_route_sid)
+    )`,
+    `CREATE TABLE lcr
+    (
+    lcr_sid CHAR(36) NOT NULL UNIQUE ,
+    name VARCHAR(64) COMMENT 'User-assigned name for this LCR table',
+    is_active BOOLEAN NOT NULL DEFAULT 1,
+    default_carrier_set_entry_sid CHAR(36) COMMENT 'default carrier/route to use when no digit match based results are found.',
+    service_provider_sid CHAR(36),
+    account_sid CHAR(36),
+    PRIMARY KEY (lcr_sid)
+    )`,
+    `CREATE TABLE lcr_carrier_set_entry
+    (
+    lcr_carrier_set_entry_sid CHAR(36),
+    workload INTEGER NOT NULL DEFAULT 1 COMMENT 'represents a proportion of traffic to send through the associated carrier; can be used for load balancing traffic across carriers with a common priority for a destination',
+    lcr_route_sid CHAR(36) NOT NULL,
+    voip_carrier_sid CHAR(36) NOT NULL,
+    priority INTEGER NOT NULL DEFAULT 0 COMMENT 'lower priority carriers are attempted first',
+    PRIMARY KEY (lcr_carrier_set_entry_sid)
+    )`,
+    'CREATE INDEX lcr_sid_idx ON lcr_routes (lcr_sid)',
+    'ALTER TABLE lcr_routes ADD FOREIGN KEY lcr_sid_idxfk (lcr_sid) REFERENCES lcr (lcr_sid)',
+    'CREATE INDEX lcr_sid_idx ON lcr (lcr_sid)',
+    'ALTER TABLE lcr ADD FOREIGN KEY default_carrier_set_entry_sid_idxfk (default_carrier_set_entry_sid) REFERENCES lcr_carrier_set_entry (lcr_carrier_set_entry_sid)',
+    'CREATE INDEX service_provider_sid_idx ON lcr (service_provider_sid)',
+    'CREATE INDEX account_sid_idx ON lcr (account_sid)',
+    'ALTER TABLE lcr_carrier_set_entry ADD FOREIGN KEY lcr_route_sid_idxfk (lcr_route_sid) REFERENCES lcr_routes (lcr_route_sid)',
+    'ALTER TABLE lcr_carrier_set_entry ADD FOREIGN KEY voip_carrier_sid_idxfk_3 (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid)',
+    'SET FOREIGN_KEY_CHECKS=1',
+  ],
+  '8004': [
+    'alter table accounts add column record_all_calls BOOLEAN NOT NULL DEFAULT false',
+    'alter table accounts add column bucket_credential VARCHAR(8192)',
+    'alter table accounts add column record_format VARCHAR(16) NOT NULL DEFAULT \'mp3\'',
+    'alter table applications add column record_all_calls BOOLEAN NOT NULL DEFAULT false',
+    'alter table phone_numbers DROP INDEX number',
+    'create unique index phone_numbers_unique_idx_voip_carrier_number ON phone_numbers (number,voip_carrier_sid)',
+    `CREATE TABLE clients
+    (
+    client_sid CHAR(36) NOT NULL UNIQUE ,
+    account_sid CHAR(36) NOT NULL,
+    is_active BOOLEAN NOT NULL DEFAULT 1,
+    username VARCHAR(64),
+    password VARCHAR(1024),
+    PRIMARY KEY (client_sid)
+    )`,
+    'CREATE INDEX client_sid_idx ON clients (client_sid)',
+    'ALTER TABLE clients ADD CONSTRAINT account_sid_idxfk_13 FOREIGN KEY account_sid_idxfk_13 (account_sid) REFERENCES accounts (account_sid)',
+    'ALTER TABLE sip_gateways ADD COLUMN protocol ENUM(\'udp\',\'tcp\',\'tls\', \'tls/srtp\') DEFAULT \'udp\''
+  ],
+  '8005': [
+    'DROP INDEX speech_credentials_idx_1 ON speech_credentials',
+    'ALTER TABLE speech_credentials ADD COLUMN label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN speech_synthesis_label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN speech_recognizer_label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN use_for_fallback_speech BOOLEAN DEFAULT false',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_vendor VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_language VARCHAR(12)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_voice VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_recognizer_vendor VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_recognizer_language VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_recognizer_label VARCHAR(64)',
+    'ALTER TABLE sip_gateways ADD COLUMN pad_crypto BOOLEAN NOT NULL DEFAULT 0',
+    'ALTER TABLE sip_gateways MODIFY port INTEGER',
+    `CREATE TABLE google_custom_voices
+    (
+    google_custom_voice_sid CHAR(36) NOT NULL UNIQUE ,
+    speech_credential_sid CHAR(36) NOT NULL,
+    model VARCHAR(512) NOT NULL,
+    reported_usage ENUM('REPORTED_USAGE_UNSPECIFIED','REALTIME','OFFLINE') DEFAULT 'REALTIME',
+    name VARCHAR(64) NOT NULL,
+    PRIMARY KEY (google_custom_voice_sid)
+    )
+    `,
+    'CREATE INDEX google_custom_voice_sid_idx ON google_custom_voices (google_custom_voice_sid)',
+    'CREATE INDEX speech_credential_sid_idx ON google_custom_voices (speech_credential_sid)',
+    'ALTER TABLE google_custom_voices ADD FOREIGN KEY speech_credential_sid_idxfk (speech_credential_sid) REFERENCES speech_credentials (speech_credential_sid) ON DELETE CASCADE',
+    'ALTER TABLE clients ADD COLUMN allow_direct_queue_calling BOOLEAN NOT NULL DEFAULT 1',
+    'ALTER TABLE clients ADD COLUMN allow_direct_user_calling BOOLEAN NOT NULL DEFAULT 1',
+    'ALTER TABLE clients ADD COLUMN allow_direct_app_calling BOOLEAN NOT NULL DEFAULT 1',
+  ],
+  9000: [
+    'ALTER TABLE sip_gateways ADD COLUMN send_options_ping BOOLEAN NOT NULL DEFAULT 0',
+    'ALTER TABLE applications MODIFY COLUMN speech_synthesis_voice VARCHAR(256)',
+    'ALTER TABLE applications MODIFY COLUMN fallback_speech_synthesis_voice VARCHAR(256)',
+    'ALTER TABLE sip_gateways ADD COLUMN use_sips_scheme BOOLEAN NOT NULL DEFAULT 0',
+  ]
+};
+
+const doIt = async() => {
+  let connection;
+  try {
+    logger.info({opts}, 'connecting to mysql database..');
+    connection = await mysql.createConnection(opts);
+  } catch (err) {
+    logger.error({err}, 'Error connecting to database with provided env vars');
+    process.exit(1);
+  }
+
+  try {
+    /* does the schema exist at all ? */
+    const [r] = await connection.execute('SELECT version from schema_version');
+    let errors = 0;
+    if (r.length) {
+      const {version} = r[0];
+      const arr = /v?(\d+)\.(\d+)\.(\d+)/.exec(version);
+      if (arr) {
+        const upgrades = [];
+        logger.info(`performing schema migration: ${version} => ${desiredVersion}`);
+        const val = (1000 * arr[1]) + (100 * arr[2]) + arr[3];
+        logger.info(`current schema value: ${val}`);
+
+        if (val < 7006) upgrades.push(...sql['7006']);
+        if (val < 7007) upgrades.push(...sql['7007']);
+        if (val < 8000) upgrades.push(...sql['8000']);
+        if (val < 8003) upgrades.push(...sql['8003']);
+        if (val < 8004) upgrades.push(...sql['8004']);
+        if (val < 8005) upgrades.push(...sql['8005']);
+        if (val < 9000) upgrades.push(...sql['9000']);
+
+        // perform all upgrades
+        logger.info({upgrades}, 'applying schema upgrades..');
+        for (const upgrade of upgrades) {
+          try {
+            await connection.execute(upgrade);
+          } catch (err) {
+            errors++;
+            logger.info({statement:upgrade, err}, 'Error applying statement');
+          }
+        }
+      }
+      if (errors === 0) await connection.execute(`UPDATE schema_version SET version = '${desiredVersion}'`);
+      await connection.end();
+      logger.info(`schema migration to ${desiredVersion} completed with ${errors} errors`);
+      return;
+    }
+  } catch (err) {
+  }
+  try {
+    await createSchema(connection);
+    await seedDatabase(connection);
+    logger.info('reset admin password..');
+    execSync(`${__dirname}/../db/reset_admin_password.js`);
+    await connection.query(`INSERT into schema_version (version) values('${desiredVersion}')`);
+    logger.info('database install/upgrade complete.');
+    await connection.end();
+  } catch (err) {
+    logger.error({err}, 'Error seeding database');
+    process.exit(1);
+  }
+};
+
+const createSchema = async(connection) => {
+  logger.info('reading schema..');
+  const sql = await readFile(`${__dirname}/../db/jambones-sql.sql`, {encoding: 'utf8'});
+  logger.info('creating schema..');
+  await connection.query(sql);
+};
+
+const seedDatabase = async(connection) => {
+  const sql = await readFile(`${__dirname}/../db/seed-production-database-open-source.sql`, {encoding: 'utf8'});
+  logger.info('seeding data..');
+  await connection.query(sql);
+};
+
+
+doIt();
+

db/webapp-tests.sql

@@ -0,0 +1,102 @@
+SET FOREIGN_KEY_CHECKS=0;
+
+-- create one service provider
+insert into service_providers (service_provider_sid, name, description, root_domain) 
+values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'jambonz.cloud', 'jambonz.cloud service provider', 'sip.yakeeda.com');
+insert into api_keys (api_key_sid, token) 
+values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36f0', '38700987-c7a4-4685-a5bb-af378f9734de');
+
+-- one sbc
+insert into sbc_addresses (sbc_address_sid, service_provider_sid, ipv4, port) values ('8d6d0fda-4550-41ab-8e2f-60761d81fe7d', null, '3.39.45.30', '5060');
+
+-- two smpp server
+insert into smpp_addresses (smpp_address_sid, service_provider_sid, ipv4, port, use_tls, is_primary) values ('e5e8345b-d533-4c29-940b-57aaccc59f8b', null, '3.39.45.30', '2775', false, true);
+insert into smpp_addresses (smpp_address_sid, service_provider_sid, ipv4, port, use_tls, is_primary) values ('ae060ef3-d5a4-4842-b331-426ec9329fbe', null, '3.39.45.30', '3550', true, true);
+
+-- one voip carrier with one gateway
+insert into voip_carriers (voip_carrier_sid, name) values ('5145b436-2f38-4029-8d4c-fd8c67831c7a', 'my test carrier');
+insert into sip_gateways (sip_gateway_sid, voip_carrier_sid, ipv4, port, inbound, outbound, is_active)
+values ('46b727eb-c7dc-44fa-b063-96e48d408e4a', '5145b436-2f38-4029-8d4c-fd8c67831c7a', '3.3.3.3', 5060, 1, 1, 1);
+
+-- create the test application and test phone number
+insert into webhooks (webhook_sid, url, method) values ('d9c205c6-a129-443e-a9c0-d1bb437d4bb7', 'https://flows.jambonz.cloud/testCall', 'POST');
+insert into webhooks (webhook_sid, url, method) values ('6ac36aeb-6bd0-428a-80a1-aed95640a296', 'https://flows.jambonz.cloud/callStatus', 'POST');
+insert into applications (application_sid, name, service_provider_sid, call_hook_sid, call_status_hook_sid, 
+speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice, speech_recognizer_vendor, speech_recognizer_language)
+values ('7a489343-02ed-471e-8df0-fc5e1b98ce8f', 'Test application', '2708b1b3-2736-40ea-b502-c53d8396247f', 
+'d9c205c6-a129-443e-a9c0-d1bb437d4bb7','6ac36aeb-6bd0-428a-80a1-aed95640a296','google', 'en-US', 'en-US-Standard-C', 'google', 'en-US');
+insert into phone_numbers (phone_number_sid, number, voip_carrier_sid, service_provider_sid)
+values ('ec028c46-1363-4b3f-81db-ee33f179d6ba', '18005551212', '5145b436-2f38-4029-8d4c-fd8c67831c7a', '2708b1b3-2736-40ea-b502-c53d8396247f');
+
+-- create our products
+insert into products (product_sid, name, category)
+values
+('c4403cdb-8e75-4b27-9726-7d8315e3216d', 'concurrent call session', 'voice_call_session'),
+('2c815913-5c26-4004-b748-183b459329df', 'registered device', 'device'),
+('35a9fb10-233d-4eb9-aada-78de5814d680', 'api call', 'api_rate');
+
+-- create predefined carriers
+insert into predefined_carriers (predefined_carrier_sid, name, requires_static_ip, e164_leading_plus, 
+requires_register, register_username, register_password, 
+register_sip_realm, tech_prefix, inbound_auth_username, inbound_auth_password, diversion)
+VALUES
+('7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', 'Twilio', 0, 1, 0, '<your-twilio-credential-username>', '<your-twilio-credential-password>', NULL, NULL, NULL, NULL, NULL),
+('032d90d5-39e8-41c0-b807-9c88cffba65c', 'Voxbone', 0, 1, 0, '<your-voxbone-outbound-username>', '<your-voxbone-outbound-password>', NULL, NULL, NULL, NULL, '<your-voxbone-DID>'),
+('17479288-bb9f-421a-89d1-f4ac57af1dca', 'Peerless Network (US)', 1, 0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
+('bdf70650-5328-47aa-b3d0-47cb219d9c6e', '382 Communications (US)', 1, 0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
+('e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'Simwood', 0, 1, 0, '<your-simwood-auth-trunk-username>',  '<your-simwood-auth-trunk-password>', NULL, NULL, NULL, NULL, NULL);
+
+-- twilio gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('d2ccfcb1-9198-4fe9-a0ca-6e49395837c4', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.172.60.0', 30, 5060, 1, 0),
+('6b1d0032-4430-41f1-87c6-f22233d394ef', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.244.51.0', 30, 5060, 1, 0),
+('0de40217-8bd5-4aa8-a9fd-1994282953c6', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.192', 30, 5060, 1, 0),
+('37bc0b20-b53c-4c31-95a6-f82b1c3713e3', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '35.156.191.128', 30, 5060, 1, 0),
+('39791f4e-b612-4882-a37e-e92711a39f3f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.65.63.192', 30, 5060, 1, 0),
+('81a0c8cb-a33e-42da-8f20-99083da6f02f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.252.254.64', 30, 5060, 1, 0),
+('eeeef07a-46b8-4ffe-a4f2-04eb32ca889e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.169.127.128', 30, 5060, 1, 0),
+('fbb6c194-4b68-4dff-9b42-52412be1c39e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '177.71.206.192', 30, 5060, 1, 0),
+('3ed1dd12-e1a7-44ff-811a-3cc5dc13dc72', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '<your-domain>.pstn.twilio.com', 32, 5060, 0, 1);
+
+-- voxbone gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('d531c582-2103-42a0-b9f0-f80c215b3ec5', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.83.45', 32, 5060, 1, 0),
+('95c888e5-c959-4d92-82c4-8597dddff75e', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.86.45', 32, 5060, 1, 0),
+('1de3b2a1-96f0-407a-bcc4-ce371d823a8d', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.82.45', 32, 5060, 1, 0),
+('50c1f91a-6080-4495-a241-6bba6e9d9688', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.85.45', 32, 5060, 1, 0),
+('e6ebad33-80d5-4dbb-bc6f-a7ae08160cc6', '032d90d5-39e8-41c0-b807-9c88cffba65c', '81.201.84.45', 32, 5060, 1, 0),
+('bc933522-18a2-47d8-9ae4-9faa8de4e927', '032d90d5-39e8-41c0-b807-9c88cffba65c', 'outbound.voxbone.com', 32, 5060, 0, 1);
+
+-- Peerless gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('4e23f698-a70a-4616-9bf0-c9dd5ab123af', '17479288-bb9f-421a-89d1-f4ac57af1dca', '208.79.54.182', 32, 5060, 1, 0),
+('e5c71c18-0511-41b8-bed9-1ba061bbcf10', '17479288-bb9f-421a-89d1-f4ac57af1dca', '208.79.52.192', 32, 5060, 0, 1),
+('226c7471-2f4f-440f-8525-37fd0512bd8b', '17479288-bb9f-421a-89d1-f4ac57af1dca', '208.79.54.185', 32, 5060, 0, 1);
+
+-- 382com gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('23e4c250-8578-4d88-99b5-a7941a58e26f', 'bdf70650-5328-47aa-b3d0-47cb219d9c6e', '64.125.111.10', 32, 5060, 1, 0),
+('c726d435-c9a7-4c37-b891-775990a54638', 'bdf70650-5328-47aa-b3d0-47cb219d9c6e', '64.124.67.11', 32, 5060, 0, 1);
+
+-- simwood gateways
+insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
+VALUES
+('441fd2e7-c845-459c-963d-6e917063ed9a', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.136.24', 29, 5060, 1, 0),
+('1e0d3e80-9973-4184-9bec-07ae564f983f', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.136.28', 28, 5060, 1, 0),
+('e56ec745-5f37-443f-afb4-7bbda31ae7ac', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.140.48', 28, 5060, 1, 0),
+('e7447e7e-2c7d-4738-ab53-097c187236ff', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.140.242', 32, 5060, 1, 0),
+('279807e7-649e-41dd-931a-00c460bcc0a2', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.143.80', 28, 5060, 1, 0),
+('f5fd66f7-97f6-4979-ab19-eea1557bc872', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.140.0', 26, 5060, 1, 0),
+('efcfefdc-451c-4e59-960a-f9e4952d964f', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.141.0', 26, 5060, 1, 0),
+('b6ae6240-55ac-4c11-892f-a71b2155ea60', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.142.0', 26, 5060, 1, 0),
+('5a976337-164b-408e-8748-d8bfb4bd5d76', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.143.0', 26, 5060, 1, 0),
+('ed0434ca-7f26-4624-9523-0419d0d2924d', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.139.0', 26, 5060, 1, 0),
+('6bfb55e5-e248-48dc-a104-4f3eedd7d7de', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.0', 24, 5060, 1, 0),
+('5f431d42-48e4-44ce-a311-d946f0b475b6', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'out.simwood.com', 32, 5060, 0, 1);
+
+
+SET FOREIGN_KEY_CHECKS=1;

lib/auth/index.js

@@ -1,5 +1,8 @@
 const Strategy = require('passport-http-bearer').Strategy;
 const {getMysqlConnection} = require('../db');
+const debug = require('debug')('jambonz:api-server');
+const {cacheClient} = require('../helpers');
+const jwt = require('jsonwebtoken');
 const sql = `
   SELECT *
   FROM api_keys
@@ -7,50 +10,100 @@ const sql = `
 
 function makeStrategy(logger) {
   return new Strategy(
-    function(token, done) {
-      logger.info(`validating with token ${token}`);
-      getMysqlConnection((err, conn) => {
+    async function(token, done) {
+      jwt.verify(token, process.env.JWT_SECRET, async(err, decoded) => {
         if (err) {
-          logger.error(err, 'Error retrieving mysql connection');
-          return done(err);
-        }
-        conn.query(sql, [token], (err, results, fields) => {
-          conn.release();
-          if (err) {
-            logger.error(err, 'Error querying for api key');
-            return done(err);
-          }
-          if (0 == results.length) return done(null, false);
-          if (results.length > 1) {
-            logger.info(`api key ${token} exists in multiple rows of api_keys table!!`);
+          if (err.name === 'TokenExpiredError') {
+            logger.debug('jwt expired');
             return done(null, false);
           }
-
-          // found api key
-          const scope = [];
-          if (results[0].account_sid === null && results[0].service_provider_sid === null) {
-            scope.push.apply(scope, ['admin', 'service_provider', 'account']);
-          }
-          else if (results[0].service_provider_sid) {
-            scope.push.apply(scope, ['service_provider', 'account']);
-          }
-          else {
-            scope.push('account');
+          /* its not a jwt obtained through login, check api keys */
+          checkApiTokens(logger, token, done);
+        }
+        else {
+          try {
+            const {user_sid} = decoded;
+            /* Valid jwt tokens are stored in redis by hashed user_id */
+            const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+            const result = await cacheClient.get(redisKey);
+            if (result === null) {
+              debug(`result from searching for ${redisKey}: ${result}`);
+              logger.info('jwt invalidated after logout');
+              return done(null, false);
+            }
+          } catch (error) {
+            debug(err);
+            logger.info({err}, 'Error checking redis for jwt');
           }
+          const { user_sid, service_provider_sid, account_sid, email, name, scope, permissions } = decoded;
 
           const user = {
-            account_sid: results[0].account_sid,
-            service_provider_sid: results[0].service_provider_sid,
-            hasScope: (s) => scope.includes(s),
-            hasAdminAuth: scope.length === 3,
-            hasServiceProviderAuth: scope.includes('service_provider') && !scope.includes('admin'),
-            hasAccountAuth: scope.includes('account') && !scope.includes('service_provider')
+            service_provider_sid,
+            account_sid,
+            user_sid,
+            jwt: token,
+            email,
+            name,
+            permissions,
+            hasScope: (s) => s === scope,
+            hasAdminAuth: scope === 'admin',
+            hasServiceProviderAuth: scope === 'service_provider',
+            hasAccountAuth: scope === 'account'
           };
-          logger.info(user, `successfully validated with scope ${scope}`);
+          logger.debug({user}, 'successfully validated jwt');
           return done(null, user, {scope});
-        });
+        }
       });
-    });
+    }
+  );
 }
 
+const checkApiTokens = (logger, token, done) => {
+  getMysqlConnection((err, conn) => {
+    if (err) {
+      logger.error(err, 'Error retrieving mysql connection');
+      return done(err);
+    }
+    conn.query(sql, [token], (err, results, fields) => {
+      conn.release();
+      if (err) {
+        logger.error(err, 'Error querying for api key');
+        return done(err);
+      }
+      if (0 == results.length) return done(null, false);
+      if (results.length > 1) {
+        logger.info(`api key ${token} exists in multiple rows of api_keys table!!`);
+        return done(null, false);
+      }
+
+      // found api key
+      let scope;
+      //const scope = [];
+      if (results[0].account_sid === null && results[0].service_provider_sid === null) {
+        //scope.push.apply(scope, ['admin', 'service_provider', 'account']);
+        scope = 'admin';
+      }
+      else if (results[0].service_provider_sid) {
+        //scope.push.apply(scope, ['service_provider', 'account']);
+        scope = 'service_provider';
+      }
+      else {
+        //scope.push('account');
+        scope = 'account';
+      }
+
+      const user = {
+        account_sid: results[0].account_sid,
+        service_provider_sid: results[0].service_provider_sid,
+        hasScope: (s) => s === scope,
+        hasAdminAuth: scope === 'admin',
+        hasServiceProviderAuth: scope === 'service_provider',
+        hasAccountAuth: scope === 'account'
+      };
+      logger.debug({user}, `successfully validated with scope ${scope}`);
+      return done(null, user, {scope});
+    });
+  });
+};
+
 module.exports = makeStrategy;

lib/db/index.js

@@ -1,5 +1,7 @@
 const getMysqlConnection = require('./mysql');
+const promisePool = require('./pool');
 
 module.exports = {
-  getMysqlConnection
+  getMysqlConnection,
+  promisePool
 };

lib/db/mysql.js

@@ -1,6 +1,7 @@
 const mysql = require('mysql2');
 const pool = mysql.createPool({
   host: process.env.JAMBONES_MYSQL_HOST,
+  port: process.env.JAMBONES_MYSQL_PORT || 3306,
   user: process.env.JAMBONES_MYSQL_USER,
   password: process.env.JAMBONES_MYSQL_PASSWORD,
   database: process.env.JAMBONES_MYSQL_DATABASE,
@@ -12,6 +13,7 @@ pool.getConnection((err, conn) => {
   conn.ping((err) => {
     if (err) return console.error(err, `Error pinging mysql at ${JSON.stringify({
       host: process.env.JAMBONES_MYSQL_HOST,
+      port: process.env.JAMBONES_MYSQL_PORT || 3306,
       user: process.env.JAMBONES_MYSQL_USER,
       password: process.env.JAMBONES_MYSQL_PASSWORD,
       database: process.env.JAMBONES_MYSQL_DATABASE,

lib/db/pool.js

@@ -0,0 +1,10 @@
+const mysql = require('mysql2');
+const pool = mysql.createPool({
+  host: process.env.JAMBONES_MYSQL_HOST,
+  port: process.env.JAMBONES_MYSQL_PORT || 3306,
+  user: process.env.JAMBONES_MYSQL_USER,
+  password: process.env.JAMBONES_MYSQL_PASSWORD,
+  database: process.env.JAMBONES_MYSQL_DATABASE,
+  connectionLimit: process.env.JAMBONES_MYSQL_CONNECTION_LIMIT || 10
+});
+module.exports = pool.promise();

lib/helpers/cache-client.js

@@ -0,0 +1,82 @@
+const {
+  addKey: addKeyRedis,
+  deleteKey: deleteKeyRedis,
+  retrieveKey: retrieveKeyRedis,
+} = require('./realtimedb-helpers');
+const { hashString } = require('../utils/password-utils');
+const logger = require('../logger');
+
+class CacheClient {
+  constructor() { }
+
+  async set(params) {
+    const {
+      redisKey,
+      value = '1',
+      time = 3600,
+    } = params || {};
+
+    try {
+      await addKeyRedis(redisKey, value, time);
+
+    } catch (err) {
+      logger.error('CacheClient.get set', {
+        error: {
+          message: err.message,
+          name: err.name
+        },
+        ...params
+      });
+    }
+  }
+
+  async get(redisKey) {
+    try {
+      const result = await retrieveKeyRedis(redisKey);
+      return result;
+    } catch (err) {
+      logger.error('CacheClient.get error', {
+        error: {
+          message: err.message,
+          name: err.name
+        },
+        redisKey
+      });
+    }
+  }
+
+
+  async delete(key) {
+    try {
+      await deleteKeyRedis(key);
+      logger.debug('CacheClient.delete key from redis', { key });
+    } catch (err) {
+      logger.error('CacheClient.delete error', {
+        error: {
+          message: err.message,
+          name: err.name
+        },
+        key
+      });
+    }
+  }
+
+  generateRedisKey(type, key, version) {
+    let suffix = '';
+    if (version) {
+      suffix = `:version:${version}`;
+    }
+
+    switch (type) {
+      case 'reset-link':
+        return `reset-link:${key}`;
+      case 'jwt':
+      default:
+        return `jwt:${hashString(key)}${suffix}`;
+    }
+  }
+}
+
+const cacheClient = new CacheClient();
+
+module.exports = { cacheClient };

lib/helpers/index.js

@@ -0,0 +1,4 @@
+module.exports = {
+  ...require('./cache-client'),
+  ...require('./realtimedb-helpers'),
+};

lib/helpers/realtimedb-helpers.js

@@ -0,0 +1,33 @@
+const logger = require('../logger');
+
+const {
+  client,
+  retrieveCall,
+  deleteCall,
+  listCalls,
+  listSortedSets,
+  purgeCalls,
+  retrieveSet,
+  addKey,
+  retrieveKey,
+  deleteKey,
+  incrKey,
+  client: redisClient,
+  listConferences
+} = require('@jambonz/realtimedb-helpers')({}, logger);
+
+module.exports = {
+  client,
+  retrieveCall,
+  deleteCall,
+  listCalls,
+  listSortedSets,
+  purgeCalls,
+  retrieveSet,
+  addKey,
+  retrieveKey,
+  deleteKey,
+  redisClient,
+  incrKey,
+  listConferences
+};

lib/logger.js

@@ -0,0 +1,7 @@
+const opts = {
+  level: process.env.JAMBONES_LOGLEVEL || 'info'
+};
+const pino = require('pino');
+const logger = pino(opts, pino.destination(1, {sync: false}));
+
+module.exports = logger;

lib/middleware.js

@@ -0,0 +1,32 @@
+const logger = require('./logger');
+
+function delayLoginMiddleware(req, res, next) {
+  if (req.path.includes('/login') || req.path.includes('/signin')) {
+    const min = 200;
+    const max = 1000;
+    /* Random delay between 200 - 1000ms */
+    const sendStatusDelay = Math.floor(Math.random() * (max - min + 1)) + min;
+
+    /* the res.json take longer, we decrease the max delay slightly to 0-800ms */
+    const jsonDelay = Math.floor(Math.random() * 800);
+    logger.debug(`delayLoginMiddleware: sendStatus ${sendStatusDelay} - json ${jsonDelay}`);
+    const sendStatus = res.sendStatus;
+    const json = res.json;
+
+    res.sendStatus = function(status) {
+      setTimeout(() => {
+        sendStatus.call(res, status);
+      }, sendStatusDelay);
+    };
+    res.json = function(body) {
+      setTimeout(() => {
+        json.call(res, body);
+      }, jsonDelay);
+    };
+  }
+  next();
+}
+
+module.exports = {
+  delayLoginMiddleware
+};

lib/models/account-limits.js

@@ -0,0 +1,41 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+const sql = 'SELECT * FROM account_limits WHERE account_sid = ?';
+
+class AccountLimits extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieve(account_sid) {
+    const [rows] = await promisePool.query(sql, [account_sid]);
+    return rows;
+  }
+
+}
+
+AccountLimits.table = 'account_limits';
+AccountLimits.fields = [
+  {
+    name: 'account_limits_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'account_sid',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'category',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'quantity',
+    type: 'number',
+    required: true
+  }
+];
+
+module.exports = AccountLimits;

lib/models/account.js

@@ -1,19 +1,89 @@
+const debug = require('debug')('jambonz:api-server');
 const Model = require('./model');
 const {getMysqlConnection} = require('../db');
+const {promisePool} = require('../db');
+const { v4: uuid } = require('uuid');
+
+const {encrypt, decrypt} = require('../utils/encrypt-decrypt');
 
 const retrieveSql = `SELECT * from accounts acc
 LEFT JOIN webhooks AS rh
-ON acc.registration_hook_sid = rh.webhook_sid`;
+ON acc.registration_hook_sid = rh.webhook_sid 
+LEFT JOIN webhooks AS qh
+ON acc.queue_event_hook_sid = qh.webhook_sid`;
+
+const insertPendingAccountSubscriptionSql = `INSERT account_subscriptions 
+(account_subscription_sid, account_sid, pending, stripe_subscription_id, 
+stripe_payment_method_id, last4, exp_month, exp_year, card_type) 
+VALUES (?,?,1,?,?,?,?,?,?)`;
+
+const activateSubscriptionSql = `UPDATE account_subscriptions 
+SET pending=0, effective_start_date = CURRENT_TIMESTAMP, stripe_subscription_id = ? 
+WHERE account_subscription_sid = ? 
+AND pending=1`;
+
+const queryPendingSubscriptionSql = `SELECT * FROM account_subscriptions 
+WHERE account_sid = ? 
+AND effective_end_date IS NULL 
+AND pending=1`;
+
+const deactivateSubscriptionSql = `UPDATE account_subscriptions 
+SET pending=1, pending_reason = ? 
+WHERE account_sid = ? 
+AND effective_end_date IS NULL 
+AND pending=0`;
+
+const updatePaymentInfoSql = `UPDATE account_subscriptions 
+SET last4 = ?, stripe_payment_method_id=?, exp_month = ?, exp_year = ?, card_type = ? 
+WHERE account_sid = ? 
+AND effective_end_date IS NULL`;
+
+const insertAccountProductsSql = `INSERT account_products 
+(account_product_sid, account_subscription_sid, product_sid, quantity) 
+VALUES (?,?,?,?);
+`;
+
+const replaceOldSubscriptionSql = `UPDATE account_subscriptions 
+SET effective_end_date = CURRENT_TIMESTAMP, change_reason = ?  
+WHERE account_sid = ? 
+AND effective_end_date IS NULL 
+AND account_subscription_sid <> ?`;
+
+const retrieveActiveSubscriptionSql = `SELECT * 
+FROM account_subscriptions 
+WHERE account_sid = ? 
+AND effective_end_date IS NULL 
+AND pending = 0`;
+
+const extractBucketCredential = (obj) => {
+  const {bucket_credential} = obj;
+  if (bucket_credential) {
+    obj.bucket_credential = JSON.parse(decrypt(bucket_credential));
+  }
+};
 
 function transmogrifyResults(results) {
   return results.map((row) => {
     const obj = row.acc;
+
+    /* registration hook */
     if (row.rh && Object.keys(row.rh).length && row.rh.url !== null) {
       Object.assign(obj, {registration_hook: row.rh});
       delete obj.registration_hook.webhook_sid;
     }
     else obj.registration_hook = null;
     delete obj.registration_hook_sid;
+
+    /* queue event hook */
+    if (row.qh && Object.keys(row.qh).length && row.qh.url !== null) {
+      Object.assign(obj, {queue_event_hook: row.qh});
+      delete obj.queue_event_hook.webhook_sid;
+    }
+    else obj.queue_event_hook = null;
+    delete obj.queue_event_hook_sid;
+
+    extractBucketCredential(obj);
+
     return obj;
   });
 }
@@ -73,6 +143,111 @@ class Account extends Model {
     });
   }
 
+  static async updateStripeCustomerId(sid, customerId) {
+    await promisePool.execute(
+      'UPDATE accounts SET stripe_customer_id = ? WHERE account_sid = ?',
+      [customerId, sid]);
+  }
+
+  static async getSubscription(sid) {
+    const [r] = await promisePool.execute(retrieveActiveSubscriptionSql, [sid]);
+    debug(r, `Account.getSubscription ${sid}`);
+    return r.length > 0 ? r[0] : null;
+  }
+
+  static async deactivateSubscription(logger, account_sid, reason) {
+    logger.debug('deactivateSubscription');
+
+    /**
+     * Two cases:
+     * (1) A subscription renewal fails.  In this case we deactivate subscription
+     * and the customer is down until they provide payment.
+     * (2) A customer adds capacity during the month, and the pro-rated amount fails.
+     * In this case, we leave the new subscription in a pending state
+     * The customer continues (for the rest of the month at least) at
+     * previous capacity levels.
+     */
+    const [r] = await promisePool.query(queryPendingSubscriptionSql, account_sid);
+    if (r.length > 0) {
+      /* leave new subscription pending */
+      await promisePool.execute(
+        'UPDATE account_subscriptions set pending_reason = ? WHERE account_subscription_sid = ?',
+        [reason, r[0].account_subscription_sid]);
+      logger.debug('deactivateSubscription - leave pending subscription in pending state');
+    }
+    else {
+      /* deactivate their current active subscription */
+      const [r] = await promisePool.execute(deactivateSubscriptionSql, [reason, account_sid]);
+      logger.debug('deactivateSubscription - deactivated subscription; customer will not have service');
+      return 1 == r.affectedRows;
+    }
+  }
+
+  static async activateSubscription(logger, account_sid, subscription_id, reason) {
+    logger.debug('activateSubscription');
+
+    const [r] = await promisePool.query(queryPendingSubscriptionSql, account_sid);
+    if (0 === r.length) return false;
+
+    const [r2] = await promisePool.execute(activateSubscriptionSql,
+      [subscription_id, r[0].account_subscription_sid]);
+    if (0 === r2.affectedRows) return false;
+
+    /* disable the old subscription, if any */
+    const [r3] = await promisePool.execute(replaceOldSubscriptionSql, [
+      reason, account_sid, r[0].account_subscription_sid]);
+    debug(r3, 'Account.activateSubscription - replaced old subscription');
+
+    /* update account.plan to paid, if it isnt already */
+    /* update account.is_active to 1, if account is deactivated */
+    await promisePool.execute(
+      'UPDATE accounts SET plan_type = \'paid\', is_active = 1 WHERE account_sid = ?',
+      [account_sid]);
+    return true;
+  }
+
+  static async updatePaymentInfo(logger, account_sid, pm) {
+    const {id, card} = pm;
+    const last4_encrypted = encrypt(card.last4);
+    await promisePool.execute(updatePaymentInfoSql,
+      [last4_encrypted, id, card.exp_month, card.exp_year, card.brand, account_sid]);
+  }
+
+  static async provisionPendingSubscription(logger, account_sid, products, payment_method, subscription_id) {
+    logger.debug('provisionPendingSubscription');
+    const account_subscription_sid = uuid();
+    const {id, card} = payment_method;
+
+    /* add a row to account_subscription */
+    let last4_encrypted = null;
+    if (card) {
+      last4_encrypted = encrypt(card.last4);
+    }
+    const [r] = await promisePool.execute(insertPendingAccountSubscriptionSql, [
+      account_subscription_sid,
+      account_sid,
+      subscription_id || null,
+      id,
+      last4_encrypted,
+      card ? card.exp_month : null,
+      card ? card.exp_year : null,
+      card ? card.brand : null
+    ]);
+    debug(r, 'Account.activateSubscription - insert account_subscriptions');
+    if (r.affectedRows !== 1) {
+      throw new Error(`failed inserting account_subscriptions for accunt_sid ${account_sid}`);
+    }
+
+    /* add a row for each product to account_products */
+    await Promise.all(products.map((product) => {
+      const {product_sid, quantity} = product;
+      const account_products_sid = uuid();
+      return promisePool.execute(insertAccountProductsSql, [
+        account_products_sid, account_subscription_sid, product_sid, quantity
+      ]);
+    }));
+    return account_subscription_sid;
+  }
 }
 
 Account.table = 'accounts';
@@ -96,6 +271,10 @@ Account.fields = [
     name: 'sip_realm',
     type: 'string',
   },
+  {
+    name: 'queue_event_hook_sid',
+    type: 'string',
+  },
   {
     name: 'registration_hook_sid',
     type: 'string',
@@ -103,6 +282,62 @@ Account.fields = [
   {
     name: 'device_calling_application_sid',
     type: 'string',
+  },
+  {
+    name: 'is_active',
+    type: 'number',
+  },
+  {
+    name: 'created_at',
+    type: 'date',
+  },
+  {
+    name: 'plan_type',
+    type: 'string',
+  },
+  {
+    name: 'stripe_customer_id',
+    type: 'string',
+  },
+  {
+    name: 'webhook_secret',
+    type: 'string',
+  },
+  {
+    name: 'disable_cdrs',
+    type: 'number',
+  },
+  {
+    name: 'subspace_client_id',
+    type: 'string',
+  },
+  {
+    name: 'subspace_client_secret',
+    type: 'string',
+  },
+  {
+    name: 'subspace_sip_teleport_id',
+    type: 'string',
+  },
+  {
+    name: 'subspace_sip_teleport_destinations',
+    type: 'string',
+  },
+  {
+    name: 'siprec_hook_sid',
+    type: 'string',
+  },
+  {
+    name: 'record_all_calls',
+    type: 'number'
+  },
+  {
+    name: 'record_format',
+    type: 'string'
+  },
+  {
+    name: 'bucket_credential',
+    type: 'string'
   }
 ];
 

lib/models/api-key.js

@@ -27,6 +27,25 @@ class ApiKey extends Model {
     });
   }
 
+  /**
+  * list all api keys for a service provider
+  */
+  static retrieveAllForSP(service_provider_sid) {
+    const sql = 'SELECT * from api_keys WHERE service_provider_sid = ?';
+    const args = [service_provider_sid];
+
+    return new Promise((resolve, reject) => {
+      getMysqlConnection((err, conn) => {
+        if (err) return reject(err);
+        conn.query(sql, args, (err, results) => {
+          conn.release();
+          if (err) return reject(err);
+          resolve(results);
+        });
+      });
+    });
+  }
+
   /**
   * update last_used api key for an account
   */

lib/models/application.js

@@ -120,6 +120,10 @@ Application.fields = [
   {
     name: 'messaging_hook_sid',
     type: 'string',
+  },
+  {
+    name: 'record_all_calls',
+    type: 'number',
   }
 ];
 

lib/models/client.js

@@ -0,0 +1,70 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class Client extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByAccountSid(account_sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE account_sid = ?`;
+    const [rows] = await promisePool.query(sql, account_sid);
+    return rows;
+  }
+
+  static async retrieveAllByServiceProviderSid(service_provider_sid) {
+    const sql = `SELECT c.client_sid, c.account_sid, c.is_active, c.username, c.hashed_password
+      FROM ${this.table} AS c LEFT JOIN accounts AS acc ON c.account_sid = acc.account_sid
+      LEFT JOIN service_providers AS sp ON sp.service_provider_sid = accs.service_provider_sid
+      WHERE sp.service_provider_sid = ?`;
+    const [rows] = await promisePool.query(sql, service_provider_sid);
+    return rows;
+  }
+
+  static async retrieveByAccountSidAndUserName(account_sid, username) {
+    const sql = `SELECT * FROM ${this.table} WHERE account_sid = ? AND username = ?`;
+    const [rows] = await promisePool.query(sql, [account_sid, username]);
+    return rows;
+  }
+}
+
+Client.table = 'clients';
+Client.fields = [
+  {
+    name: 'client_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'account_sid',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'is_active',
+    type: 'number'
+  },
+  {
+    name: 'username',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'password',
+    type: 'string'
+  },
+  {
+    name: 'allow_direct_app_calling',
+    type: 'number'
+  },
+  {
+    name: 'allow_direct_queue_calling',
+    type: 'number'
+  },
+  {
+    name: 'allow_direct_user_calling',
+    type: 'number'
+  }
+];
+
+module.exports = Client;

lib/models/google-custom-voice.js

@@ -0,0 +1,61 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class GoogleCustomVoice extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllBySpeechCredentialSid(speech_credential_sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE speech_credential_sid = ?`;
+    const [rows] = await promisePool.query(sql, speech_credential_sid);
+    return rows;
+  }
+
+  static async deleteAllBySpeechCredentialSid(speech_credential_sid) {
+    const sql = `DELETE FROM ${this.table} WHERE speech_credential_sid = ?`;
+    const [rows] = await promisePool.query(sql, speech_credential_sid);
+    return rows;
+  }
+
+  static async retrieveAllByLabel(service_provider_sid, account_sid, label) {
+    let sql;
+    if (account_sid) {
+      sql = `SELECT gcv.* FROM ${this.table} gcv 
+LEFT JOIN speech_credentials sc ON gcv.speech_credential_sid = sc.speech_credential_sid 
+WHERE sc.account_sid = ? OR (sc.account_sid is NULL && sc.service_provider_sid = ?) 
+${label ? 'AND label = ?' : 'AND label is NULL'}`;
+    } else {
+      sql = `SELECT gcv.* FROM ${this.table} gcv 
+LEFT JOIN speech_credentials sc ON gcv.speech_credential_sid = sc.speech_credential_sid 
+WHERE sc.service_provider_sid = ? ${label ? 'AND label = ?' : 'AND label is NULL'}`;
+    }
+    const [rows] = await promisePool.query(sql, [...(account_sid ?
+      [account_sid, service_provider_sid] : [service_provider_sid]), label]);
+    return rows;
+  }
+}
+GoogleCustomVoice.table = 'google_custom_voices';
+GoogleCustomVoice.fields = [
+  {
+    name: 'google_custom_voice_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'model',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'reported_usage',
+    type: 'number'
+  },
+  {
+    name: 'name',
+    type: 'string',
+    required: true
+  }
+];
+
+module.exports = GoogleCustomVoice;

lib/models/lcr-carrier-set-entry.js

@@ -0,0 +1,47 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class LcrCarrierSetEntry extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByLcrRouteSid(sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE lcr_route_sid = ? ORDER BY priority`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows;
+  }
+
+  static async deleteByLcrRouteSid(sid) {
+    const sql = `DELETE FROM ${this.table} WHERE lcr_route_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows.affectedRows;
+  }
+}
+
+LcrCarrierSetEntry.table = 'lcr_carrier_set_entry';
+LcrCarrierSetEntry.fields = [
+  {
+    name: 'lcr_carrier_set_entry_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'workload',
+    type: 'number'
+  },
+  {
+    name: 'lcr_route_sid',
+    type: 'string'
+  },
+  {
+    name: 'voip_carrier_sid',
+    type: 'string'
+  },
+  {
+    name: 'priority',
+    type: 'number'
+  }
+];
+
+module.exports = LcrCarrierSetEntry;

lib/models/lcr-route.js

@@ -0,0 +1,54 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+
+class LcrRoutes extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByLcrSid(sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE lcr_sid = ? ORDER BY priority`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows;
+  }
+
+  static async deleteByLcrSid(sid) {
+    const sql = `DELETE FROM ${this.table} WHERE lcr_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows.affectedRows;
+  }
+
+  static async countAllByLcrSid(sid) {
+    const sql = `SELECT COUNT(*) AS count FROM ${this.table} WHERE lcr_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows.length ? rows[0].count : 0;
+  }
+}
+
+LcrRoutes.table = 'lcr_routes';
+LcrRoutes.fields = [
+  {
+    name: 'lcr_route_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'lcr_sid',
+    type: 'string'
+  },
+  {
+    name: 'regex',
+    type: 'string'
+  },
+  {
+    name: 'description',
+    type: 'string'
+  },
+  {
+    name: 'priority',
+    type: 'number'
+  }
+];
+
+module.exports = LcrRoutes;

lib/models/lcr.js

@@ -0,0 +1,54 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class Lcr extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByAccountSid(account_sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE account_sid = ?`;
+    const [rows] = await promisePool.query(sql, account_sid);
+    return rows;
+  }
+
+  static async retrieveAllByServiceProviderSid(sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE service_provider_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows;
+  }
+
+  static async releaseDefaultEntry(sid) {
+    const sql = `UPDATE ${this.table} SET default_carrier_set_entry_sid = null WHERE lcr_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows;
+  }
+}
+
+Lcr.table = 'lcr';
+Lcr.fields = [
+  {
+    name: 'lcr_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'name',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'account_sid',
+    type: 'string'
+  },
+  {
+    name: 'service_provider_sid',
+    type: 'string'
+  },
+  {
+    name: 'default_carrier_set_entry_sid',
+    type: 'string'
+  }
+];
+
+module.exports = Lcr;

lib/models/model.js

@@ -1,5 +1,5 @@
 const Emitter = require('events');
-const uuidv4 = require('uuid/v4');
+const { v4: uuidv4 } = require('uuid');
 const assert = require('assert');
 const {getMysqlConnection} = require('../db');
 const {DbErrorBadRequest} = require('../utils/errors');
@@ -107,7 +107,7 @@ class Model extends Emitter {
       if (pk.name in obj) throw new DbErrorBadRequest(`primary key ${pk.name} is immutable`);
       getMysqlConnection((err, conn) => {
         if (err) return reject(err);
-        conn.query(`UPDATE ${this.table} SET ? WHERE ${pk.name} = '${sid}'`, obj, (err, results, fields) => {
+        conn.query(`UPDATE ${this.table} SET ? WHERE ${pk.name} = ?`, [obj, sid], (err, results, fields) => {
           conn.release();
           if (err) return reject(err);
           resolve(results.affectedRows);

lib/models/password-settings.js

@@ -0,0 +1,70 @@
+const {promisePool} = require('../db');
+
+class PasswordSettings {
+
+  /**
+   * Retrieve object from database
+   */
+  static async retrieve() {
+    const [r] = await promisePool.execute(`SELECT * FROM ${this.table}`);
+    return r;
+  }
+
+  /**
+  * Update object into the database
+  */
+  static async update(obj) {
+    let sql = `UPDATE ${this.table} SET `;
+    const values = [];
+    const keys = Object.keys(obj);
+    this.fields.forEach(({name}) => {
+      if (keys.includes(name)) {
+        sql = sql + `${name} = ?,`;
+        values.push(obj[name]);
+      }
+    });
+    if (values.length) {
+      sql = sql.slice(0, -1);
+      await promisePool.execute(sql, values);
+    }
+  }
+
+  /**
+     * insert object into the database
+     */
+  static async make(obj) {
+    let params = '', marks = '';
+    const values = [];
+    const keys = Object.keys(obj);
+    this.fields.forEach(({name}) => {
+      if (keys.includes(name)) {
+        params = params + `${name},`;
+        marks = marks + '?,';
+        values.push(obj[name]);
+      }
+    });
+    if (values.length) {
+      params = `(${params.slice(0, -1)})`;
+      marks = `values(${marks.slice(0, -1)})`;
+      return await promisePool.execute(`INSERT into ${this.table} ${params} ${marks}`, values);
+    }
+  }
+}
+
+
+PasswordSettings.table = 'password_settings';
+PasswordSettings.fields = [
+  {
+    name: 'min_password_length',
+    type: 'number'
+  },
+  {
+    name: 'require_digit',
+    type: 'number'
+  },
+  {
+    name: 'require_special_character',
+    type: 'number'
+  }
+];
+module.exports = PasswordSettings;

lib/models/phone-number.js

@@ -1,9 +1,39 @@
 const Model = require('./model');
+const {promisePool} = require('../db');
+const sqlRetrieveAll = 'SELECT * from phone_numbers WHERE account_sid = ? ORDER BY number';
+const sqlRetrieveOne = 'SELECT * from phone_numbers WHERE phone_number_sid = ? AND account_sid = ? ORDER BY number';
+const sqlSP = `SELECT * 
+FROM phone_numbers 
+WHERE account_sid IN 
+(
+  SELECT account_sid 
+  FROM accounts 
+  WHERE service_provider_sid = ?
+) ORDER BY number`;
 
 class PhoneNumber extends Model {
   constructor() {
     super();
   }
+
+  static async retrieveAll(account_sid) {
+    if (!account_sid) return await super.retrieveAll();
+    const [rows] = await promisePool.query(sqlRetrieveAll, account_sid);
+    return rows;
+  }
+  static async retrieveAllForSP(service_provider_sid) {
+    const [rows] = await promisePool.query(sqlSP, service_provider_sid);
+    return rows;
+  }
+
+  /**
+   * retrieve a phone number
+   */
+  static async retrieve(sid, account_sid) {
+    if (!account_sid) return super.retrieve(sid);
+    const [rows] = await promisePool.query(sqlRetrieveOne, [sid, account_sid]);
+    return rows;
+  }
 }
 
 PhoneNumber.table = 'phone_numbers';
@@ -20,8 +50,7 @@ PhoneNumber.fields = [
   },
   {
     name: 'voip_carrier_sid',
-    type: 'string',
-    required: true
+    type: 'string'
   },
   {
     name: 'account_sid',

lib/models/predefined-carrier.js

@@ -0,0 +1,63 @@
+const Model = require('./model');
+
+class PredefinedCarrier extends Model {
+  constructor() {
+    super();
+  }
+}
+
+PredefinedCarrier.table = 'predefined_carriers';
+PredefinedCarrier.fields = [
+  {
+    name: 'predefined_carrier_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'name',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'requires_static_ip',
+    type: 'number'
+  },
+  {
+    name: 'e164_leading_plus',
+    type: 'number'
+  },
+  {
+    name: 'requires_register',
+    type: 'number'
+  },
+  {
+    name: 'register_username',
+    type: 'string'
+  },
+  {
+    name: 'register_sip_realm',
+    type: 'string'
+  },
+  {
+    name: 'register_password',
+    type: 'string'
+  },
+  {
+    name: 'tech_prefix',
+    type: 'string'
+  },
+  {
+    name: 'inbound_auth_username',
+    type: 'string'
+  },
+  {
+    name: 'inbound_auth_password',
+    type: 'string'
+  },
+  {
+    name: 'diversion',
+    type: 'string'
+  },
+];
+
+module.exports = PredefinedCarrier;

lib/models/product.js

@@ -0,0 +1,28 @@
+const Model = require('./model');
+
+class Product extends Model {
+  constructor() {
+    super();
+  }
+}
+
+Product.table = 'products';
+Product.fields = [
+  {
+    name: 'product_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'name',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'category',
+    type: 'string',
+    required: true
+  },
+];
+
+module.exports = Product;

lib/models/service-provider-limits.js

@@ -0,0 +1,39 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+const sql = 'SELECT * FROM service_provider_limits WHERE service_provider_sid = ?';
+
+class ServiceProviderLimits extends Model {
+  constructor() {
+    super();
+  }
+  static async retrieve(service_provider_sid) {
+    const [rows] = await promisePool.query(sql, [service_provider_sid]);
+    return rows;
+  }
+}
+
+ServiceProviderLimits.table = 'service_provider_limits';
+ServiceProviderLimits.fields = [
+  {
+    name: 'service_provider_limits_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'service_provider_sid',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'category',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'quantity',
+    type: 'number',
+    required: true
+  }
+];
+
+module.exports = ServiceProviderLimits;

lib/models/service-provider.js

@@ -89,6 +89,10 @@ ServiceProvider.fields = [
   {
     name: 'ms_teams_fqdn',
     type: 'string',
+  },
+  {
+    name: 'lcr_sid',
+    type: 'string'
   }
 
 ];

lib/models/sip-gateway.js

@@ -1,9 +1,18 @@
 const Model = require('./model');
+const {promisePool} = require('../db');
+const retrieveSql = 'SELECT * from sip_gateways WHERE voip_carrier_sid = ?';
 
 class SipGateway extends Model {
   constructor() {
     super();
   }
+  /**
+   * list all sip gateways for a voip_carrier
+   */
+  static async retrieveForVoipCarrier(voip_carrier_sid) {
+    const [rows] = await promisePool.query(retrieveSql, voip_carrier_sid);
+    return rows;
+  }
 }
 
 SipGateway.table = 'sip_gateways';
@@ -26,6 +35,10 @@ SipGateway.fields = [
     name: 'port',
     type: 'number'
   },
+  {
+    name: 'netmask',
+    type: 'number'
+  },
   {
     name: 'inbound',
     type: 'number'
@@ -38,6 +51,10 @@ SipGateway.fields = [
     name: 'is_active',
     type: 'number'
   },
+  {
+    name: 'pad_crypto',
+    type: 'number'
+  },
   {
     name: 'account_sid',
     type: 'string'
@@ -45,6 +62,10 @@ SipGateway.fields = [
   {
     name: 'application_sid',
     type: 'string'
+  },
+  {
+    name: 'protocol',
+    type: 'string'
   }
 ];
 

lib/models/smpp-gateway.js

@@ -0,0 +1,60 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+const retrieveSql = 'SELECT * from smpp_gateways WHERE voip_carrier_sid = ?';
+
+class SmppGateway extends Model {
+  constructor() {
+    super();
+  }
+  /**
+   * list all sip gateways for a voip_carrier
+   */
+  static async retrieveForVoipCarrier(voip_carrier_sid) {
+    const [rows] = await promisePool.query(retrieveSql, voip_carrier_sid);
+    return rows;
+  }
+}
+
+SmppGateway.table = 'smpp_gateways';
+SmppGateway.fields = [
+  {
+    name: 'smpp_gateway_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'voip_carrier_sid',
+    type: 'string'
+  },
+  {
+    name: 'ipv4',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'port',
+    type: 'number'
+  },
+  {
+    name: 'netmask',
+    type: 'number'
+  },
+  {
+    name: 'inbound',
+    type: 'number'
+  },
+  {
+    name: 'outbound',
+    type: 'number'
+  },
+  {
+    name: 'is_primary',
+    type: 'number'
+  },
+  {
+    name: 'use_tls',
+    type: 'number'
+  }
+];
+
+module.exports = SmppGateway;

lib/models/smpp.js

@@ -0,0 +1,55 @@
+const Model = require('./model');
+const {getMysqlConnection} = require('../db');
+
+class Smpp extends Model {
+  constructor() {
+    super();
+  }
+
+  /**
+  * list all SBCs either for a given service provider, or those not associated with a
+  * service provider (i.e. community SBCs)
+  */
+  static retrieveAll(service_provider_sid) {
+    const sql = service_provider_sid ?
+      'SELECT * from smpp_addresses WHERE service_provider_sid = ?' :
+      'SELECT * from smpp_addresses WHERE service_provider_sid IS NULL';
+    const args = service_provider_sid ? [service_provider_sid] : [];
+
+    return new Promise((resolve, reject) => {
+      getMysqlConnection((err, conn) => {
+        if (err) return reject(err);
+        conn.query(sql, args, (err, results) => {
+          conn.release();
+          if (err) return reject(err);
+          resolve(results);
+        });
+      });
+    });
+  }
+
+}
+
+Smpp.table = 'smpp_addresses';
+Smpp.fields = [
+  {
+    name: 'smpp_address_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'ipv4',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'port',
+    type: 'number'
+  },
+  {
+    name: 'service_provider_sid',
+    type: 'string'
+  }
+];
+
+module.exports = Smpp;

lib/models/speech-credential.js

@@ -0,0 +1,108 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+const retrieveSql = 'SELECT * from speech_credentials WHERE account_sid = ?';
+const retrieveSqlForSP = 'SELECT * from speech_credentials WHERE service_provider_sid = ? and account_sid is null';
+
+class SpeechCredential extends Model {
+  constructor() {
+    super();
+  }
+
+  /**
+   * list all credentials for an account
+   */
+  static async retrieveAll(account_sid) {
+    const [rows] = await promisePool.query(retrieveSql, account_sid);
+    return rows;
+  }
+  static async retrieveAllForSP(service_provider_sid) {
+    const [rows] = await promisePool.query(retrieveSqlForSP, service_provider_sid);
+    return rows;
+  }
+
+  static async getSpeechCredentialsByVendorAndLabel(service_provider_sid, account_sid, vendor, label) {
+    let sql;
+    if (account_sid) {
+      sql = `SELECT * FROM speech_credentials WHERE account_sid = ? AND vendor = ? ${label ? 'AND label = ?' : ''}`;
+    } else {
+      sql = `SELECT * FROM speech_credentials WHERE service_provider_sid = ? AND vendor = ?
+        ${label ? 'AND label = ?' : ''}`;
+    }
+    const [rows] = await promisePool.query(sql, [account_sid ? account_sid : service_provider_sid, vendor, label]);
+    return rows;
+  }
+
+  static async disableStt(account_sid) {
+    await promisePool.execute('UPDATE speech_credentials SET use_for_stt = 0 WHERE account_sid = ?', [account_sid]);
+  }
+  static async disableTts(account_sid) {
+    await promisePool.execute('UPDATE speech_credentials SET use_for_tts = 0 WHERE account_sid = ?', [account_sid]);
+  }
+
+  static async ttsTestResult(sid, success) {
+    await promisePool.execute(
+      'UPDATE speech_credentials SET last_tested = NOW(), tts_tested_ok = ? WHERE speech_credential_sid = ?',
+      [success, sid]);
+  }
+  static async sttTestResult(sid, success) {
+    await promisePool.execute(
+      'UPDATE speech_credentials SET last_tested = NOW(), stt_tested_ok = ? WHERE speech_credential_sid = ?',
+      [success, sid]);
+  }
+}
+
+SpeechCredential.table = 'speech_credentials';
+SpeechCredential.fields = [
+  {
+    name: 'speech_credential_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'account_sid',
+    type: 'string',
+  },
+  {
+    name: 'service_provider_sid',
+    type: 'string',
+  },
+  {
+    name: 'vendor',
+    type: 'string',
+    required: true,
+  },
+  {
+    name: 'credential',
+    type: 'string',
+  },
+  {
+    name: 'use_for_tts',
+    type: 'number'
+  },
+  {
+    name: 'use_for_stt',
+    type: 'number'
+  },
+  {
+    name: 'tts_tested_ok',
+    type: 'number'
+  },
+  {
+    name: 'stt_tested_ok',
+    type: 'number'
+  },
+  {
+    name: 'last_used',
+    type: 'date'
+  },
+  {
+    name: 'last_tested',
+    type: 'date'
+  },
+  {
+    name: 'label',
+    type: 'string'
+  }
+];
+
+module.exports = SpeechCredential;

lib/models/system-information.js

@@ -0,0 +1,38 @@
+const Model = require('./model');
+const { promisePool } = require('../db');
+class SystemInformation extends Model {
+  constructor() {
+    super();
+  }
+
+  static async add(body) {
+    let [sysInfo] = await this.retrieveAll();
+    if (sysInfo) {
+      const sql = `UPDATE ${this.table} SET ?`;
+      await promisePool.query(sql, body);
+    } else {
+      const sql = `INSERT INTO ${this.table} SET ?`;
+      await promisePool.query(sql, body);
+    }
+    [sysInfo] = await this.retrieveAll();
+    return sysInfo;
+  }
+}
+
+SystemInformation.table = 'system_information';
+SystemInformation.fields = [
+  {
+    name: 'domain_name',
+    type: 'string',
+  },
+  {
+    name: 'sip_domain_name',
+    type: 'string',
+  },
+  {
+    name: 'monitoring_domain_name',
+    type: 'string',
+  },
+];
+
+module.exports = SystemInformation;

lib/models/user.js

@@ -0,0 +1,117 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+const sqlAll = `
+SELECT u.user_sid, u.name, u.email, u.account_sid, u.service_provider_sid, u.is_active, 
+u.force_change, u.phone, u.pending_email, u.provider, u.provider_userid, 
+u.email_activation_code, u.email_validated, 
+sp.name as service_provider_name, acc.name as account_name  
+FROM users u   
+LEFT JOIN service_providers as sp ON u.service_provider_sid = sp.service_provider_sid 
+LEFT JOIN accounts acc ON u.account_sid = acc.account_sid  
+`;
+const sqlAccount = `
+SELECT u.user_sid, u.name, u.email, u.account_sid, u.service_provider_sid, u.is_active, 
+u.force_change, u.phone, u.pending_email, u.provider, u.provider_userid, 
+u.email_activation_code, u.email_validated, 
+sp.name as service_provider_name, acc.name as account_name  
+FROM users u   
+LEFT JOIN service_providers as sp ON u.service_provider_sid = sp.service_provider_sid 
+LEFT JOIN accounts acc ON u.account_sid = acc.account_sid  
+WHERE u.account_sid = ? 
+`;
+const sqlSP = `
+SELECT u.user_sid, u.name, u.email, u.account_sid, u.service_provider_sid, u.is_active, 
+u.force_change, u.phone, u.pending_email, u.provider, u.provider_userid, 
+u.email_activation_code, u.email_validated, 
+sp.name as service_provider_name, acc.name as account_name  
+FROM users u   
+LEFT JOIN service_providers as sp ON u.service_provider_sid = sp.service_provider_sid 
+LEFT JOIN accounts acc ON u.account_sid = acc.account_sid  
+WHERE u.service_provider_sid = ? 
+`;
+
+class User extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAll() {
+    const [rows] = await promisePool.query(sqlAll);
+    return rows;
+  }
+
+  static async retrieveAllForAccount(account_sid) {
+    const [rows] = await promisePool.query(sqlAccount, [account_sid]);
+    return rows;
+  }
+
+  static async retrieveAllForServiceProvider(service_provider_sid) {
+    const [rows] = await promisePool.query(sqlSP, [service_provider_sid]);
+    return rows;
+  }
+}
+
+User.table = 'users';
+User.fields = [
+  {
+    name: 'user_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'name',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'email',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'pending_email',
+    type: 'string'
+  },
+  {
+    name: 'phone',
+    type: 'string'
+  },
+  {
+    name: 'hashed_password',
+    type: 'string'
+  },
+  {
+    name: 'account_sid',
+    type: 'string'
+  },
+  {
+    name: 'service_provider_sid',
+    type: 'string'
+  },
+  {
+    name: 'force_change',
+    type: 'number'
+  },
+  {
+    name: 'provider',
+    type: 'string'
+  },
+  {
+    name: 'provider_userid',
+    type: 'string'
+  },
+  {
+    name: 'email_activation_code',
+    type: 'string'
+  },
+  {
+    name: 'email_validated',
+    type: 'number'
+  },
+  {
+    name: 'is_active',
+    type: 'number'
+  },
+];
+
+module.exports = User;

lib/models/voip-carrier.js

@@ -1,9 +1,28 @@
 const Model = require('./model');
+const {promisePool} = require('../db');
+const retrieveSql = 'SELECT * from voip_carriers vc WHERE vc.account_sid = ?';
+const retrieveSqlForSP = 'SELECT * from voip_carriers vc WHERE vc.service_provider_sid = ?';
+
 
 class VoipCarrier extends Model {
   constructor() {
     super();
   }
+  static async retrieveAll(account_sid) {
+    if (!account_sid) return super.retrieveAll();
+    const [rows] = await promisePool.query(retrieveSql, account_sid);
+    if (rows) {
+      rows.map((r) => r.register_status = JSON.parse(r.register_status || '{}'));
+    }
+    return rows;
+  }
+  static async retrieveAllForSP(service_provider_sid) {
+    const [rows] = await promisePool.query(retrieveSqlForSP, service_provider_sid);
+    if (rows) {
+      rows.map((r) => r.register_status = JSON.parse(r.register_status || '{}'));
+    }
+    return rows;
+  }
 }
 
 VoipCarrier.table = 'voip_carriers';
@@ -26,6 +45,10 @@ VoipCarrier.fields = [
     name: 'account_sid',
     type: 'string',
   },
+  {
+    name: 'service_provider_sid',
+    type: 'string',
+  },
   {
     name: 'application_sid',
     type: 'string'
@@ -33,6 +56,86 @@ VoipCarrier.fields = [
   {
     name: 'e164_leading_plus',
     type: 'number'
+  },
+  {
+    name: 'requires_register',
+    type: 'number'
+  },
+  {
+    name: 'register_use_tls',
+    type: 'number'
+  },
+  {
+    name: 'register_username',
+    type: 'string'
+  },
+  {
+    name: 'register_sip_realm',
+    type: 'string'
+  },
+  {
+    name: 'register_password',
+    type: 'string'
+  },
+  {
+    name: 'tech_prefix',
+    type: 'string'
+  },
+  {
+    name: 'inbound_auth_username',
+    type: 'string'
+  },
+  {
+    name: 'inbound_auth_password',
+    type: 'string'
+  },
+  {
+    name: 'diversion',
+    type: 'string'
+  },
+  {
+    name: 'is_active',
+    type: 'number'
+  },
+  {
+    name: 'smpp_system_id',
+    type: 'string'
+  },
+  {
+    name: 'smpp_password',
+    type: 'string'
+  },
+  {
+    name: 'smpp_inbound_system_id',
+    type: 'string'
+  },
+  {
+    name: 'smpp_inbound_password',
+    type: 'string'
+  },
+  {
+    name: 'smpp_enquire_link_interval',
+    type: 'number'
+  },
+  {
+    name: 'smpp_system_id',
+    type: 'string'
+  },
+  {
+    name: 'register_from_user',
+    type: 'string'
+  },
+  {
+    name: 'register_from_domain',
+    type: 'string'
+  },
+  {
+    name: 'register_public_ip_in_contact',
+    type: 'number'
+  },
+  {
+    name: 'register_status',
+    type: 'string'
   }
 ];
 

lib/record/azure-storage.js

@@ -0,0 +1,71 @@
+const { Writable } = require('stream');
+const { BlobServiceClient } = require('@azure/storage-blob');
+const { v4: uuidv4 } = require('uuid');
+const streamBuffers = require('stream-buffers');
+
+class AzureStorageUploadStream extends Writable {
+  constructor(logger, opts) {
+    super(opts);
+    const blobServiceClient = BlobServiceClient.fromConnectionString(opts.connection_string);
+    this.blockBlobClient = blobServiceClient.getContainerClient(opts.bucketName).getBlockBlobClient(opts.Key);
+
+    this.metadata = opts.metadata;
+    this.blocks = [];
+
+    this.bufferSize = 2 * 1024 * 1024; // Buffer size set to 2MB
+    this.buffer = new streamBuffers.WritableStreamBuffer({
+      initialSize: this.bufferSize,
+      incrementAmount: this.bufferSize
+    });
+  }
+
+  async _write(chunk, encoding, callback) {
+    this.buffer.write(chunk, encoding);
+
+    if (this.buffer.size() >= this.bufferSize) {
+      const blockID = uuidv4().replace(/-/g, '');
+      this.blocks.push(blockID);
+
+      try {
+        const dataToWrite = this.buffer.getContents();
+        await this.blockBlobClient.stageBlock(blockID, dataToWrite, dataToWrite.length);
+        callback();
+      } catch (error) {
+        callback(error);
+      }
+    } else {
+      callback();
+    }
+  }
+
+  async _final(callback) {
+    // Write any remaining data in buffer
+    if (this.buffer.size() > 0) {
+      const remainingData = this.buffer.getContents();
+      const blockID = uuidv4().replace(/-/g, '');
+      this.blocks.push(blockID);
+
+      try {
+        await this.blockBlobClient.stageBlock(blockID, remainingData, remainingData.length);
+      } catch (error) {
+        callback(error);
+        return;
+      }
+    }
+
+    try {
+      await this.blockBlobClient.commitBlockList(this.blocks);
+      // remove all null/undefined props
+      const filteredObj = Object.entries(this.metadata).reduce((acc, [key, val]) => {
+        if (val !== undefined && val !== null) acc[key] = val;
+        return acc;
+      }, {});
+      await this.blockBlobClient.setMetadata(filteredObj);
+      callback();
+    } catch (error) {
+      callback(error);
+    }
+  }
+}
+
+module.exports = AzureStorageUploadStream;

lib/record/encoder.js

@@ -0,0 +1,61 @@
+const { Transform } = require('stream');
+const lamejs = require('@jambonz/lamejs');
+
+class PCMToMP3Encoder extends Transform {
+  constructor(options, logger) {
+    super(options);
+
+    const channels = options.channels || 1;
+    const sampleRate = options.sampleRate || 8000;
+    const bitRate = options.bitRate || 128;
+
+    this.encoder = new lamejs.Mp3Encoder(channels, sampleRate, bitRate);
+    this.channels = channels;
+    this.logger = logger;
+  }
+
+  _transform(chunk, encoding, callback) {
+    try {
+      // Convert chunk buffer into Int16Array for lamejs
+      const samples = new Int16Array(chunk.buffer, chunk.byteOffset, chunk.length / 2);
+
+      // Split input samples into left and right channel arrays if stereo
+      let leftChannel, rightChannel;
+      if (this.channels === 2) {
+        leftChannel = new Int16Array(samples.length / 2);
+        rightChannel = new Int16Array(samples.length / 2);
+
+        for (let i = 0; i < samples.length; i += 2) {
+          leftChannel[i / 2] = samples[i];
+          rightChannel[i / 2] = samples[i + 1];
+        }
+      } else {
+        leftChannel = samples;
+      }
+
+      // Encode the input data
+      const mp3Data = this.encoder.encodeBuffer(leftChannel, rightChannel);
+
+      if (mp3Data.length > 0) {
+        this.push(Buffer.from(mp3Data));
+      }
+      callback();
+    } catch (err) {
+      this.logger.error(
+        { err },
+        'Error while mp3 transform');
+    }
+  }
+
+  _flush(callback) {
+    // Finalize encoding and flush the internal buffers
+    const mp3Data = this.encoder.flush();
+
+    if (mp3Data.length > 0) {
+      this.push(Buffer.from(mp3Data));
+    }
+    callback();
+  }
+}
+
+module.exports = PCMToMP3Encoder;

lib/record/google-storage.js

@@ -0,0 +1,62 @@
+const { Storage } = require('@google-cloud/storage');
+const { Writable } = require('stream');
+const streamBuffers = require('stream-buffers');
+
+class GoogleStorageUploadStream extends Writable {
+
+  constructor(logger, opts) {
+    super(opts);
+    this.logger = logger;
+    this.metadata = opts.metadata;
+
+    const storage = new Storage(opts.bucketCredential);
+    this.gcsFile = storage.bucket(opts.bucketName).file(opts.Key);
+    this.writeStream = this.gcsFile.createWriteStream();
+
+    this.bufferSize = 2 * 1024 * 1024; // Buffer size set to 2MB
+    this.buffer = new streamBuffers.WritableStreamBuffer({
+      initialSize: this.bufferSize,
+      incrementAmount: this.bufferSize
+    });
+
+    this.writeStream.on('error', (err) => this.logger.error(err));
+    this.writeStream.on('finish', () => {
+      this.logger.info('Google storage Upload completed.');
+      this._addMetadata();
+    });
+  }
+
+  _write(chunk, encoding, callback) {
+    this.buffer.write(chunk, encoding);
+
+    // Write to GCS when buffer reaches desired size
+    if (this.buffer.size() >= this.bufferSize) {
+      const dataToWrite = this.buffer.getContents();
+      this.writeStream.write(dataToWrite, callback);
+    } else {
+      callback();
+    }
+  }
+
+  _final(callback) {
+    // Write any remaining data in the buffer to GCS
+    if (this.buffer.size() > 0) {
+      const remainingData = this.buffer.getContents();
+      this.writeStream.write(remainingData);
+    }
+
+    this.writeStream.end();
+    this.writeStream.once('finish', callback);
+  }
+
+  async _addMetadata() {
+    try {
+      await this.gcsFile.setMetadata({metadata: this.metadata});
+      this.logger.info('Google storage Upload and metadata setting completed.');
+    } catch (err) {
+      this.logger.error(err, 'Google storage An error occurred while setting metadata');
+    }
+  }
+}
+
+module.exports = GoogleStorageUploadStream;

lib/record/index.js

@@ -0,0 +1,6 @@
+
+async function record(logger, socket) {
+  return require('./upload')(logger, socket);
+}
+
+module.exports = record;

lib/record/s3-multipart-upload-stream.js

@@ -0,0 +1,103 @@
+const { Writable } = require('stream');
+const {
+  S3Client,
+  CreateMultipartUploadCommand,
+  UploadPartCommand,
+  CompleteMultipartUploadCommand,
+} = require('@aws-sdk/client-s3');
+
+class S3MultipartUploadStream extends Writable {
+  constructor(logger, opts) {
+    super(opts);
+    this.logger = logger;
+    this.bucketName = opts.bucketName;
+    this.objectKey = opts.Key;
+    this.uploadId = null;
+    this.partNumber = 1;
+    this.multipartETags = [];
+    this.buffer = Buffer.alloc(0);
+    this.minPartSize = 5 * 1024 * 1024; // 5 MB
+    this.s3 = new S3Client(opts.bucketCredential);
+    this.metadata = opts.metadata;
+  }
+
+  async _initMultipartUpload() {
+    const command = new CreateMultipartUploadCommand({
+      Bucket: this.bucketName,
+      Key: this.objectKey,
+      Metadata: this.metadata
+    });
+    const response = await this.s3.send(command);
+    return response.UploadId;
+  }
+
+  async _uploadBuffer() {
+    const uploadPartCommand = new UploadPartCommand({
+      Bucket: this.bucketName,
+      Key: this.objectKey,
+      PartNumber: this.partNumber,
+      UploadId: this.uploadId,
+      Body: this.buffer,
+    });
+
+    const uploadPartResponse = await this.s3.send(uploadPartCommand);
+    this.multipartETags.push({
+      ETag: uploadPartResponse.ETag,
+      PartNumber: this.partNumber,
+    });
+    this.partNumber += 1;
+  }
+
+  async _write(chunk, encoding, callback) {
+    try {
+      if (!this.uploadId) {
+        this.uploadId = await this._initMultipartUpload();
+      }
+
+      this.buffer = Buffer.concat([this.buffer, chunk]);
+
+      if (this.buffer.length >= this.minPartSize) {
+        await this._uploadBuffer();
+        this.buffer = Buffer.alloc(0);
+      }
+
+      callback(null);
+    } catch (error) {
+      callback(error);
+    }
+  }
+
+  async _finalize(err) {
+    try {
+      if (this.buffer.length > 0) {
+        await this._uploadBuffer();
+      }
+
+      const completeMultipartUploadCommand = new CompleteMultipartUploadCommand({
+        Bucket: this.bucketName,
+        Key: this.objectKey,
+        MultipartUpload: {
+          Parts: this.multipartETags.sort((a, b) => a.PartNumber - b.PartNumber),
+        },
+        UploadId: this.uploadId,
+      });
+
+      await this.s3.send(completeMultipartUploadCommand);
+      this.logger.info('Finished upload to S3');
+    } catch (error) {
+      this.logger.error('Error completing multipart upload:', error);
+      throw error;
+    }
+  }
+
+  async _final(callback) {
+    try {
+      await this._finalize();
+      callback(null);
+    } catch (error) {
+      callback(error);
+    }
+  }
+}
+
+module.exports = S3MultipartUploadStream;

lib/record/upload.js

@@ -0,0 +1,97 @@
+const Account = require('../models/account');
+const Websocket = require('ws');
+const PCMToMP3Encoder = require('./encoder');
+const wav = require('wav');
+const { getUploader } = require('./utils');
+const { pipeline } = require('stream');
+
+async function upload(logger, socket) {
+  socket._recvInitialMetadata = false;
+  socket.on('message', async function(data, isBinary) {
+    try {
+      if (!isBinary && !socket._recvInitialMetadata) {
+        socket._recvInitialMetadata = true;
+        logger.debug(`initial metadata: ${data}`);
+        const obj = JSON.parse(data.toString());
+        logger.info({ obj }, 'received JSON message from jambonz');
+        const { sampleRate, accountSid, callSid, direction, from, to,
+          callId, applicationSid, originatingSipIp, originatingSipTrunkName } = obj;
+        const account = await Account.retrieve(accountSid);
+        if (account && account.length && account[0].bucket_credential) {
+          const obj = account[0].bucket_credential;
+          // add tags to metadata
+          const metadata = {
+            accountSid,
+            callSid,
+            direction,
+            from,
+            to,
+            callId,
+            applicationSid,
+            originatingSipIp,
+            originatingSipTrunkName,
+            sampleRate: `${sampleRate}`
+          };
+          if (obj.tags && obj.tags.length) {
+            obj.tags.forEach((tag) => {
+              metadata[tag.Key] = tag.Value;
+            });
+          }
+          // create S3 path
+          const day = new Date();
+          let key = `${day.getFullYear()}/${(day.getMonth() + 1).toString().padStart(2, '0')}`;
+          key += `/${day.getDate().toString().padStart(2, '0')}/${callSid}.${account[0].record_format}`;
+
+          // Uploader
+          const uploadStream = getUploader(key, metadata, obj, logger);
+          if (!uploadStream) {
+            logger.info('There is no available record uploader, close the socket.');
+            socket.close();
+          }
+
+          /**encoder */
+          let encoder;
+          if (account[0].record_format === 'wav') {
+            encoder = new wav.Writer({ channels: 2, sampleRate, bitDepth: 16 });
+          } else {
+            // default is mp3
+            encoder = new PCMToMP3Encoder({
+              channels: 2,
+              sampleRate: sampleRate,
+              bitrate: 128
+            }, logger);
+          }
+
+          /* start streaming data */
+          pipeline(
+            Websocket.createWebSocketStream(socket),
+            encoder,
+            uploadStream,
+            (error) => {
+              if (error) {
+                logger.error({ error }, 'pipeline error, cannot upload data to storage');
+                socket.close();
+              }
+            }
+          );
+        } else {
+          logger.info(`account ${accountSid} does not have any bucket credential, close the socket`);
+          socket.close();
+        }
+      }
+    } catch (err) {
+      logger.error({ err, data }, 'error parsing message during connection');
+    }
+  });
+  socket.on('error', function(err) {
+    logger.error({ err }, 'record upload: error');
+  });
+  socket.on('close', (data) => {
+    logger.info({ data }, 'record upload: close');
+  });
+  socket.on('end', function(err) {
+    logger.error({ err }, 'record upload: socket closed from jambonz');
+  });
+}
+
+module.exports = upload;

lib/record/utils.js

@@ -0,0 +1,58 @@
+const AzureStorageUploadStream = require('./azure-storage');
+const GoogleStorageUploadStream = require('./google-storage');
+const S3MultipartUploadStream = require('./s3-multipart-upload-stream');
+
+const getUploader = (key, metadata, bucket_credential, logger) => {
+  const uploaderOpts = {
+    bucketName: bucket_credential.name,
+    Key: key,
+    metadata
+  };
+  try {
+    switch (bucket_credential.vendor) {
+      case 'aws_s3':
+        uploaderOpts.bucketCredential = {
+          credentials: {
+            accessKeyId: bucket_credential.access_key_id,
+            secretAccessKey: bucket_credential.secret_access_key,
+          },
+          region: bucket_credential.region || 'us-east-1'
+        };
+        return new S3MultipartUploadStream(logger, uploaderOpts);
+      case 's3_compatible':
+        uploaderOpts.bucketCredential = {
+          endpoint: bucket_credential.endpoint,
+          credentials: {
+            accessKeyId: bucket_credential.access_key_id,
+            secretAccessKey: bucket_credential.secret_access_key,
+          },
+          region: 'us-east-1',
+          forcePathStyle: true
+        };
+        return new S3MultipartUploadStream(logger, uploaderOpts);
+      case 'google':
+        const serviceKey = JSON.parse(bucket_credential.service_key);
+        uploaderOpts.bucketCredential = {
+          projectId: serviceKey.project_id,
+          credentials: {
+            client_email: serviceKey.client_email,
+            private_key: serviceKey.private_key
+          }
+        };
+        return new GoogleStorageUploadStream(logger, uploaderOpts);
+      case 'azure':
+        uploaderOpts.connection_string = bucket_credential.connection_string;
+        return new AzureStorageUploadStream(logger, uploaderOpts);
+      default:
+        logger.error(`unknown bucket vendor: ${bucket_credential.vendor}`);
+        break;
+    }
+  } catch (err) {
+    logger.error(`Error creating uploader, vendor: ${bucket_credential.vendor}, reason: ${err.message}`);
+  }
+  return null;
+};
+
+module.exports = {
+  getUploader
+};

lib/routes/api/account-test.js

@@ -0,0 +1,57 @@
+const router = require('express').Router();
+const {promisePool} = require('../../db');
+const sysError = require('../error');
+const retrieveApplicationsSql = `SELECT * from applications app
+LEFT JOIN webhooks AS ch
+ON app.call_hook_sid = ch.webhook_sid
+LEFT JOIN webhooks AS sh
+ON app.call_status_hook_sid = sh.webhook_sid 
+LEFT JOIN webhooks AS mh
+ON app.messaging_hook_sid = mh.webhook_sid
+WHERE service_provider_sid = ?`;
+
+const transmogrifyResults = (results) => {
+  return results.map((row) => {
+    const obj = row.app;
+    if (row.ch && Object.keys(row.ch).length && row.ch.url !== null) {
+      Object.assign(obj, {call_hook: row.ch});
+    }
+    else obj.call_hook = null;
+    if (row.sh && Object.keys(row.sh).length && row.sh.url !== null) {
+      Object.assign(obj, {call_status_hook: row.sh});
+    }
+    else obj.call_status_hook = null;
+    if (row.mh && Object.keys(row.mh).length && row.mh.url !== null) {
+      Object.assign(obj, {messaging_hook: row.mh});
+    }
+    else obj.messaging_hook = null;
+    delete obj.call_hook_sid;
+    delete obj.call_status_hook_sid;
+    delete obj.messaging_hook_sid;
+    return obj;
+  });
+};
+
+router.get('/:service_provider_sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const {service_provider_sid} = req.params;
+
+  try {
+    const [r] = await promisePool.query('SELECT * from service_providers where service_provider_sid = ?',
+      service_provider_sid);
+    if (r.length === 0) {
+      logger.info(`/AccountTest invalid service_provider_sid ${service_provider_sid}`);
+      return res.sendStatus(404);
+    }
+
+    const [numbers] = await promisePool.query('SELECT number FROM phone_numbers WHERE service_provider_sid = ?',
+      service_provider_sid);
+    const [results] = await promisePool.query({sql: retrieveApplicationsSql, nestTables: true}, service_provider_sid);
+
+    res.json({phonenumbers: numbers.map((n) => n.number), applications: transmogrifyResults(results)});
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/activation-code.js

@@ -0,0 +1,117 @@
+const router = require('express').Router();
+const debug = require('debug')('jambonz:api-server');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const {promisePool} = require('../../db');
+const {validateEmail, emailSimpleText} = require('../../utils/email-utils');
+const sysError = require('../error');
+const sqlRetrieveUser = `SELECT * from users user
+LEFT JOIN accounts AS account
+ON user.account_sid = account.account_sid
+WHERE user.user_sid = ?`;
+
+const validateRequest = async(req, res) => {
+  const payload = req.body || {};
+
+  /* valid type */
+  if (!['email', 'phone'].includes(payload.type)) {
+    throw new DbErrorBadRequest(`invalid activation type: ${payload.type}`);
+  }
+
+  /* valid user? */
+  const [rows] = await promisePool.query('SELECT * from users WHERE user_sid = ?',
+    payload.user_sid);
+  if (0 === rows.length) throw new DbErrorBadRequest('invalid user_sid');
+
+  /* valid email? */
+  if (payload.type === 'email' && !validateEmail(payload.value)) throw new DbErrorBadRequest('invalid email');
+
+};
+
+router.post('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const {user_sid, type, code, value} = req.body;
+
+  try {
+    await validateRequest(req, res);
+
+    const fields = type === 'email' ?
+      ['email', 'email_validated', 'email_activation_code'] :
+      ['phone', 'phone_validated', 'phone_activation_code'];
+    const sql =
+    `UPDATE users set ${fields[0]} = ?, ${fields[1]} = 0,  ${fields[2]} = ? WHERE user_sid = ?`;
+    const [r] = await promisePool.execute(sql, [value, code, user_sid]);
+    logger.debug({r}, 'Result from adding activation code');
+    debug({r}, 'Result from adding activation code');
+
+    if (process.env.NODE_ENV !== 'test') {
+      if (type === 'email') {
+        /* send code via email */
+        const text = '';
+        const subject = '';
+        await emailSimpleText(logger, value, subject, text);
+      }
+      else {
+        /* send code via SMS */
+      }
+    }
+    res.sendStatus(204);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.put('/:code', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const code = req.params.code;
+  const {user_sid, type} = req.body;
+
+  try {
+    let activateAccount = false;
+    let deactivateOldUsers = false;
+    let account_sid;
+    if (type === 'email') {
+      /* check whether this is first-time activation of account during sign-up/register */
+      const [r] = await promisePool.query({sql: sqlRetrieveUser, nestTables: true}, user_sid);
+      logger.debug({r}, 'activationcode - selected user');
+      if (r.length) {
+        const {user, account} = r[0];
+        account_sid = account.account_sid;
+        const [otherUsers] = await promisePool.query('SELECT * from users WHERE account_sid = ? AND user_sid <> ?',
+          [account_sid, user_sid]);
+        logger.debug({otherUsers}, `activationcode - users other than ${user_sid}`);
+        if (0 === otherUsers.length && user.provider === 'local' && !user.email_validated) {
+          logger.debug('activationcode - activating account');
+          activateAccount = true;
+        }
+        else if (otherUsers.length) {
+          logger.debug('activationcode - adding new user for existing account');
+          deactivateOldUsers = true;
+        }
+      }
+    }
+    const fields = type === 'email' ?
+      ['email_validated', 'email_activation_code'] :
+      ['phone_validated', 'phone_activation_code'];
+    const sql = `UPDATE users set ${fields[0]} = 1, ${fields[1]} = NULL WHERE ${fields[1]} = ? AND user_sid = ?`;
+    const [r] = await promisePool.execute(sql, [code, user_sid]);
+    logger.debug({r}, 'Result from validating code');
+    debug({r}, 'Result from validating code');
+
+    if (activateAccount) {
+      await promisePool.execute('UPDATE accounts SET is_active=1 WHERE account_sid = ?', [account_sid]);
+    }
+    else if (deactivateOldUsers) {
+      const [r] = await promisePool.execute('DELETE FROM users WHERE account_sid = ? AND user_sid <> ?',
+        [account_sid, user_sid]);
+      logger.debug({r}, 'Result from deleting old/replaced users');
+    }
+
+    if (1 === r.affectedRows) return res.sendStatus(204);
+    throw new DbErrorBadRequest('invalid user or activation code');
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+
+module.exports = router;

lib/routes/api/add-from-predefined-carrier.js

@@ -0,0 +1,82 @@
+const router = require('express').Router();
+const PredefinedCarrier = require('../../models/predefined-carrier');
+const VoipCarrier = require('../../models/voip-carrier');
+const SipGateway = require('../../models/sip-gateway');
+const SmppGateway = require('../../models/smpp-gateway');
+const {parseServiceProviderSid} = require('./utils');
+const short = require('short-uuid');
+const {promisePool} = require('../../db');
+const sysError = require('../error');
+
+
+const sqlSelectCarrierByNameForSP = `SELECT * FROM voip_carriers 
+WHERE service_provider_sid = ? 
+AND name = ?`;
+const sqlSelectTemplateSipGateways = `SELECT * FROM predefined_sip_gateways 
+WHERE predefined_carrier_sid = ?`;
+const sqlSelectTemplateSmppGateways = `SELECT * FROM predefined_smpp_gateways 
+WHERE predefined_carrier_sid = ?`;
+
+
+router.post('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const {sid } = req.params;
+  let service_provider_sid;
+  const {account_sid} = req.user;
+
+  try {
+    if (!account_sid) {
+      service_provider_sid = parseServiceProviderSid(req);
+    } else {
+      service_provider_sid = req.user.service_provider_sid;
+    }
+
+    const [template] = await PredefinedCarrier.retrieve(sid);
+    logger.debug({template}, `Retrieved template carrier for sid ${sid}`);
+    if (!template) return res.sendStatus(404);
+
+    /* make sure not to add the same carrier twice */
+    const [r2] = await promisePool.query(sqlSelectCarrierByNameForSP, [service_provider_sid, template.name]);
+
+    if (r2.length > 0) {
+      template.name =  `${template.name}-${short.generate()}`;
+    }
+
+    /* retrieve all the sip gateways */
+    const [r3] = await promisePool.query(sqlSelectTemplateSipGateways, template.predefined_carrier_sid);
+    logger.debug({r3}, `retrieved template sip gateways for ${template.name}`);
+
+    /* retrieve all the smpp gateways */
+    const [r4] = await promisePool.query(sqlSelectTemplateSmppGateways, template.predefined_carrier_sid);
+    logger.debug({r4}, `retrieved template smpp gateways for ${template.name}`);
+
+    /* add a voip_carrier */
+    // eslint-disable-next-line no-unused-vars
+    const {requires_static_ip, predefined_carrier_sid, ...obj} = template;
+    const uuid = await VoipCarrier.make({...obj, account_sid, service_provider_sid});
+
+    /* add all the sipp gateways */
+    for (const gw of r3) {
+      // eslint-disable-next-line no-unused-vars
+      const {predefined_carrier_sid, predefined_sip_gateway_sid, ...obj} = gw;
+      logger.debug({obj}, 'adding sip gateway');
+      await SipGateway.make({...obj, voip_carrier_sid: uuid});
+    }
+
+    /* add all the smpp gateways */
+    for (const gw of r4) {
+      // eslint-disable-next-line no-unused-vars
+      const {predefined_carrier_sid, predefined_smpp_gateway_sid, ...obj} = gw;
+      logger.debug({obj}, 'adding smpp gateway');
+      await SmppGateway.make({...obj, voip_carrier_sid: uuid});
+    }
+
+    logger.debug({sid: uuid}, 'Successfully added carrier from predefined list');
+    res.status(201).json({sid: uuid});
+  } catch (err) {
+    logger.error({err}, 'Error adding voip_carrier from template');
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/alerts.js

@@ -0,0 +1,52 @@
+const router = require('express').Router();
+const sysError = require('../error');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const { parseServiceProviderSid } = require('./utils');
+
+const parseAccountSid = (url) => {
+  const arr = /Accounts\/([^\/]*)/.exec(url);
+  if (arr) return arr[1];
+};
+
+router.get('/', async(req, res) => {
+  const {logger, queryAlerts, queryAlertsSP} = req.app.locals;
+  try {
+    logger.debug({opts: req.query}, 'GET /Alerts');
+    const account_sid = parseAccountSid(req.originalUrl);
+    const service_provider_sid = account_sid ? null : parseServiceProviderSid(req.originalUrl);
+    const {page, count, alert_type, days, start, end} = req.query || {};
+    if (!page || page < 1) throw new DbErrorBadRequest('missing or invalid "page" query arg');
+    if (!count || count < 25 || count > 500) throw new DbErrorBadRequest('missing or invalid "count" query arg');
+
+    if (account_sid) {
+      const data = await queryAlerts({
+        account_sid,
+        page,
+        page_size: count,
+        alert_type,
+        days,
+        start: days ? undefined : start,
+        end: days ? undefined : end,
+      });
+
+      res.status(200).json(data);
+    }
+    else {
+      const data = await queryAlertsSP({
+        service_provider_sid,
+        page,
+        page_size: count,
+        alert_type,
+        days,
+        start: days ? undefined : start,
+        end: days ? undefined : end,
+      });
+
+      res.status(200).json(data);
+    }
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/api-keys.js

@@ -3,9 +3,8 @@ const {DbErrorBadRequest} = require('../../utils/errors');
 const ApiKey = require('../../models/api-key');
 const Account = require('../../models/account');
 const decorate = require('./decorate');
-const uuidv4 = require('uuid/v4');
-const assert = require('assert');
-const sysError = require('./error');
+const { v4: uuidv4 } = require('uuid');
+const sysError = require('../error');
 const preconditions = {
   'add': validateAddToken,
   'delete': validateDeleteToken
@@ -71,10 +70,7 @@ async function validateDeleteToken(req, sid) {
 router.post('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    if ('add' in preconditions) {
-      assert(typeof preconditions.add === 'function');
-      await preconditions.add(req);
-    }
+    await validateAddToken(req);
     const uuid = await ApiKey.make(req.body);
     res.status(201).json({sid: uuid, token: req.body.token});
   } catch (err) {

lib/routes/api/applications.js

@@ -1,14 +1,46 @@
 const router = require('express').Router();
-const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, DbErrorUnprocessableRequest, DbErrorForbidden} = require('../../utils/errors');
 const Application = require('../../models/application');
 const Account = require('../../models/account');
 const Webhook = require('../../models/webhook');
+const {promisePool} = require('../../db');
 const decorate = require('./decorate');
-const sysError = require('./error');
+const sysError = require('../error');
+const { validate } = require('@jambonz/verb-specifications');
+const { parseApplicationSid } = require('./utils');
 const preconditions = {
   'add': validateAdd,
-  'update': validateUpdate,
-  'delete': validateDelete
+  'update': validateUpdate
+};
+
+const validateRequest = async(req, account_sid) => {
+  try {
+    if (req.user.hasScope('admin')) {
+      return;
+    }
+
+    if (req.user.hasScope('account')) {
+      if (account_sid === req.user.account_sid) {
+        return;
+      }
+
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      const [r] = await promisePool.execute(
+        'SELECT service_provider_sid from accounts WHERE account_sid = ?', [account_sid]
+      );
+
+      if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+        return;
+      }
+
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+  } catch (error) {
+    throw error;
+  }
 };
 
 /* only user-level tokens can add applications */
@@ -21,7 +53,7 @@ async function validateAdd(req) {
     if (!req.body.account_sid) throw new DbErrorBadRequest('missing required field: \'account_sid\'');
     const result = await Account.retrieve(req.body.account_sid, req.user.service_provider_sid);
     if (result.length === 0) {
-      throw new DbErrorBadRequest('insufficient privileges to create an application under the specified account');
+      throw new DbErrorForbidden('insufficient privileges');
     }
   }
   if (req.body.call_hook && typeof req.body.call_hook !== 'object') {
@@ -33,9 +65,26 @@ async function validateAdd(req) {
 }
 
 async function validateUpdate(req, sid) {
-  if (req.user.account_sid && sid !== req.user.account_sid) {
-    throw new DbErrorBadRequest('you may not update or delete an application associated with a different account');
+  const app = await Application.retrieve(sid);
+  if (req.user.hasAccountAuth) {
+    if (!app || 0 === app.length || app[0].account_sid !== req.user.account_sid) {
+      throw new DbErrorForbidden('insufficient privileges');
+    }
   }
+
+  if (req.user.hasServiceProviderAuth) {
+    const [r] = await promisePool.execute(
+      'SELECT service_provider_sid from accounts WHERE account_sid = ?', [app[0].account_sid]
+    );
+
+    if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+      return;
+    }
+
+    throw new DbErrorForbidden('insufficient permissions');
+  }
+
+
   if (req.body.call_hook && typeof req.body.call_hook !== 'object') {
     throw new DbErrorBadRequest('\'call_hook\' must be an object when updating an application');
   }
@@ -45,14 +94,29 @@ async function validateUpdate(req, sid) {
 }
 
 async function validateDelete(req, sid) {
-  if (req.user.account_sid && sid !== req.user.account_sid) {
-    throw new DbErrorBadRequest('you may not update or delete an application associated with a different account');
+  const result = await Application.retrieve(sid);
+  if (req.user.hasAccountAuth) {
+    if (!result || 0 === result.length) throw new DbErrorBadRequest('application does not exist');
+    if (result[0].account_sid !== req.user.account_sid) {
+      throw new DbErrorUnprocessableRequest('insufficient permissions');
+    }
+  }
+  if (req.user.hasServiceProviderAuth) {
+    const [r] = await promisePool.execute(
+      'SELECT service_provider_sid from accounts WHERE account_sid = ?', [result[0].account_sid]
+    );
+
+    if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+      return;
+    }
+
+    throw new DbErrorForbidden('insufficient permissions');
   }
   const assignedPhoneNumbers = await Application.getForeignKeyReferences('phone_numbers.application_sid', sid);
   if (assignedPhoneNumbers > 0) throw new DbErrorUnprocessableRequest('cannot delete application with phone numbers');
 }
 
-decorate(router, Application, ['delete'], preconditions);
+decorate(router, Application, [], preconditions);
 
 /* add */
 router.post('/', async(req, res) => {
@@ -69,6 +133,16 @@ router.post('/', async(req, res) => {
       }
     }
 
+    // validate app json if required
+    if (obj['app_json']) {
+      const app_json = JSON.parse(obj['app_json']);
+      try {
+        validate(logger, app_json);
+      } catch (err) {
+        throw new DbErrorBadRequest(err);
+      }
+    }
+
     const uuid = await Application.make(obj);
     res.status(201).json({sid: uuid});
   } catch (err) {
@@ -93,22 +167,65 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const application_sid = parseApplicationSid(req);
     const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
     const account_sid = req.user.hasAccountAuth ? req.user.account_sid : null;
-    const results = await Application.retrieve(req.params.sid, service_provider_sid, account_sid);
+    const results = await Application.retrieve(application_sid, service_provider_sid, account_sid);
     if (results.length === 0) return res.status(404).end();
-    return res.status(200).json(results);
+    await validateRequest(req, results[0].account_sid);
+    return res.status(200).json(results[0]);
   }
   catch (err) {
     sysError(logger, res, err);
   }
 });
 
-/* update */
-router.put('/:sid', async(req, res) => {
-  const sid = req.params.sid;
+/* delete */
+router.delete('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const sid = parseApplicationSid(req);
+    await validateDelete(req, sid);
+
+    const [application] = await promisePool.query('SELECT * FROM applications WHERE application_sid = ?', sid);
+    const {call_hook_sid, call_status_hook_sid, messaging_hook_sid} = application[0];
+    logger.info({call_hook_sid, call_status_hook_sid, messaging_hook_sid, sid}, 'deleting application');
+    await promisePool.execute('DELETE from applications where application_sid = ?', [sid]);
+
+    if (call_hook_sid) {
+      /* remove call hook if only used by this app */
+      const sql = 'SELECT COUNT(*) as count FROM applications WHERE call_hook_sid = ?';
+      const [r] = await promisePool.query(sql, call_hook_sid);
+      if (r[0]?.count === 0) {
+        await promisePool.execute('DELETE from webhooks where webhook_sid = ?', [call_hook_sid]);
+      }
+    }
+    if (call_status_hook_sid) {
+      const sql = 'SELECT COUNT(*) as count FROM applications WHERE call_status_hook_sid = ?';
+      const [r] = await promisePool.query(sql, call_status_hook_sid);
+      if (r[0]?.count === 0) {
+        await promisePool.execute('DELETE from webhooks where webhook_sid = ?', [call_status_hook_sid]);
+      }
+    }
+    if (messaging_hook_sid) {
+      const sql = 'SELECT COUNT(*) as count FROM applications WHERE messaging_hook_sid = ?';
+      const [r] = await promisePool.query(sql, messaging_hook_sid);
+      if (r[0]?.count === 0) {
+        await promisePool.execute('DELETE from webhooks where webhook_sid = ?', [messaging_hook_sid]);
+      }
+    }
+
+    res.status(204).end();
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+/* update */
+router.put('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const sid = parseApplicationSid(req);
     await validateUpdate(req, sid);
 
     // create webhooks if provided
@@ -131,6 +248,16 @@ router.put('/:sid', async(req, res) => {
       delete obj[prop];
     }
 
+    // validate app json if required
+    if (obj['app_json']) {
+      const app_json = JSON.parse(obj['app_json']);
+      try {
+        validate(logger, app_json);
+      } catch (err) {
+        throw new DbErrorBadRequest(err);
+      }
+    }
+
     const rowsAffected = await Application.update(sid, obj);
     if (rowsAffected === 0) {
       return res.status(404).end();

lib/routes/api/availability.js

@@ -0,0 +1,31 @@
+const router = require('express').Router();
+const {DbErrorBadRequest} = require('../../utils/errors');
+const {promisePool} = require('../../db');
+const sysError = require('../error');
+
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const {type, value} = req.query;
+
+  try {
+
+    if (['email', 'phone'].includes(type)) {
+      const field = type === 'email' ? 'email' : 'phone';
+      const sql = `SELECT * from users WHERE ${field} = ?`;
+      const [r] = await promisePool.execute(sql, [value]);
+      res.json({available: 0 === r.length});
+    }
+    else if (type === 'subdomain') {
+      const sql = 'SELECT * from accounts WHERE sip_realm = ?';
+      const [r] = await promisePool.execute(sql, [value]);
+      res.json({available: 0 === r.length});
+    }
+    else throw new DbErrorBadRequest(`invalid type: ${type}`);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+
+module.exports = router;

lib/routes/api/beta-invite-codes.js

@@ -0,0 +1,32 @@
+const router = require('express').Router();
+const sysError = require('../error');
+const {promisePool} = require('../../db');
+const short = require('short-uuid');
+const translator = short('0123456789ABCXZ');
+
+router.post('/', async(req, res) => {
+  const {logger} = req.app.locals;
+  logger.debug({payload: req.body}, 'POST /BetaInviteCodes');
+  try {
+    const {count} = req.body || {};
+    const total = Math.max(count || 1, 1);
+    const codes = [];
+    let added = 0;
+    while (added < total) {
+      const code = translator.new().substring(0, 6);
+      if (!codes.find((c) => c === code)) {
+        codes.push(code);
+        added++;
+      }
+    }
+
+    const values = codes.map((c) => `('${c}')`).join(',');
+    const sql = `INSERT INTO beta_invite_codes (invite_code) VALUES ${values}`;
+    const [r] = await promisePool.query(sql);
+    res.status(200).json({status: 'ok', added: r.affectedRows, codes});
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/change-password.js

@@ -0,0 +1,50 @@
+const router = require('express').Router();
+const {DbErrorBadRequest} = require('../../utils/errors');
+const {generateHashedPassword, verifyPassword} = require('../../utils/password-utils');
+const {promisePool} = require('../../db');
+const {cacheClient} = require('../../helpers');
+const sysError = require('../error');
+const sqlUpdatePassword = `UPDATE users 
+SET hashed_password= ? 
+WHERE user_sid = ?`;
+
+router.post('/', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {user_sid} = req.user;
+  const {old_password, new_password} = req.body;
+  try {
+    if (!old_password || !new_password) throw new DbErrorBadRequest('missing old_password or new_password');
+
+    /* validate existing password */
+    {
+      const [r] = await promisePool.query('SELECT * from users where user_sid = ?', user_sid);
+      logger.debug({user: [r[0]]}, 'change password for user');
+
+      if (r[0].provider !== 'local') {
+        throw new DbErrorBadRequest('user is using oauth authentication');
+      }
+
+      const isCorrect = await verifyPassword(r[0].hashed_password, old_password);
+      if (!isCorrect) {
+        const key = cacheClient.generateRedisKey('reset-link', old_password);
+        const user_sid = await cacheClient.get(key);
+        if (!user_sid) throw new DbErrorBadRequest('old_password is incorrect');
+        await cacheClient.delete(key);
+      }
+    }
+
+    /* store new password */
+    const passwordHash = await generateHashedPassword(new_password);
+    const [r] = await promisePool.execute(sqlUpdatePassword, [passwordHash, user_sid]);
+    if (r.affectedRows !== 1) throw new Error('failed to update user with new password');
+    res.sendStatus(204);
+  } catch (err) {
+    sysError(logger, res, err);
+    return;
+  }
+
+  const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+  await cacheClient.delete(redisKey);
+});
+
+module.exports = router;

lib/routes/api/charges.js

@@ -0,0 +1,46 @@
+const router = require('express').Router();
+const sysError = require('../error');
+//const {DbErrorUnprocessableRequest, DbErrorBadRequest} = require('../../utils/errors');
+
+
+/**
+ * retrieve charges for an account and/or call
+ */
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    res.status(200).json([
+      {
+        charge_sid: 'f8d2a604-ed29-4eac-9efc-8f58b0e438ca',
+        account_sid: req.user.account_sid,
+        call_billing_record_sid: 'e4be80a4-6597-49cf-8605-6b94493fada1',
+        billed_at: '2020-01-01 15:10:10',
+        billed_activity: 'outbound-call',
+        call_secs_billed: 392,
+        amount_charged: 0.0200
+      },
+      {
+        charge_sid: 'd9659f3f-3a94-455c-9e8e-3b36f250ffc8',
+        account_sid: req.user.account_sid,
+        call_billing_record_sid: 'e4be80a4-6597-49cf-8605-6b94493fada1',
+        billed_at: '2020-01-01 15:10:10',
+        billed_activity: 'tts',
+        tts_chars_billed: 100,
+        amount_charged: 0.0130
+      },
+      {
+        charge_sid: 'adcc1e79-eb79-4370-ab74-4c2e9a41339a',
+        account_sid: req.user.account_sid,
+        call_billing_record_sid: 'e4be80a4-6597-49cf-8605-6b94493fada1',
+        billed_at: '2020-01-01 15:10:10',
+        billed_activity:  'stt',
+        stt_secs_billed: 30,
+        amount_charged: 0.0015
+      }
+    ]);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/clients.js

@@ -0,0 +1,88 @@
+const router = require('express').Router();
+const decorate = require('./decorate');
+const sysError = require('../error');
+const Client = require('../../models/client');
+const Account = require('../../models/account');
+const { DbErrorBadRequest, DbErrorForbidden } = require('../../utils/errors');
+const { encrypt, decrypt, obscureKey } = require('../../utils/encrypt-decrypt');
+
+const commonCheck = async(req) => {
+  if (req.user.hasAccountAuth) {
+    req.body.account_sid = req.user.account_sid;
+  } else if (req.user.hasServiceProviderAuth && req.body.account_sid) {
+    const accounts = await Account.retrieve(req.body.account_sid, req.user.service_provider_sid);
+    if (accounts.length === 0) {
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+  }
+
+  if (req.body.password) {
+    req.body.password = encrypt(req.body.password);
+  }
+};
+
+const validateAdd = async(req) => {
+  await commonCheck(req);
+
+  const clients = await Client.retrieveByAccountSidAndUserName(req.body.account_sid, req.body.username);
+  if (clients.length) {
+    throw new DbErrorBadRequest('the client\'s username already exists');
+  }
+};
+
+const validateUpdate = async(req, sid) => {
+  await commonCheck(req);
+
+  const clients = await Client.retrieveByAccountSidAndUserName(req.body.account_sid, req.body.username);
+  if (clients.length && clients[0].client_sid !== sid) {
+    throw new DbErrorBadRequest('the client\'s username already exists');
+  }
+};
+
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+};
+
+decorate(router, Client, ['add', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = req.user.hasAdminAuth ?
+      await Client.retrieveAll() : req.user.hasAccountAuth ?
+        await Client.retrieveAllByAccountSid(req.user.hasAccountAuth ? req.user.account_sid : null) :
+        await Client.retrieveAllByServiceProviderSid(req.user.service_provider_sid);
+    const ret = results.map((c) => {
+      c.password = obscureKey(decrypt(c.password), 1);
+      return c;
+    });
+    res.status(200).json(ret);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = await Client.retrieve(req.params.sid);
+    if (results.length === 0) return res.sendStatus(404);
+    const client = results[0];
+    client.password = obscureKey(decrypt(client.password), 1);
+    if (req.user.hasAccountAuth && client.account_sid !== req.user.account_sid) {
+      return res.sendStatus(404);
+    } else if (req.user.hasServiceProviderAuth) {
+      const accounts = await Account.retrieve(client.account_sid, req.user.service_provider_sid);
+      if (!accounts.length) {
+        return res.sendStatus(404);
+      }
+    }
+    return res.status(200).json(client);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/decorate.js

@@ -1,5 +1,5 @@
 const assert = require('assert');
-const sysError = require('./error');
+const sysError = require('../error');
 
 module.exports = decorate;
 
@@ -58,7 +58,7 @@ function retrieve(router, klass) {
     const logger = req.app.locals.logger;
     try {
       const results = await klass.retrieve(req.params.sid);
-      if (results.length === 0) return res.status(404).end();
+      if (results.length === 0) return res.sendStatus(404);
       return res.status(200).json(results[0]);
     }
     catch (err) {
@@ -78,7 +78,7 @@ function update(router, klass, preconditions) {
       }
       const rowsAffected = await klass.update(sid, req.body);
       if (rowsAffected === 0) {
-        return res.status(404).end();
+        return res.sendStatus(404);
       }
       res.status(204).end();
     } catch (err) {
@@ -99,9 +99,9 @@ function remove(router, klass, preconditions) {
       const rowsAffected = await klass.remove(sid);
       if (rowsAffected === 0) {
         logger.info(`unable to delete ${klass.name} with sid ${sid}: not found`);
-        return res.status(404).end();
+        return res.sendStatus(404);
       }
-      res.status(204).end();
+      res.sendStatus(204);
     } catch (err) {
       sysError(logger, res, err);
     }

lib/routes/api/error.js

@@ -1,6 +1,10 @@
-const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const { BadRequestError, DbErrorBadRequest, DbErrorUnprocessableRequest } = require('../../utils/errors');
 
 function sysError(logger, res, err) {
+  if (err instanceof BadRequestError) {
+    logger.info(err, err.message);
+    return res.status(400).json({msg: 'Bad request'});
+  }
   if (err instanceof DbErrorBadRequest) {
     logger.info(err, 'invalid client request');
     return res.status(400).json({msg: err.message});

lib/routes/api/forgot-password.js

@@ -0,0 +1,95 @@
+const router = require('express').Router();
+//const debug = require('debug')('jambonz:api-server');
+const short = require('short-uuid');
+const translator = short();
+const {validateEmail, emailSimpleText} = require('../../utils/email-utils');
+const {promisePool} = require('../../db');
+const {cacheClient} = require('../../helpers');
+const sysError = require('../error');
+const assert = require('assert');
+const sql = `SELECT * from users user 
+LEFT JOIN accounts AS acc 
+ON acc.account_sid = user.account_sid 
+WHERE user.email = ?`;
+
+function createOauthEmailText(provider) {
+  return `Hi there!
+
+  Someone (presumably you!) requested to reset their password.
+  However, the account associated with this email is using oauth identification via ${provider},
+  Please change your password through that provider, if you wish to.
+  
+  If you did not make this request, please delete this email.  No further action is required.
+  
+  Best,
+  
+  Jambonz support team`;
+}
+
+function createResetEmailText(link) {
+  assert(process.env.JAMBONZ_BASE_URL, 'process.env.JAMBONZ_BASE_URL is missing');
+  const baseUrl = process.env.JAMBONZ_BASE_URL;
+
+  return `Hi there!
+
+  Someone (presumably you!) requested to reset their password.
+  Please follow the link below to reset your password:
+  
+  ${baseUrl}/reset-password/${link}
+  
+  This link is valid for 1 hour only.
+  If you did not make this request, please delete this email.  No further action is required.
+  
+  Best,
+  
+  Jambonz support team`;
+}
+
+router.post('/', async(req, res) => {
+  const {logger, addKey} = req.app.locals;
+  const {email} = req.body;
+
+  let obj;
+  try {
+    if (!email || !validateEmail(email)) {
+      return res.status(400).json({error: 'invalid or missing email'});
+    }
+
+    const [r] = await promisePool.query({sql, nestTables: true}, email);
+    if (0 === r.length) {
+      logger.info('user not found');
+      return res.status(400).json({error: 'failed to reset your password'});
+    }
+    obj = r[0];
+    if (!obj.user.is_active) {
+      logger.info(obj.user.name, 'user is inactive');
+      return res.status(400).json({error: 'failed to reset your password'});
+    } else if (obj.acc.account_sid !== null && !obj.acc.is_active) {
+      logger.info(obj.acc.account_sid, 'account is inactive');
+      return res.status(400).json({error: 'failed to reset your password'});
+    }
+    res.sendStatus(204);
+  } catch (err) {
+    sysError(logger, res, err);
+    return;
+  }
+
+  if (obj.user.provider !== 'local') {
+    /* send email indicating they need to change via their oauth provider */
+    emailSimpleText(logger, email, 'Reset password request', createOauthEmailText(obj.user.provider));
+
+  }
+  else {
+    /* generate a link for this user to reset, send email */
+    const link = translator.generate();
+    const redisKey = cacheClient.generateRedisKey('reset-link', link);
+    addKey(redisKey, obj.user.user_sid, 3600)
+      .catch((err) => logger.error({err}, 'Error adding reset link to redis'));
+    emailSimpleText(logger, email, 'Reset password request', createResetEmailText(link));
+  }
+
+  const redisKey = cacheClient.generateRedisKey('jwt', obj.user.user_sid, 'v2');
+  await cacheClient.delete(redisKey);
+});
+
+module.exports = router;

lib/routes/api/google-custom-voices.js

@@ -0,0 +1,77 @@
+const router = require('express').Router();
+const GoogleCustomVoice = require('../../models/google-custom-voice');
+const SpeechCredential = require('../../models/speech-credential');
+const decorate = require('./decorate');
+const {DbErrorBadRequest, DbErrorForbidden} = require('../../utils/errors');
+const sysError = require('../error');
+
+const validateCredentialPermission = async(req) => {
+  const credential = await SpeechCredential.retrieve(req.body.speech_credential_sid);
+  if (!credential || credential.length === 0) {
+    throw new DbErrorBadRequest('Invalid speech_credential_sid');
+  }
+  const cred = credential[0];
+
+  if (req.user.hasServiceProviderAuth && cred.service_provider_sid !== req.user.service_provider_sid) {
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+  if (req.user.hasAccountAuth && cred.account_sid !== req.user.account_sid) {
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+};
+
+const validateAdd = async(req) => {
+  if (!req.body.speech_credential_sid) {
+    throw new DbErrorBadRequest('missing speech_credential_sid');
+  }
+
+  await validateCredentialPermission(req);
+};
+
+const validateUpdate = async(req) => {
+  if (req.body.speech_credential_sid) {
+    await validateCredentialPermission(req);
+  }
+};
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+};
+
+decorate(router, GoogleCustomVoice, ['add', 'retrieve', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const account_sid = req.user.account_sid || req.query.account_sid;
+  const service_provider_sid = req.user.service_provider_sid || req.query.service_provider_sid;
+  const speech_credential_sid = req.query.speech_credential_sid;
+  const label = req.query.label;
+  try {
+    let results = [];
+    if (speech_credential_sid) {
+      const [cred] = await SpeechCredential.retrieve(speech_credential_sid);
+      if (!cred) {
+        return res.sendStatus(404);
+      }
+      if (account_sid && cred.account_sid && cred.account_sid !== account_sid) {
+        throw new DbErrorForbidden('Insufficient privileges');
+      }
+      if (service_provider_sid && cred.service_provider_sid && cred.service_provider_sid !== service_provider_sid) {
+        throw new DbErrorForbidden('Insufficient privileges');
+      }
+      results = await GoogleCustomVoice.retrieveAllBySpeechCredentialSid(speech_credential_sid);
+    } else {
+      if (!account_sid && !service_provider_sid) {
+        throw new DbErrorBadRequest('missing account_sid or service_provider_sid in query parameters');
+      }
+      results = await GoogleCustomVoice.retrieveAllByLabel(service_provider_sid, account_sid, label);
+    }
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+
+module.exports = router;

lib/routes/api/index.js

@@ -1,26 +1,62 @@
 const api = require('express').Router();
 
-function isAdminScope(req, res, next) {
+const isAdminScope = (req, res, next) => {
   if (req.user.hasScope('admin')) return next();
   res.status(403).json({
     status: 'fail',
     message: 'insufficient privileges'
   });
-}
+};
+// const isAdminOrSPScope = (req, res, next) => {
+//   if (req.user.hasScope('admin') || req.user.hasScope('service_provider')) return next();
+//   res.status(403).json({
+//     status: 'fail',
+//     message: 'insufficient privileges'
+//   });
+// };
 
-api.use('/ServiceProviders', isAdminScope, require('./service-providers'));
-api.use('/VoipCarriers', isAdminScope, require('./voip-carriers'));
-api.use('/SipGateways', isAdminScope, require('./sip-gateways'));
-api.use('/PhoneNumbers', isAdminScope, require('./phone-numbers'));
+api.use('/BetaInviteCodes', isAdminScope, require('./beta-invite-codes'));
+api.use('/SystemInformation', isAdminScope, require('./system-information'));
+api.use('/TtsCache', isAdminScope, require('./tts-cache'));
+api.use('/ServiceProviders', require('./service-providers'));
+api.use('/VoipCarriers', require('./voip-carriers'));
+api.use('/Webhooks', require('./webhooks'));
+api.use('/SipGateways', require('./sip-gateways'));
+api.use('/SmppGateways', require('./smpp-gateways'));
+api.use('/PhoneNumbers', require('./phone-numbers'));
 api.use('/ApiKeys', require('./api-keys'));
 api.use('/Accounts', require('./accounts'));
 api.use('/Applications', require('./applications'));
 api.use('/MicrosoftTeamsTenants', require('./tenants'));
-api.use('/Sbcs', isAdminScope, require('./sbcs'));
+api.use('/Sbcs', require('./sbcs'));
 api.use('/Users', require('./users'));
+api.use('/register', require('./register'));
+api.use('/signin', require('./signin'));
 api.use('/login', require('./login'));
+api.use('/logout', require('./logout'));
+api.use('/forgot-password', require('./forgot-password'));
+api.use('/change-password', require('./change-password'));
+api.use('/ActivationCode', require('./activation-code'));
+api.use('/Availability', require('./availability'));
+api.use('/AccountTest', require('./account-test'));
+//api.use('/Products', require('./products'));
+api.use('/Prices', require('./prices'));
+api.use('/StripeCustomerId', require('./stripe-customer-id'));
+api.use('/Subscriptions', require('./subscriptions'));
+api.use('/Invoices', require('./invoices'));
+api.use('/InviteCodes', require('./invite-codes'));
+api.use('/PredefinedCarriers', require('./predefined-carriers'));
+api.use('/PasswordSettings', require('./password-settings'));
+// Least Cost Routing
+api.use('/Lcrs', require('./lcrs'));
+api.use('/LcrRoutes', require('./lcr-routes'));
+api.use('/LcrCarrierSetEntries', require('./lcr-carrier-set-entries'));
+api.use('/Clients', require('./clients'));
+// Google Custom Voices
+api.use('/GoogleCustomVoices', require('./google-custom-voices'));
 
 // messaging
+api.use('/Smpps', require('./smpps'));   // our smpp server info
 api.use('/messaging', require('./sms-inbound'));      // inbound SMS from carrier
 api.use('/outboundSMS', require('./sms-outbound'));   // outbound SMS from feature server
 

lib/routes/api/invite-codes.js

@@ -0,0 +1,34 @@
+const router = require('express').Router();
+const sysError = require('../error');
+const {promisePool} = require('../../db');
+const sqlClaim = `UPDATE beta_invite_codes 
+SET in_use = 1 
+WHERE invite_code = ? 
+AND in_use = 0`;
+const sqlTest = `SELECT * FROM beta_invite_codes 
+WHERE invite_code = ? 
+AND in_use = 0`;
+router.post('/', async(req, res) => {
+  const {logger} = req.app.locals;
+  try {
+    const {code, test} = req.body;
+    logger.debug({code}, 'POST /InviteCodes');
+    if ('test' === process.env.NODE_ENV) {
+      if (code.endsWith('0')) return res.sendStatus(404);
+      res.sendStatus(204);
+      return;
+    }
+    if (test) {
+      const [r] = await promisePool.execute(sqlTest, [code]);
+      res.sendStatus(1 === r.length ? 204 : 404);
+    }
+    else {
+      const [r] = await promisePool.execute(sqlClaim, [code]);
+      res.sendStatus(1 === r.affectedRows ? 204 : 404);
+    }
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/invoices.js

@@ -0,0 +1,26 @@
+const router = require('express').Router();
+const assert = require('assert');
+const Account = require('../../models/account');
+const {
+  retrieveUpcomingInvoice
+} = require('../../utils/stripe-utils');
+const sysError = require('../error');
+
+/* retrieve */
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const {account_sid} = req.user;
+  try {
+    const results = await Account.retrieve(account_sid);
+    assert.ok(1 === results.length, `account ${account_sid} not found`);
+    const {stripe_customer_id} = results[0];
+    if (!stripe_customer_id) return res.sendStatus(404);
+    const invoice = await retrieveUpcomingInvoice(logger, stripe_customer_id);
+    res.status(200).json(invoice);
+  } catch (err) {
+    if (err.statusCode) return res.sendStatus(err.statusCode);
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/lcr-carrier-set-entries.js

@@ -0,0 +1,65 @@
+const router = require('express').Router();
+const LcrCarrierSetEntry = require('../../models/lcr-carrier-set-entry');
+const LcrRoute = require('../../models/lcr-route');
+const decorate = require('./decorate');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const sysError = require('../error');
+
+const validateAdd = async(req) => {
+  const {lookupCarrierBySid} = req.app.locals;
+  if (!req.body.lcr_route_sid) {
+    throw new DbErrorBadRequest('missing lcr_route_sid');
+  }
+  // check lcr_route_sid is exist
+  const lcrRoute = await LcrRoute.retrieve(req.body.lcr_route_sid);
+  if (lcrRoute.length === 0) {
+    throw new DbErrorBadRequest('unknown lcr_route_sid');
+  }
+  // check voip_carrier_sid is exist
+  if (!req.body.voip_carrier_sid) {
+    throw new DbErrorBadRequest('missing voip_carrier_sid');
+  }
+  const carrier = await lookupCarrierBySid(req.body.voip_carrier_sid);
+  if (!carrier) {
+    throw new DbErrorBadRequest('unknown voip_carrier_sid');
+  }
+};
+
+const validateUpdate = async(req) => {
+  const {lookupCarrierBySid} = req.app.locals;
+  if (req.body.lcr_route_sid) {
+    const lcrRoute = await LcrRoute.retrieve(req.body.lcr_route_sid);
+    if (lcrRoute.length === 0) {
+      throw new DbErrorBadRequest('unknown lcr_route_sid');
+    }
+  }
+
+  // check voip_carrier_sid is exist
+  if (req.body.voip_carrier_sid) {
+    const carrier = await lookupCarrierBySid(req.body.voip_carrier_sid);
+    if (!carrier) {
+      throw new DbErrorBadRequest('unknown voip_carrier_sid');
+    }
+  }
+};
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+};
+
+decorate(router, LcrCarrierSetEntry, ['add', 'retrieve', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const lcr_route_sid = req.query.lcr_route_sid;
+  try {
+    const results = await LcrCarrierSetEntry.retrieveAllByLcrRouteSid(lcr_route_sid);
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+
+module.exports = router;

lib/routes/api/lcr-routes.js

@@ -0,0 +1,96 @@
+const router = require('express').Router();
+const LcrRoute = require('../../models/lcr-route');
+const Lcr = require('../../models/lcr');
+const LcrCarrierSetEntry = require('../../models/lcr-carrier-set-entry');
+const decorate = require('./decorate');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const sysError = require('../error');
+
+const validateAdd = async(req) => {
+  // check if lcr sid is available
+  if (!req.body.lcr_sid) {
+    throw new DbErrorBadRequest('missing parameter lcr_sid');
+  }
+
+  const lcr = await Lcr.retrieve(req.body.lcr_sid);
+  if (lcr.length === 0) {
+    throw new DbErrorBadRequest('unknown lcr_sid');
+  }
+};
+
+const validateUpdate = async(req) => {
+  if (req.body.lcr_sid) {
+    const lcr = await Lcr.retrieve(req.body.lcr_sid);
+    if (lcr.length === 0) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+};
+
+const validateDelete = async(req, sid) => {
+  // delete all lcr carrier set entries
+  await LcrCarrierSetEntry.deleteByLcrRouteSid(sid);
+};
+
+const checkUserScope = async(req, lcr_sid) => {
+  if (!lcr_sid) {
+    throw new DbErrorBadRequest('missing lcr_sid');
+  }
+
+  if (req.user.hasAdminAuth) return;
+
+  const lcrList = await Lcr.retrieve(lcr_sid);
+  if (lcrList.length === 0) throw new DbErrorBadRequest('unknown lcr_sid');
+  const lcr = lcrList[0];
+  if (req.user.hasAccountAuth) {
+    if (!lcr.account_sid || lcr.account_sid !== req.user.account_sid) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+
+  if (req.user.hasServiceProviderAuth) {
+    if (!lcr.service_provider_sid  || lcr.service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+};
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+  delete: validateDelete,
+};
+
+decorate(router, LcrRoute, ['add', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const lcr_sid = req.query.lcr_sid;
+  try {
+    await checkUserScope(req, lcr_sid);
+    const results = await LcrRoute.retrieveAllByLcrSid(lcr_sid);
+    for (const r of results) {
+      r.lcr_carrier_set_entries = await LcrCarrierSetEntry.retrieveAllByLcrRouteSid(r.lcr_route_sid);
+    }
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const lcr_route_sid = req.params.sid;
+  try {
+    const results = await LcrRoute.retrieve(lcr_route_sid);
+    if (results.length === 0) return res.sendStatus(404);
+    const route = results[0];
+    route.lcr_carrier_set_entries = await LcrCarrierSetEntry.retrieveAllByLcrRouteSid(route.lcr_route_sid);
+    res.status(200).json(route);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+
+module.exports = router;

lib/routes/api/lcrs.js

@@ -0,0 +1,225 @@
+const router = require('express').Router();
+const Lcr = require('../../models/lcr');
+const LcrCarrierSetEntry = require('../../models/lcr-carrier-set-entry');
+const LcrRoutes = require('../../models/lcr-route');
+const decorate = require('./decorate');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const sysError = require('../error');
+const ServiceProvider = require('../../models/service-provider');
+
+const validateAssociatedTarget = async(req, sid) => {
+  const {lookupAccountBySid} = req.app.locals;
+  if (req.body.account_sid) {
+    // Add only for account
+    req.body.service_provider_sid = null;
+    const account = await lookupAccountBySid(req.body.account_sid);
+    if (!account) throw new DbErrorBadRequest('unknown account_sid');
+    const lcr = await Lcr.retrieveAllByAccountSid(req.body.account_sid);
+    if (lcr.length > 0 && (!sid || sid !== lcr[0].lcr_sid)) {
+      throw new DbErrorBadRequest(`Account: ${account.name}  already has an active call routing table.`);
+    }
+  } else if (req.body.service_provider_sid) {
+    const serviceProviders = await ServiceProvider.retrieve(req.body.service_provider_sid);
+    if (serviceProviders.length === 0) throw new DbErrorBadRequest('unknown service_provider_sid');
+    const serviceProvider = serviceProviders[0];
+    const lcr = await Lcr.retrieveAllByServiceProviderSid(req.body.service_provider_sid);
+    if (lcr.length > 0 && (!sid || sid !== lcr[0].lcr_sid)) {
+      throw new DbErrorBadRequest(`Service Provider: ${serviceProvider.name} already 
+      has an active call routing table.`);
+    }
+  }
+};
+
+const validateAdd = async(req) => {
+  if (req.user.hasAccountAuth) {
+    // Account just create LCR for himself
+    req.body.account_sid = req.user.account_sid;
+  } else if (req.user.hasServiceProviderAuth) {
+    // SP just can create LCR for himself
+    req.body.service_provider_sid = req.user.service_provider_sid;
+    req.body.account_sid = null;
+  }
+
+  await validateAssociatedTarget(req);
+  // check if lcr_carrier_set_entry is available
+  if (req.body.lcr_carrier_set_entry) {
+    const e = await LcrCarrierSetEntry.retrieve(req.body.lcr_carrier_set_entry);
+    if (e.length === 0) throw new DbErrorBadRequest('unknown lcr_carrier_set_entry');
+  }
+
+};
+
+const validateUserPermissionForExistingEntity = async(req, sid) => {
+  const r = await Lcr.retrieve(sid);
+  if (r.length === 0) {
+    throw new DbErrorBadRequest('unknown lcr_sid');
+  }
+  const lcr = r[0];
+  if (req.user.hasAccountAuth) {
+    if (lcr.account_sid != req.user.account_sid) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  } else if (req.user.hasServiceProviderAuth) {
+    if (lcr.service_provider_sid != req.user.service_provider_sid) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+};
+
+const validateUpdate = async(req, sid) => {
+  await validateUserPermissionForExistingEntity(req, sid);
+  await validateAssociatedTarget(req, sid);
+};
+
+const validateDelete = async(req, sid) => {
+  if (req.user.hasAccountAuth) {
+    /* can only delete Lcr for the user's account */
+    const r = await Lcr.retrieve(sid);
+    const lcr = r.length > 0 ? r[0] : null;
+    if (!lcr || (req.user.account_sid && lcr.account_sid != req.user.account_sid)) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+  await Lcr.releaseDefaultEntry(sid);
+  // fetch lcr route
+  const lcr_routes = await LcrRoutes.retrieveAllByLcrSid(sid);
+  // delete all lcr carrier set entries
+  for (const e of lcr_routes) {
+    await LcrCarrierSetEntry.deleteByLcrRouteSid(e.lcr_route_sid);
+  }
+
+  // delete all lcr routes
+  await LcrRoutes.deleteByLcrSid(sid);
+};
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+  delete: validateDelete
+};
+
+decorate(router, Lcr, ['add', 'update', 'delete'], preconditions);
+
+const validateLcrBatchAdd = async(lcr_sid, body, lookupCarrierBySid) => {
+  for (const lcr_route of body) {
+    lcr_route.lcr_sid = lcr_sid;
+    if (!lcr_route.lcr_carrier_set_entries || lcr_route.lcr_carrier_set_entries.length === 0) {
+      throw new DbErrorBadRequest('Lcr Route batch process require lcr_carrier_set_entries');
+    }
+    for (const entry of lcr_route.lcr_carrier_set_entries) {
+      // check voip_carrier_sid is exist
+      if (!entry.voip_carrier_sid) {
+        throw new DbErrorBadRequest('One of lcr_carrier_set_entries is missing voip_carrier_sid');
+      }
+      const carrier = await lookupCarrierBySid(entry.voip_carrier_sid);
+      if (!carrier) {
+        throw new DbErrorBadRequest('unknown voip_carrier_sid');
+      }
+    }
+  }
+};
+
+
+const addNewLcrRoute = async(lcr_route) => {
+  const lcr_sid = lcr_route.lcr_sid;
+  const lcr_carrier_set_entries = lcr_route.lcr_carrier_set_entries;
+  delete lcr_route.lcr_carrier_set_entries;
+  const lcr_route_sid = await LcrRoutes.make(lcr_route);
+  for (const entry of lcr_carrier_set_entries) {
+    entry.lcr_route_sid = lcr_route_sid;
+    const lcr_carrier_set_entry_sid = await LcrCarrierSetEntry.make(entry);
+    if (lcr_route.priority === 9999) {
+      // this is default lcr set entry
+      const [lcr] = await Lcr.retrieve(lcr_sid);
+      if (lcr) {
+        lcr.default_carrier_set_entry_sid = lcr_carrier_set_entry_sid;
+        delete lcr.lcr_sid;
+        await Lcr.update(lcr_sid, lcr);
+      }
+    }
+  }
+};
+
+router.post('/:sid/Routes', async(req, res) => {
+  const results = await Lcr.retrieve(req.params.sid);
+  if (results.length === 0) return res.sendStatus(404);
+  const {logger, lookupCarrierBySid} = req.app.locals;
+  try {
+    const body = req.body;
+    await validateLcrBatchAdd(req.params.sid, body, lookupCarrierBySid);
+    for (const lcr_route of body) {
+      await addNewLcrRoute(lcr_route, lookupCarrierBySid);
+    }
+    res.sendStatus(204);
+
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.put('/:sid/Routes', async(req, res) => {
+  const results = await Lcr.retrieve(req.params.sid);
+  if (results.length === 0) return res.sendStatus(404);
+  const {logger, lookupCarrierBySid} = req.app.locals;
+  try {
+    const body = req.body;
+    await validateLcrBatchAdd(req.params.sid, body, lookupCarrierBySid);
+    for (const lcr_route of body) {
+      if (lcr_route.lcr_route_sid) {
+        const lcr_route_sid = lcr_route.lcr_route_sid;
+        delete lcr_route.lcr_route_sid;
+        const lcr_carrier_set_entries = lcr_route.lcr_carrier_set_entries;
+        delete lcr_route.lcr_carrier_set_entries;
+        await LcrRoutes.update(lcr_route_sid, lcr_route);
+        for (const entry of lcr_carrier_set_entries) {
+          const lcr_carrier_set_entry_sid = entry.lcr_carrier_set_entry_sid;
+          delete entry.lcr_carrier_set_entry_sid;
+          await LcrCarrierSetEntry.update(lcr_carrier_set_entry_sid, entry);
+        }
+      } else {
+        // Route is not available yet, let create it now
+        await addNewLcrRoute(lcr_route, lookupCarrierBySid);
+      }
+    }
+    res.sendStatus(204);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = req.user.hasAdminAuth ?
+      await Lcr.retrieveAll() : req.user.hasAccountAuth ?
+        await Lcr.retrieveAllByAccountSid(req.user.hasAccountAuth ? req.user.account_sid : null) :
+        await Lcr.retrieveAllByServiceProviderSid(req.user.service_provider_sid);
+
+    for (const lcr of results) {
+      lcr.number_routes = await LcrRoutes.countAllByLcrSid(lcr.lcr_sid);
+    }
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = await Lcr.retrieve(req.params.sid);
+    if (results.length === 0) return res.sendStatus(404);
+    const lcr = results[0];
+    if (req.user.hasAccountAuth && lcr.account_sid !== req.user.account_sid) {
+      return res.sendStatus(404);
+    } else if (req.user.hasServiceProviderAuth && lcr.service_provider_sid !== req.user.service_provider_sid) {
+      return res.sendStatus(404);
+    }
+    lcr.number_routes = await LcrRoutes.countAllByLcrSid(lcr.lcr_sid);
+    return res.status(200).json(lcr);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/limits.js

@@ -0,0 +1,133 @@
+const router = require('express').Router();
+const sysError = require('../error');
+const AccountLimits = require('../../models/account-limits');
+const ServiceProviderLimits = require('../../models/service-provider-limits');
+const {parseAccountSid, parseServiceProviderSid} = require('./utils');
+const {promisePool} = require('../../db');
+const sqlDeleteSPLimits = `
+DELETE FROM service_provider_limits 
+WHERE service_provider_sid = ? 
+`;
+const sqlDeleteSPLimitsByCategory = `
+DELETE FROM service_provider_limits 
+WHERE service_provider_sid = ? 
+AND category = ?
+`;
+const sqlDeleteAccountLimits = `
+DELETE FROM account_limits 
+WHERE account_sid = ? 
+`;
+const sqlDeleteAccountLimitsByCategory = `
+DELETE FROM account_limits 
+WHERE account_sid = ? 
+AND category = ?
+`;
+
+router.post('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const {
+    category,
+    quantity
+  } = req.body;
+
+  try {
+    let service_provider_sid;
+    const account_sid = parseAccountSid(req);
+    if (!account_sid) {
+      if (!req.user.hasServiceProviderAuth && !req.user.hasAdminAuth) {
+        logger.error('POST /SpeechCredentials invalid credentials');
+        return res.sendStatus(403);
+      }
+      service_provider_sid = parseServiceProviderSid(req);
+    }
+
+    let uuid;
+    if (account_sid) {
+      const existing = (await AccountLimits.retrieve(account_sid) || [])
+        .find((el) => el.category === category);
+      if (existing) {
+        uuid = existing.account_limits_sid;
+        await AccountLimits.update(uuid, {category, quantity});
+      }
+      else {
+        uuid = await AccountLimits.make({
+          account_sid,
+          category,
+          quantity
+        });
+      }
+    }
+    else {
+      const existing = (await ServiceProviderLimits.retrieve(service_provider_sid) || [])
+        .find((el) => el.category === category);
+      if (existing) {
+        uuid = existing.service_provider_limits_sid;
+        await ServiceProviderLimits.update(uuid, {category, quantity});
+      }
+      else {
+        uuid = await ServiceProviderLimits.make({
+          service_provider_sid,
+          category,
+          quantity
+        });
+      }
+    }
+    res.status(201).json({sid: uuid});
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+/**
+ * retrieve all limits for an account or service provider
+ */
+router.get('/', async(req, res) => {
+  let service_provider_sid;
+  const logger = req.app.locals.logger;
+
+  try {
+    const account_sid = parseAccountSid(req);
+    if (!account_sid) service_provider_sid = parseServiceProviderSid(req);
+    const limits = account_sid ?
+      await AccountLimits.retrieve(account_sid) :
+      await ServiceProviderLimits.retrieve(service_provider_sid);
+
+    if (req.query?.category) {
+      return res.status(200).json(limits.filter((el) => el.category === req.query.category));
+    }
+    res.status(200).json(limits);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.delete('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+
+  try {
+    const account_sid = parseAccountSid(req);
+    const {category} = req.query;
+    const service_provider_sid = parseServiceProviderSid(req);
+    if (account_sid) {
+      if (category) {
+        await promisePool.execute(sqlDeleteAccountLimitsByCategory, [account_sid, category]);
+      }
+      else {
+        await promisePool.execute(sqlDeleteAccountLimits, [account_sid]);
+      }
+    }
+    else {
+      if (category) {
+        await promisePool.execute(sqlDeleteSPLimitsByCategory, [service_provider_sid, category]);
+      }
+      else {
+        await promisePool.execute(sqlDeleteSPLimits, [service_provider_sid]);
+      }
+    }
+    res.status(204).end();
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/login.js

@@ -1,74 +1,109 @@
 const router = require('express').Router();
-const crypto = require('crypto');
-const {getMysqlConnection} = require('../../db');
-
+const jwt = require('jsonwebtoken');
+const {verifyPassword} = require('../../utils/password-utils');
+const {promisePool} = require('../../db');
+const {cacheClient} = require('../../helpers');
+const Account = require('../../models/account');
+const ServiceProvider = require('../../models/service-provider');
+const sysError = require('../error');
+const retrievePemissionsSql = `
+SELECT p.name 
+FROM permissions p, user_permissions up 
+WHERE up.permission_sid = p.permission_sid 
+AND up.user_sid = ? 
+`;
 const retrieveSql = 'SELECT * from users where name = ?';
-const tokenSql = 'SELECT token from api_keys where account_sid IS NULL AND service_provider_sid IS NULL';
 
-const sha512 = function(password, salt) {
-  const hash = crypto.createHmac('sha512', salt); /** Hashing algorithm sha512 */
-  hash.update(password);
-  var value = hash.digest('hex');
-  return {
-    salt:salt,
-    passwordHash:value
-  };
-};
-
-router.post('/', (req, res) => {
-  const logger = req.app.locals.logger;
+router.post('/', async(req, res) => {
+  const {logger, incrKey, retrieveKey} = req.app.locals;
   const {username, password} = req.body;
   if (!username || !password) {
     logger.info('Bad POST to /login is missing username or password');
     return res.sendStatus(400);
   }
 
-  getMysqlConnection((err, conn) => {
-    if (err) {
-      logger.error({err}, 'Error getting db connection');
-      return res.sendStatus(500);
+  try {
+    const [r] = await promisePool.query(retrieveSql, username);
+    if (r.length === 0) {
+      logger.info(`Failed login attempt for user ${username}`);
+      return res.sendStatus(403);
     }
-    conn.query(retrieveSql, [username], (err, results) => {
-      conn.release();
-      if (err) {
-        logger.error({err}, 'Error getting db connection');
-        return res.sendStatus(500);
-      }
-      if (0 === results.length) {
-        logger.info(`Failed login attempt for user ${username}`);
-        return res.sendStatus(403);
+    logger.info({r}, 'successfully retrieved user account');
+
+    const maxLoginAttempts = process.env.LOGIN_ATTEMPTS_MAX_RETRIES || 6;
+    const loginAttempsBlocked = await retrieveKey(`login:${r[0].user_sid}`) >= maxLoginAttempts;
+
+    if (loginAttempsBlocked) {
+      logger.info(`User ${r[0].user_sid} was blocked due to excessive login attempts with incorrect credentials.`);
+      return res.status(403)
+        .json({error: 'Maximum login attempts reached. Please try again later or reset your password.'});
+    }
+
+    const isCorrect = await verifyPassword(r[0].hashed_password, password);
+    if (!isCorrect) {
+      const attempTime = process.env.LOGIN_ATTEMPTS_TIME || 1800;
+      const newAttempt = await incrKey(`login:${r[0].user_sid}`, attempTime)
+        .catch((err) => logger.error({err}, 'Error adding logging attempt to redis'));
+      if (newAttempt >= maxLoginAttempts) {
+        logger.info(`User ${r[0].user_sid} is now blocked due to excessive login attempts with incorrect credentials.`);
+        return res.status(403)
+          .json({error: `Maximum login attempts reached. Please try again in ${attempTime} seconds.`});
       }
+      return res.sendStatus(403);
+    }
+    const force_change = !!r[0].force_change;
 
-      logger.info({results}, 'successfully retrieved account');
-      const salt = results[0].salt;
-      const trueHash = results[0].hashed_password;
-      const forceChange = results[0].force_change;
+    const [p] = await promisePool.query(retrievePemissionsSql, r[0].user_sid);
+    const permissions = p.map((x) => x.name);
+    const obj = {user_sid: r[0].user_sid, scope: 'admin', force_change, permissions};
+    if (r[0].service_provider_sid && r[0].account_sid) {
+      const account = await Account.retrieve(r[0].account_sid);
+      const service_provider = await ServiceProvider.retrieve(r[0].service_provider_sid);
+      obj.scope = 'account';
+      obj.service_provider_sid = r[0].service_provider_sid;
+      obj.account_sid = r[0].account_sid;
+      obj.account_name = account[0].name;
+      obj.service_provider_name = service_provider[0].name;
+    }
+    else if (r[0].service_provider_sid) {
+      const service_provider = await ServiceProvider.retrieve(r[0].service_provider_sid);
+      obj.scope = 'service_provider';
+      obj.service_provider_sid = r[0].service_provider_sid;
+      obj.service_provider_name = service_provider[0].name;
+    }
+    const payload = {
+      scope: obj.scope,
+      permissions,
+      ...(obj.service_provider_sid && {
+        service_provider_sid: obj.service_provider_sid,
+        service_provider_name: obj.service_provider_name
+      }),
+      ...(obj.account_sid && {
+        account_sid: obj.account_sid,
+        account_name: obj.account_name,
+        service_provider_name: obj.service_provider_name
+      }),
+      user_sid: obj.user_sid
+    };
 
-      const {passwordHash} = sha512(password, salt);
-      if (trueHash !== passwordHash) return res.sendStatus(403);
+    const expiresIn = parseInt(process.env.JWT_EXPIRES_IN || 60) * 60;
+    const token = jwt.sign(
+      payload,
+      process.env.JWT_SECRET,
+      { expiresIn }
+    );
+    res.json({token, ...obj});
 
-      if (forceChange) return res.json({user_sid: results[0].user_sid, force_change: true});
-
-      getMysqlConnection((err, conn) => {
-        if (err) {
-          logger.error({err}, 'Error getting db connection');
-          return res.sendStatus(500);
-        }
-        conn.query(tokenSql, (err, tokenResults) => {
-          conn.release();
-          if (err) {
-            logger.error({err}, 'Error getting db connection');
-            return res.sendStatus(500);
-          }
-          if (0 === tokenResults.length) {
-            logger.error('Database has no admin token provisioned...run reset_admin_password');
-            return res.sendStatus(500);
-          }
-          res.json({user_sid: results[0].user_sid, token: tokenResults[0].token});
-        });
-      });
+    /* Store jwt based on user_id after successful login */
+    await cacheClient.set({
+      redisKey: cacheClient.generateRedisKey('jwt', obj.user_sid, 'v2'),
+      value: token,
+      time: expiresIn,
     });
-  });
+
+  } catch (err) {
+    sysError(logger, res, err);
+  }
 });
 
 

lib/routes/api/logout.js

@@ -0,0 +1,22 @@
+const router = require('express').Router();
+const debug = require('debug')('jambonz:api-server');
+const {cacheClient} = require('../../helpers');
+const sysError = require('../error');
+
+router.post('/', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {user_sid} = req.user;
+
+  debug(`logout user and invalidate jwt token for user: ${user_sid}`);
+
+  try {
+    const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+    await cacheClient.delete(redisKey);
+
+    res.sendStatus(204);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/password-settings.js

@@ -0,0 +1,45 @@
+const router = require('express').Router();
+const sysError = require('../error');
+const PasswordSettings = require('../../models/password-settings');
+const { DbErrorBadRequest } = require('../../utils/errors');
+
+const validate = (obj) => {
+  if (obj.min_password_length && (
+    obj.min_password_length < 8 ||
+    obj.min_password_length > 20
+  )) {
+    throw new DbErrorBadRequest('invalid min_password_length property: should be between 8-20');
+  }
+};
+
+router.post('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    if (!req.user.hasAdminAuth) {
+      return res.sendStatus(403);
+    }
+    validate(req.body);
+    const [existing] = (await PasswordSettings.retrieve() || []);
+    if (existing) {
+      await PasswordSettings.update(req.body);
+    } else {
+      await PasswordSettings.make(req.body);
+    }
+    res.status(201).json({});
+  }
+  catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const [results] = (await PasswordSettings.retrieve() || []);
+    return res.status(200).json(results || {min_password_length: 8});
+  }
+  catch (err) {
+    sysError(logger, res, err);
+  }
+});
+module.exports = router;

lib/routes/api/phone-numbers.js

@@ -1,46 +1,86 @@
 const router = require('express').Router();
-const {DbErrorUnprocessableRequest, DbErrorBadRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, DbErrorForbidden} = require('../../utils/errors');
 const PhoneNumber = require('../../models/phone-number');
 const VoipCarrier = require('../../models/voip-carrier');
+const Account = require('../../models/account');
 const decorate = require('./decorate');
-const validateNumber = require('../../utils/phone-number-syntax');
+const {promisePool} = require('../../db');
+const {e164} = require('../../utils/phone-number-utils');
 const preconditions = {
   'add': validateAdd,
   'delete': checkInUse,
   'update': validateUpdate
 };
+const sysError = require('../error');
+const { parsePhoneNumberSid } = require('./utils');
+
 
 /* check for required fields when adding */
 async function validateAdd(req) {
   try {
-    if (!req.body.voip_carrier_sid) throw new DbErrorBadRequest('voip_carrier_sid is required');
+    /* account level user can only act on carriers associated to his/her account */
+    if (req.user.hasAccountAuth) {
+      req.body.account_sid = req.user.account_sid;
+    }
+
+    if (req.user.hasServiceProviderAuth) {
+      req.body.service_provider_sid = req.user.service_provider_sid;
+    }
+
     if (!req.body.number) throw new DbErrorBadRequest('number is required');
-    validateNumber(req.body.number);
+    const formattedNumber = e164(req.body.number);
+    req.body.number = formattedNumber;
   } catch (err) {
     throw new DbErrorBadRequest(err.message);
   }
 
   /* check that voip carrier exists */
-  const result = await VoipCarrier.retrieve(req.body.voip_carrier_sid);
-  if (!result || result.length === 0) {
-    throw new DbErrorBadRequest(`voip_carrier not found for sid ${req.body.voip_carrier_sid}`);
+  if (req.body.voip_carrier_sid) {
+    const result = await VoipCarrier.retrieve(req.body.voip_carrier_sid);
+    if (!result || result.length === 0) {
+      throw new DbErrorBadRequest(`voip_carrier not found for sid ${req.body.voip_carrier_sid}`);
+    }
   }
 }
 
 /* can not delete a phone number if it in use */
 async function checkInUse(req, sid) {
   const phoneNumber = await PhoneNumber.retrieve(sid);
-  if (phoneNumber.account_sid) {
-    throw new DbErrorUnprocessableRequest('cannot delete phone number that is assigned to an account');
+  if (req.user.hasAccountAuth) {
+    if (phoneNumber && phoneNumber.length && phoneNumber[0].account_sid !== req.user.account_sid) {
+      throw new DbErrorForbidden('insufficient privileges');
+    }
+  }
+  if (!req.user.hasAccountAuth && phoneNumber.account_sid) {
+    throw new DbErrorForbidden('insufficient privileges');
   }
 }
 
 /* can not change number or voip carrier */
 async function validateUpdate(req, sid) {
-  //const result = await PhoneNumber.retrieve(sid);
   if (req.body.voip_carrier_sid) throw new DbErrorBadRequest('voip_carrier_sid may not be modified');
   if (req.body.number) throw new DbErrorBadRequest('number may not be modified');
 
+  const phoneNumber = await PhoneNumber.retrieve(sid);
+  if (req.user.hasAccountAuth) {
+    if (phoneNumber && phoneNumber.length && phoneNumber[0].account_sid !== req.user.account_sid) {
+      throw new DbErrorForbidden('insufficient privileges');
+    }
+  }
+  if (req.user.hasServiceProviderAuth) {
+    let service_provider_sid;
+
+    if (!phoneNumber[0].service_provider_sid) {
+      const [r] = await Account.retrieve(phoneNumber[0].account_sid);
+      service_provider_sid = r.service_provider_sid;
+    } else {
+      service_provider_sid = phoneNumber[0].service_provider_sid;
+    }
+
+    if (phoneNumber && phoneNumber.length && service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorForbidden('insufficient privileges');
+    }
+  }
   // TODO: if we are assigning to an account, verify it exists
 
   // TODO: if we are assigning to an application, verify it is associated to the same account
@@ -48,6 +88,46 @@ async function validateUpdate(req, sid) {
   // TODO: if we are removing from an account, verify we are also removing from application.
 }
 
-decorate(router, PhoneNumber, ['*'], preconditions);
+decorate(router, PhoneNumber, ['add', 'update', 'delete'], preconditions);
+
+/* list */
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = req.user.hasServiceProviderAuth ?
+      await PhoneNumber.retrieveAllForSP(req.user.service_provider_sid) :
+      await PhoneNumber.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null);
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+/* retrieve */
+router.get('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const sid = parsePhoneNumberSid(req);
+    const account_sid = req.user.hasAccountAuth ? req.user.account_sid : null;
+    const results = await PhoneNumber.retrieve(sid, account_sid);
+    if (results.length === 0) return res.status(404).end();
+
+    if (req.user.hasServiceProviderAuth && results.length === 1) {
+      const account_sid = results[0].account_sid;
+      const [r] = await promisePool.execute(
+        'SELECT service_provider_sid from accounts WHERE account_sid = ?', [account_sid]);
+      if (r.length === 1 && r[0].service_provider_sid !== req.user.service_provider_sid) {
+        throw new DbErrorBadRequest('insufficient privileges');
+      }
+    }
+    if (req.user.hasAccountAuth && results.length > 1) {
+      return res.status(200).json(results.filter((r) => r.phone_number_sid === sid)[0]);
+    }
+    return res.status(200).json(results[0]);
+  }
+  catch (err) {
+    sysError(logger, res, err);
+  }
+});
 
 module.exports = router;

lib/routes/api/predefined-carriers.js

@@ -0,0 +1,7 @@
+const router = require('express').Router();
+const PredefinedCarrier = require('../../models/predefined-carrier');
+const decorate = require('./decorate');
+
+decorate(router, PredefinedCarrier, ['list']);
+
+module.exports = router;

lib/routes/api/prices.js

@@ -0,0 +1,108 @@
+const router = require('express').Router();
+const Product = require('../../models/product');
+const {promisePool} = require('../../db');
+const sysError = require('../error');
+const sqlRetrieveSpecialOffers = `SELECT *
+FROM account_offers offer
+LEFT JOIN products AS product ON product.product_sid = offer.product_sid 
+WHERE offer.account_sid = ?`;
+
+const combineProductAndPrice = (localProducts, product, prices) => {
+  const lp = localProducts.find((lp) => lp.category === product.metadata.jambonz_category);
+  return {
+    product_sid: lp.product_sid,
+    name: lp.name,
+    category: lp.category,
+    stripe_product_id: product.id,
+    description: product.description,
+    unit_label: product.unit_label,
+    prices: prices.map((price) => {
+      return {
+        stripe_price_id: price.id,
+        billing_scheme: price.billing_scheme,
+        currency: price.currency,
+        recurring: price.recurring,
+        tiers_mode: price.tiers_mode,
+        tiers: price.tiers,
+        type: price.type,
+        unit_amount: price.unit_amount,
+        unit_amount_decimal: price.unit_amount_decimal
+      };
+    })
+  };
+};
+
+
+/* list */
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const {account_sid} = req.user || {};
+  const {listProducts, retrieveProduct, retrievePricesForProduct} = require('../../utils/stripe-utils');
+  try {
+    const localProducts = await Product.retrieveAll();
+
+    /**
+     * If this request is for a specific account (we have an account_sid)
+     * then check to see if we have any special offers for this account
+     */
+    const selectedProducts = [];
+    if (account_sid) {
+      const [r] = await promisePool.query({sql: sqlRetrieveSpecialOffers, nestTables: true}, account_sid);
+      logger.debug({r}, `retrieved special offer ids for account_sid ${account_sid}`);
+
+      if (r.length > 0) {
+        /* retrieve all the offers for this account */
+        const products = await Promise.all(r.map((row) => retrieveProduct(logger, row.offer.stripe_product_id)));
+        logger.debug({products}, `retrieved special offer products for account_sid ${account_sid}`);
+        const prices = await Promise.all(products.map((prod) => retrievePricesForProduct(logger, prod.id)));
+        logger.debug({prices}, `retrieved special offer prices for account_sid ${account_sid}`);
+
+        for (let i = 0; i < products.length; i++) {
+          selectedProducts.push(combineProductAndPrice(localProducts, products[i], prices[i].data));
+        }
+      }
+    }
+
+    /**
+     * we must return at least pricing for sessions and devices, so find and use
+     * the general pricing if no account-specific product was specified for these
+     */
+    const haveSessionPricing = selectedProducts.find((prod) => prod.category === 'voice_call_session');
+    const haveDevicePricing = selectedProducts.find((prod) => prod.category === 'device');
+
+    if (haveSessionPricing && haveDevicePricing) {
+      logger.debug({selectedProducts}, 'found account level offers for sessions and devices');
+      return res.status(200).json(selectedProducts);
+    }
+
+    /* need to get default pricing */
+    const allProducts = await listProducts(logger);
+    logger.debug({allProducts}, 'retrieved all products');
+    const defaultProducts = allProducts.data.filter((prod) =>
+      ['voice_call_session', 'device'].includes(prod.metadata.jambonz_category) &&
+      'general' === prod.metadata.availability);
+    logger.debug({defaultProducts}, 'default products');
+
+    if (!haveSessionPricing) {
+      const product = defaultProducts.find((prod) => 'voice_call_session' === prod.metadata.jambonz_category);
+      if (product) {
+        logger.debug(`retrieving prices for product id ${product.id}`);
+        const prices = await retrievePricesForProduct(logger, product.id);
+        selectedProducts.push(combineProductAndPrice(localProducts, product, prices.data));
+      }
+    }
+    if (!haveDevicePricing) {
+      const product = defaultProducts.find((prod) => 'device' === prod.metadata.jambonz_category);
+      if (product) {
+        logger.debug(`retrieving prices for product id ${product.id}`);
+        const prices = await retrievePricesForProduct(logger, product.id);
+        selectedProducts.push(combineProductAndPrice(localProducts, product, prices.data));
+      }
+    }
+    res.status(200).json(selectedProducts);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/products.js

@@ -0,0 +1,78 @@
+const assert = require('assert');
+const router = require('express').Router();
+const Product = require('../../models/product');
+const {listProducts, listPrices} = require('../../utils/stripe-utils');
+const sysError = require('./error');
+
+/* list */
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const [stripeProducts, localProducts] = await Promise.all([listProducts(), Product.retrieveAll()]);
+    console.log(stripeProducts);
+    console.log(localProducts);
+    const arr = localProducts.map((p) => {
+      const stripe = stripeProducts.data
+        .find((s) => s.metadata.jambonz_category === p.category);
+      assert.ok(stripe, `No stripe product found for category ${p.category}`);
+      Object.assign(p, {
+        stripe_product_id: stripe.id,
+        statement_descriptor: stripe.statement_descriptor,
+        description: stripe.description,
+        unit_label: stripe.unit_label
+      });
+      return p;
+    });
+    res.status(200).json(arr);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+/* get */
+router.get('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const [allPrices, results] = await Promise.all([listPrices(), Product.retrieve(req.params.sid)]);
+    if (results.length === 0) return res.sendStatus(404);
+    const product = results[0];
+    const prices = allPrices.data
+      .filter((p) => p.active && p.product.active)
+      .filter((p) => p.product.metadata.jambonz_category === product.category);
+    assert(prices.length > 0, `No pricing data found for product ${req.params.sid}`);
+    const stripe = prices[0].product;
+    Object.assign(product, {
+      stripe_product_id: stripe.id,
+      statement_descriptor: stripe.statement_descriptor,
+      description: stripe.description,
+      unit_label: stripe.unit_label
+    });
+
+    // get pricing
+    Object.assign(product, {
+      pricing: {
+        billing_scheme: stripe.billing_scheme,
+        type: prices[0].type
+      }
+    });
+
+    product.pricing.fees = prices.map((price) => {
+      const obj = {
+        stripe_price_id: price.id,
+        currency: price.currency,
+        unit_amount: price.unit_amount,
+        unit_amount_decimal: price.unit_amount_decimal
+      };
+      if (price.tiers) {
+        obj.tiers = price.tiers;
+        obj.tiers_mode = price.tiers_mode;
+      }
+      return obj;
+    });
+    res.status(200).json(product);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/recent-calls.js

@@ -0,0 +1,205 @@
+const router = require('express').Router();
+const sysError = require('../error');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const {getHomerApiKey, getHomerSipTrace, getHomerPcap} = require('../../utils/homer-utils');
+const {getJaegerTrace} = require('../../utils/jaeger-utils');
+const Account = require('../../models/account');
+const {
+  getS3Object,
+  getGoogleStorageObject,
+  getAzureStorageObject,
+  deleteS3Object,
+  deleteGoogleStorageObject,
+  deleteAzureStorageObject
+} = require('../../utils/storage-utils');
+
+const parseAccountSid = (url) => {
+  const arr = /Accounts\/([^\/]*)/.exec(url);
+  if (arr) return arr[1];
+};
+
+const parseServiceProviderSid = (url) => {
+  const arr = /ServiceProviders\/([^\/]*)/.exec(url);
+  if (arr) return arr[1];
+};
+
+router.get('/', async(req, res) => {
+  const {logger, queryCdrs, queryCdrsSP} = req.app.locals;
+  try {
+    logger.debug({opts: req.query}, 'GET /RecentCalls');
+    const account_sid = parseAccountSid(req.originalUrl);
+    const service_provider_sid = account_sid ? null : parseServiceProviderSid(req.originalUrl);
+    const {page, count, trunk, direction, days, answered, start, end, filter} = req.query || {};
+    if (!page || page < 1) throw new DbErrorBadRequest('missing or invalid "page" query arg');
+    if (!count || count < 25 || count > 500) throw new DbErrorBadRequest('missing or invalid "count" query arg');
+
+    if (account_sid) {
+      const data = await queryCdrs({
+        account_sid,
+        page,
+        page_size: count,
+        trunk,
+        direction,
+        days,
+        answered,
+        start: days ? undefined : start,
+        end: days ? undefined : end,
+        filter
+      });
+      res.status(200).json(data);
+    }
+    else {
+      const data = await queryCdrsSP({
+        service_provider_sid,
+        page,
+        page_size: count,
+        trunk,
+        direction,
+        days,
+        answered,
+        start: days ? undefined : start,
+        end: days ? undefined : end,
+        filter
+      });
+      res.status(200).json(data);
+    }
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:call_id', async(req, res) => {
+  const {logger} = req.app.locals;
+  try {
+    const token = await getHomerApiKey(logger);
+    if (!token) return res.sendStatus(400, {msg: 'Failed to get Homer API token; check server config'});
+    const obj = await getHomerSipTrace(logger, token, req.params.call_id);
+    if (!obj) {
+      logger.info(`/RecentCalls: unable to get sip traces from Homer for ${req.params.call_id}`);
+      return res.sendStatus(404);
+    }
+    res.status(200).json(obj);
+  } catch (err) {
+    logger.error({err}, '/RecentCalls error retrieving sip traces from homer');
+    res.sendStatus(err.statusCode || 500);
+  }
+});
+
+router.get('/:call_id/:method/pcap', async(req, res) => {
+  const {logger} = req.app.locals;
+  try {
+    const token = await getHomerApiKey(logger);
+    if (!token) return res.sendStatus(400, {msg: 'getHomerApiKey: Failed to get Homer API token; check server config'});
+    const stream = await getHomerPcap(logger, token, [req.params.call_id], req.params.method);
+    if (!stream) {
+      logger.info(`getHomerApiKey: unable to get sip traces from Homer for ${req.params.call_id}`);
+      return res.sendStatus(404);
+    }
+    res.set({
+      'Content-Type': 'application/octet-stream',
+      'Content-Disposition': `attachment; filename=callid-${req.params.call_id}.pcap`
+    });
+    stream.pipe(res);
+  } catch (err) {
+    logger.error({err}, 'getHomerApiKey error retrieving sip traces from homer');
+    res.sendStatus(err.statusCode || 500);
+  }
+});
+
+router.get('/trace/:trace_id', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {trace_id} = req.params;
+  try {
+    const obj = await getJaegerTrace(logger, trace_id);
+    if (!obj) {
+      logger.info(`/RecentCalls: unable to get spans from jaeger for ${trace_id}`);
+      return res.sendStatus(404);
+    }
+    res.status(200).json(obj.result);
+  } catch (err) {
+    logger.error({err}, `/RecentCalls error retrieving jaeger trace ${trace_id}`);
+    res.sendStatus(500);
+  }
+});
+
+router.get('/:call_sid/record/:year/:month/:day/:format', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {call_sid, year, month, day, format} = req.params;
+
+  try {
+    const account_sid = parseAccountSid(req.originalUrl);
+    const r = await Account.retrieve(account_sid);
+    if (r.length === 0 || !r[0].bucket_credential) return res.sendStatus(404);
+    const {bucket_credential} = r[0];
+    const getOptions  = {
+      ...bucket_credential,
+      key: `${year}/${month}/${day}/${call_sid}.${format || 'mp3'}`
+    };
+    let stream;
+    switch (bucket_credential.vendor) {
+      case 'aws_s3':
+      case 's3_compatible':
+        stream = await getS3Object(logger, getOptions);
+        break;
+      case 'google':
+        stream = await getGoogleStorageObject(logger, getOptions);
+        break;
+      case 'azure':
+        stream = await getAzureStorageObject(logger, getOptions);
+        break;
+      default:
+        logger.error(`There is no handler for fetching record from ${bucket_credential.vendor}`);
+        return res.sendStatus(500);
+    }
+    res.set({
+      'Content-Type': `audio/${format || 'mp3'}`
+    });
+    if (stream) {
+      stream.pipe(res);
+    } else {
+      return res.sendStatus(404);
+    }
+  }  catch (err) {
+    logger.error({err}, ` error retrieving recording ${call_sid}`);
+    res.sendStatus(404);
+  }
+});
+
+router.delete('/:call_sid/record/:year/:month/:day/:format', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {call_sid, year, month, day, format} = req.params;
+
+  try {
+    const account_sid = parseAccountSid(req.originalUrl);
+    const r = await Account.retrieve(account_sid);
+    if (r.length === 0 || !r[0].bucket_credential) return res.sendStatus(404);
+    const {bucket_credential} = r[0];
+
+    const deleteOptions  = {
+      ...bucket_credential,
+      key: `${year}/${month}/${day}/${call_sid}.${format || 'mp3'}`
+    };
+
+    switch (bucket_credential.vendor) {
+      case 'aws_s3':
+      case 's3_compatible':
+        await deleteS3Object(logger, deleteOptions);
+        break;
+      case 'google':
+        await deleteGoogleStorageObject(logger, deleteOptions);
+        break;
+      case 'azure':
+        await deleteAzureStorageObject(logger, deleteOptions);
+        break;
+      default:
+        logger.error(`There is no handler for deleting record from ${bucket_credential.vendor}`);
+        return res.sendStatus(500);
+    }
+    res.sendStatus(204);
+  }  catch (err) {
+    logger.error({err}, ` error deleting recording ${call_sid}`);
+    res.sendStatus(404);
+  }
+});
+
+module.exports = router;

lib/routes/api/register.js

@@ -0,0 +1,391 @@
+const router = require('express').Router();
+const debug = require('debug')('jambonz:api-server');
+const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {promisePool} = require('../../db');
+const {doGithubAuth, doGoogleAuth, doLocalAuth} = require('../../utils/oauth-utils');
+const {validateEmail} = require('../../utils/email-utils');
+const {cacheClient} = require('../../helpers');
+const { v4: uuid } = require('uuid');
+const short = require('short-uuid');
+const translator = short();
+const jwt = require('jsonwebtoken');
+const {setupFreeTrial, createTestCdrs, createTestAlerts} = require('./utils');
+const {generateHashedPassword} = require('../../utils/password-utils');
+const sysError = require('../error');
+const insertUserSql = `INSERT into users 
+(user_sid, account_sid, name, email, provider, provider_userid, email_validated) 
+values (?, ?, ?, ?, ?, ?, 1)`;
+const insertUserLocalSql = `INSERT into users 
+(user_sid, account_sid, name, email, email_activation_code, email_validated, provider,
+hashed_password, service_provider_sid) 
+values (?, ?, ?, ?, ?, 0, 'local', ?, ?)`;
+const insertAccountSql = `INSERT into accounts 
+(account_sid, service_provider_sid, name, is_active, webhook_secret, trial_end_date) 
+values (?, ?, ?, ?, ?, CURDATE() + INTERVAL 21 DAY)`;
+const insertWebookSql = `INSERT INTO webhooks (webhook_sid, url, method) 
+VALUES (?, ?, ?)`;
+const insertApplicationSql = `INSERT INTO applications 
+(application_sid, account_sid, name, call_hook_sid, call_status_hook_sid, 
+speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice, 
+speech_recognizer_vendor, speech_recognizer_language)
+VALUES (?,?,?,?,?,?,?,?,?,?)`;
+const queryRootDomainSql = `SELECT root_domain 
+FROM service_providers 
+WHERE service_providers.service_provider_sid = ?`;
+const insertSignupHistorySql = `INSERT into signup_history
+(email, name) 
+values (?, ?)`;
+
+const addLocalUser = async(logger, user_sid, account_sid,
+  name, email, email_activation_code, passwordHash, service_provider_sid) => {
+  const [r] = await promisePool.execute(insertUserLocalSql,
+    [
+      user_sid,
+      account_sid,
+      name,
+      email,
+      email_activation_code,
+      passwordHash,
+      service_provider_sid
+    ]);
+  debug({r}, 'Result from adding user');
+};
+const addOauthUser = async(logger, user_sid, account_sid,
+  name, email, provider, provider_userid) => {
+  const [r] = await promisePool.execute(insertUserSql,
+    [
+      user_sid,
+      account_sid,
+      name,
+      email,
+      provider,
+      provider_userid
+    ]);
+  logger.debug({r}, 'Result from adding user');
+};
+
+const validateRequest = async(req, user_sid) => {
+  const payload = req.body || {};
+
+  /* check required properties are there */
+  ['provider', 'service_provider_sid'].forEach((prop) => {
+    if (!payload[prop]) throw new DbErrorBadRequest(`missing ${prop}`);
+  });
+
+  /* valid service provider? */
+  const [rows] = await promisePool.query('SELECT * from service_providers WHERE service_provider_sid = ?',
+    payload.service_provider_sid);
+  if (0 === rows.length) throw new DbErrorUnprocessableRequest('invalid service_provider_sid');
+
+  /* valid provider? */
+  if (!['local', 'github', 'google', 'twitter'].includes(payload.provider)) {
+    throw new DbErrorUnprocessableRequest(`invalid provider: ${payload.provider}`);
+  }
+
+  /* if local provider then email/password */
+  if ('local' === payload.provider) {
+    if (!payload.email || !payload.password) throw new DbErrorBadRequest('missing email or password');
+
+    /* valid email? */
+    if (!validateEmail(payload.email)) throw new DbErrorBadRequest('invalid email');
+
+    /* valid password? */
+    if (payload.password.length < 6) throw new DbErrorBadRequest('password must be at least 6 characters');
+
+    /* is this email available? */
+    if (user_sid) {
+      const [rows] = await promisePool.query('SELECT * from users WHERE email = ? AND user_sid <> ?',
+        [payload.email, user_sid]);
+      if (rows.length > 0) throw new DbErrorUnprocessableRequest('account already exists for this email');
+    }
+    else {
+      const [rows] = await promisePool.query('SELECT * from users WHERE email = ?', payload.email);
+      if (rows.length > 0) throw new DbErrorUnprocessableRequest('account already exists for this email');
+    }
+
+    /* verify that we have a code to email them */
+    if (!payload.email_activation_code) throw new DbErrorBadRequest('email activation code required');
+  }
+  else {
+    ['oauth2_code', 'oauth2_state', 'oauth2_client_id', 'oauth2_redirect_uri'].forEach((prop) => {
+      if (!payload[prop]) throw new DbErrorBadRequest(`missing ${prop} for provider ${payload.provider}`);
+    });
+  }
+};
+
+const parseAuthorizationToken = (logger, req) => {
+  const notfound = {};
+  const authHeader = req.get('Authorization');
+  if (!authHeader) return Promise.resolve(notfound);
+
+  return new Promise((resolve) => {
+    const arr = /^Bearer (.*)$/.exec(req.get('Authorization'));
+    if (!arr) return resolve(notfound);
+    jwt.verify(arr[1], process.env.JWT_SECRET, async(err, decoded) => {
+      if (err) return resolve(notfound);
+      logger.debug({jwt: decoded}, 'register - create new user for existing account');
+      resolve(decoded);
+    });
+  });
+};
+
+/**
+ * called to create a new user and account
+ * or new user with existing account, in case of "change auth mechanism"
+ */
+router.post('/', async(req, res) => {
+  const {logger, writeCdrs, writeAlerts, AlertType} = req.app.locals;
+  const userProfile = {};
+
+  try {
+    const {user_sid, account_sid} = await parseAuthorizationToken(logger, req);
+    await validateRequest(req, user_sid);
+
+    logger.debug({payload: req.body}, 'POST /register');
+
+    if (req.body.provider === 'github') {
+      const user = await doGithubAuth(logger, req.body);
+      logger.info({user}, 'retrieved user details from github');
+      Object.assign(userProfile, {
+        name: user.email,
+        email: user.email,
+        email_validated: user.email_validated,
+        avatar_url: user.avatar_url,
+        provider: 'github',
+        provider_userid: user.login
+      });
+    }
+    else if (req.body.provider === 'google') {
+      const user = await doGoogleAuth(logger, req.body);
+      logger.info({user}, 'retrieved user details from google');
+      Object.assign(userProfile, {
+        name: user.email || user.email,
+        email: user.email,
+        email_validated: user.verified_email,
+        picture: user.picture,
+        provider: 'google',
+        provider_userid: user.id
+      });
+    }
+    else if (req.body.provider === 'local') {
+      const user = await doLocalAuth(logger, req.body);
+      logger.info({user}, 'retrieved user details for local provider');
+      debug({user}, 'retrieved user details for local provider');
+      Object.assign(userProfile, {
+        name: user.email,
+        email: user.email,
+        provider: 'local',
+        email_activation_code: user.email_activation_code
+      });
+    }
+
+    if (req.body.provider !== 'local') {
+      /* when using oauth2, check to see if user already exists */
+      const [users] = await promisePool.query(
+        'SELECT * from users WHERE provider = ? AND provider_userid = ?',
+        [userProfile.provider, userProfile.provider_userid]);
+      logger.debug({users}, `Result from retrieving user for ${userProfile.provider}:${userProfile.provider_userid}`);
+      if (1 === users.length) {
+
+        /* if changing existing account to oauth, no other user with that provider/userid must exist */
+        if (user_sid) {
+          throw new DbErrorUnprocessableRequest('account already exists for this oauth user/provider');
+        }
+        Object.assign(userProfile, {
+          user_sid: users[0].user_sid,
+          account_sid: users[0].account_sid,
+          name: users[0].name,
+          email: users[0].email,
+          phone: users[0].phone,
+          pristine: false,
+          email_validated: users[0].email_validated ? true : false,
+          phone_validated: users[0].phone_validated ? true : false,
+          scope: users[0].scope
+        });
+
+        const [accounts] =  await promisePool.query('SELECT * from accounts WHERE account_sid = ?',
+          userProfile.account_sid);
+        if (accounts.length === 0) throw new DbErrorUnprocessableRequest('user exists with no associated account');
+        Object.assign(userProfile, {
+          is_active: accounts[0].is_active == 1,
+          tutorial_completion: accounts[0].tutorial_completion
+        });
+      }
+      else {
+        /* you can not register from the sign-in page */
+        if (req.body.locationBeforeAuth === '/sign-in') {
+          logger.debug('redirecting user to /register so they accept Ts & Cs');
+          return res.status(404).json({msg: 'registering a new account not allowed from the sign-in page'});
+        }
+        /* new user, but check if we already have an account with that email */
+        let sql = 'SELECT * from users WHERE email = ?';
+        const args = [userProfile.email];
+        if (user_sid) {
+          sql += ' AND user_sid <> ?';
+          args.push(user_sid);
+        }
+        logger.debug(`sql is ${sql}`);
+        const [accounts] = await promisePool.execute(sql, args);
+        if (accounts.length > 0) {
+          throw new DbErrorBadRequest(`user already exists with email ${userProfile.email}`);
+        }
+      }
+    }
+    if (userProfile.pristine !== false && !user_sid) {
+      /* add a new user and account */
+      /* get root domain */
+      const [sp] = await promisePool.query(queryRootDomainSql, req.body.service_provider_sid);
+      if (0 === sp.length) throw new Error(`service_provider not found for sid ${req.body.service_provider_sid}`);
+      if (!sp[0].root_domain) {
+        throw new Error(`root_domain missing for service provider ${req.body.service_provider_sid}`);
+      }
+
+      userProfile.root_domain = sp[0].root_domain;
+      userProfile.account_sid = uuid();
+      userProfile.user_sid = uuid();
+
+      const [r1] = await promisePool.execute(insertAccountSql,
+        [
+          userProfile.account_sid,
+          req.body.service_provider_sid,
+          userProfile.name || userProfile.email,
+          req.body.provider !== 'local',
+          `wh_secret_${translator.generate()}`
+        ]);
+      logger.debug({r1}, 'Result from adding account');
+
+      /* add to signup history */
+      let isReturningUser = false;
+      try {
+        await promisePool.execute(insertSignupHistorySql,
+          [userProfile.email, userProfile.name || userProfile.email]);
+      } catch (err) {
+        if (err.code === 'ER_DUP_ENTRY') {
+          logger.info(`register: user is signing up for a second trial: ${userProfile.email}`);
+          isReturningUser = true;
+        }
+      }
+
+      /* write sample cdrs and alerts in test environment */
+      if ('test' === process.env.NODE_ENV) {
+        await createTestCdrs(writeCdrs, userProfile.account_sid);
+        await createTestAlerts(writeAlerts, AlertType, userProfile.account_sid);
+        logger.debug('added test data for cdrs and alerts');
+      }
+      /* assign starter set of products */
+      await setupFreeTrial(logger, userProfile.account_sid, isReturningUser);
+
+      /* add a user for the account */
+      if (req.body.provider === 'local') {
+        /* hash password */
+        debug(`salting password: ${req.body.password}`);
+        const passwordHash = await generateHashedPassword(req.body.password);
+        debug(`hashed password: ${passwordHash}`);
+        await addLocalUser(logger, userProfile.user_sid, userProfile.account_sid,
+          userProfile.name, userProfile.email, userProfile.email_activation_code,
+          passwordHash, req.body.service_provider_sid);
+        debug('added local user');
+      }
+      else {
+        await addOauthUser(logger, userProfile.user_sid, userProfile.account_sid,
+          userProfile.name, userProfile.email, userProfile.provider,
+          userProfile.provider_userid);
+      }
+
+      /* add hello-world and dial-time as starter applications */
+      const callStatusSid = uuid();
+      const helloWordSid = uuid();
+      const dialTimeSid = uuid();
+      const echoSid = uuid();
+
+      /* 4 webhooks */
+      await promisePool.execute(insertWebookSql,
+        [callStatusSid, 'https://public-apps.jambonz.cloud/call-status', 'POST']);
+      await promisePool.execute(insertWebookSql,
+        [helloWordSid, 'https://public-apps.jambonz.cloud/hello-world', 'POST']);
+      await promisePool.execute(insertWebookSql,
+        [dialTimeSid, 'https://public-apps.jambonz.cloud/dial-time', 'POST']);
+      await promisePool.execute(insertWebookSql,
+        [echoSid, 'https://public-apps.jambonz.cloud/echo', 'POST']);
+
+      /* 2 applications */
+      await promisePool.execute(insertApplicationSql, [uuid(), userProfile.account_sid, 'hello world',
+        helloWordSid, callStatusSid, 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US']);
+      await promisePool.execute(insertApplicationSql, [uuid(), userProfile.account_sid, 'dial time clock',
+        dialTimeSid, callStatusSid, 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US']);
+      await promisePool.execute(insertApplicationSql, [uuid(), userProfile.account_sid, 'simple echo test',
+        echoSid, callStatusSid, 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US']);
+
+      Object.assign(userProfile, {
+        pristine: true,
+        is_active: req.body.provider !== 'local',
+        email_validated: userProfile.provider !== 'local',
+        phone_validated: false,
+        tutorial_completion: 0,
+        scope: 'read-write'
+      });
+    }
+    else if (user_sid) {
+      /* add a new user for existing account */
+      userProfile.user_sid = uuid();
+      userProfile.account_sid = account_sid;
+
+      /* changing auth mechanism, add user for existing account */
+      logger.debug(`register - creating new user for existing account ${account_sid}`);
+      if (req.body.provider === 'local') {
+        /* hash password */
+        const passwordHash = await generateHashedPassword(req.body.password);
+
+        await addLocalUser(logger, userProfile.user_sid, userProfile.account_sid,
+          userProfile.name, userProfile.email, userProfile.email_activation_code,
+          passwordHash, req.body.service_provider_sid);
+
+        /* note: we deactivate the old user once the new email is validated */
+      }
+      else {
+        await addOauthUser(logger, userProfile.user_sid, userProfile.account_sid,
+          userProfile.name, userProfile.email, userProfile.provider,
+          userProfile.provider_userid);
+
+        /* deactivate the old/replaced user */
+        const [r] = await promisePool.execute('DELETE FROM users WHERE user_sid = ?', [user_sid]);
+        logger.debug({r}, 'register - removed old user');
+        const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+        await cacheClient.delete(redisKey);
+      }
+    }
+
+    const expiresIn = parseInt(process.env.JWT_EXPIRES_IN || 60) * 60 ;
+    // generate a json web token for this user
+    const token = jwt.sign({
+      user_sid: userProfile.user_sid,
+      account_sid: userProfile.account_sid,
+      service_provider_sid: req.body.service_provider_sid,
+      scope: 'account',
+      email: userProfile.email,
+      name: userProfile.name
+    }, process.env.JWT_SECRET, { expiresIn });
+
+    logger.debug({
+      user_sid: userProfile.user_sid,
+      account_sid: userProfile.account_sid
+    }, 'generated jwt');
+
+    res.json({jwt: token, ...userProfile});
+
+    /* Store jwt based on user_id after successful login */
+    await cacheClient.set({
+      redisKey: cacheClient.generateRedisKey('jwt', userProfile.user_sid, 'v2'),
+      value: token,
+      time: expiresIn,
+    });
+
+  } catch (err) {
+    debug(err, 'Error');
+    sysError(logger, res, err);
+  }
+
+});
+
+
+module.exports = router;

lib/routes/api/sbcs.js

@@ -1,15 +1,47 @@
 const router = require('express').Router();
 const Sbc = require('../../models/sbc');
 const decorate = require('./decorate');
-const sysError = require('./error');
+const sysError = require('../error');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const {promisePool} = require('../../db');
 
-decorate(router, Sbc, ['add', 'delete']);
+const validate = (req, res) => {
+  if (req.user.hasScope('admin')) return;
+  res.status(403).json({
+    status: 'fail',
+    message: 'insufficient privileges'
+  });
+};
+
+const preconditions = {
+  'add': validate,
+  'delete': validate
+};
+
+decorate(router, Sbc, ['add', 'delete'], preconditions);
 
 /* list */
 router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const results = await Sbc.retrieveAll(req.query.service_provider_sid);
+    let service_provider_sid = req.query.service_provider_sid;
+
+    if (req.user.hasAccountAuth) {
+      const [r] = await promisePool.query('SELECT * from accounts WHERE account_sid = ?', req.user.account_sid);
+      if (0 === r.length) throw new DbErrorBadRequest('invalid account_sid');
+
+      service_provider_sid = r[0].service_provider_sid;
+    }
+
+    if (req.user.hasServiceProviderAuth) {
+      service_provider_sid = req.user.service_provider_sid;
+    }
+
+    /** generally, we have a global set of SBCs that all accounts use.
+     * However, we can have a set of SBCs that are specific for use by a service provider.
+     */
+    let results = await Sbc.retrieveAll(service_provider_sid);
+    if (results.length === 0) results = await Sbc.retrieveAll();
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);

lib/routes/api/service-providers.js

@@ -1,26 +1,211 @@
 const router = require('express').Router();
-const {DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {promisePool} = require('../../db');
+const {DbErrorForbidden} = require('../../utils/errors');
 const Webhook = require('../../models/webhook');
 const ServiceProvider = require('../../models/service-provider');
-const sysError = require('./error');
+const Account = require('../../models/account');
+const VoipCarrier = require('../../models/voip-carrier');
+const Application = require('../../models/application');
+const PhoneNumber = require('../../models/phone-number');
+const ApiKey = require('../../models/api-key');
+const {
+  hasServiceProviderPermissions,
+  parseServiceProviderSid,
+  parseVoipCarrierSid,
+} = require('./utils');
+const sysError = require('../error');
 const decorate = require('./decorate');
 const preconditions = {
-  'delete': noActiveAccounts
+  'delete': noActiveAccountsOrUsers
 };
+const sqlDeleteSipGateways = `DELETE from sip_gateways 
+WHERE voip_carrier_sid IN (
+  SELECT voip_carrier_sid 
+  FROM voip_carriers 
+  WHERE service_provider_sid = ?
+)`;
+const sqlDeleteSmppGateways = `DELETE from smpp_gateways 
+WHERE voip_carrier_sid IN (
+  SELECT voip_carrier_sid 
+  FROM voip_carriers 
+  WHERE service_provider_sid = ?
+)`;
 
-/* can not delete a service provider if it has any active accounts */
-async function noActiveAccounts(req, sid) {
+/* only admin users can add a service provider */
+function validateAdd(req) {
+  if (!req.user.hasAdminAuth) {
+    throw new DbErrorForbidden('only admin users can add a service provider');
+  }
+}
+
+async function validateRetrieve(req) {
+  try {
+    const service_provider_sid = parseServiceProviderSid(req);
+
+    if (req.user.hasScope('admin')) {
+      return;
+    }
+
+    if (req.user.hasScope('service_provider') || req.user.hasScope('account')) {
+      if (service_provider_sid === req.user.service_provider_sid) return;
+    }
+
+    throw new DbErrorForbidden('insufficient permissions');
+  } catch (error) {
+    throw error;
+  }
+}
+
+function validateUpdate(req) {
+  try {
+    const service_provider_sid = parseServiceProviderSid(req);
+
+    if (req.user.hasScope('admin')) {
+      return;
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      if (service_provider_sid === req.user.service_provider_sid) return;
+    }
+
+    throw new DbErrorForbidden('insufficient permissions to update service provider');
+  } catch (error) {
+    throw error;
+  }
+}
+
+/* can not delete a service provider if it has any active accounts or users*/
+async function noActiveAccountsOrUsers(req, sid) {
+  if (!req.user.hasAdminAuth) {
+    throw new DbErrorForbidden('only admin users can delete a service provider');
+  }
   const activeAccounts = await ServiceProvider.getForeignKeyReferences('accounts.service_provider_sid', sid);
-  if (activeAccounts > 0) throw new DbErrorUnprocessableRequest('cannot delete service provider with active accounts');
+  const activeUsers = await ServiceProvider.getForeignKeyReferences('users.service_provider_sid', sid);
+  if (activeAccounts > 0 && activeUsers > 0) throw new DbErrorForbidden('insufficient privileges');
+
+  if (activeAccounts > 0) throw new DbErrorForbidden('insufficient privileges');
+  if (activeUsers > 0) throw new DbErrorForbidden('insufficient privileges');
+
+  /* ok we can delete -- no active accounts.  remove carriers and speech credentials */
+  await promisePool.execute('DELETE from speech_credentials WHERE service_provider_sid = ?', [sid]);
+  await promisePool.query(sqlDeleteSipGateways, [sid]);
+  await promisePool.query(sqlDeleteSmppGateways, [sid]);
+  await promisePool.query('DELETE from voip_carriers WHERE service_provider_sid = ?', [sid]);
+  await promisePool.query('DELETE from api_keys WHERE service_provider_sid = ?', [sid]);
 }
 
 decorate(router, ServiceProvider, ['delete'], preconditions);
 
+router.use('/:sid/RecentCalls', hasServiceProviderPermissions, require('./recent-calls'));
+router.use('/:sid/Alerts', hasServiceProviderPermissions, require('./alerts'));
+router.use('/:sid/SpeechCredentials', require('./speech-credentials'));
+router.use('/:sid/Limits', hasServiceProviderPermissions, require('./limits'));
+router.use('/:sid/PredefinedCarriers', hasServiceProviderPermissions, require('./add-from-predefined-carrier'));
+router.get('/:sid/Accounts', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    await validateRetrieve(req);
+    const service_provider_sid = parseServiceProviderSid(req);
+    let results = await Account.retrieveAll(service_provider_sid);
+    if (req.user.hasScope('account')) {
+      results = results.filter((r) => r.account_sid === req.user.account_sid);
+    }
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid/Applications', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    await validateRetrieve(req);
+    const service_provider_sid = parseServiceProviderSid(req);
+    let results = await Application.retrieveAll(service_provider_sid);
+    if (req.user.hasScope('account')) {
+      results = results.filter((r) => r.account_sid === req.user.account_sid);
+    }
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+router.get('/:sid/PhoneNumbers', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    await validateRetrieve(req);
+    const service_provider_sid = parseServiceProviderSid(req);
+    let results = await PhoneNumber.retrieveAllForSP(service_provider_sid);
+    if (req.user.hasScope('account')) {
+      results = results.filter((r) => r.account_sid === req.user.account_sid);
+    }
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+router.get('/:sid/VoipCarriers', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    await validateRetrieve(req);
+    const service_provider_sid = parseServiceProviderSid(req);
+    const carriers = await VoipCarrier.retrieveAllForSP(service_provider_sid);
+
+    if (req.user.hasScope('account')) {
+      return res.status(200).json(carriers.filter((c) => c.account_sid === req.user.account_sid || !c.account_sid));
+    }
+
+    res.status(200).json(carriers);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+router.post('/:sid/VoipCarriers', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    validateUpdate(req);
+    const service_provider_sid = parseServiceProviderSid(req);
+    const uuid = await VoipCarrier.make({...req.body, service_provider_sid});
+    res.status(201).json({sid: uuid});
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+router.put('/:sid/VoipCarriers/:voip_carrier_sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    validateUpdate(req);
+    const sid = parseVoipCarrierSid(req);
+    const rowsAffected = await VoipCarrier.update(sid, req.body);
+    if (rowsAffected === 0) {
+      return res.sendStatus(404);
+    }
+    res.status(204).end();
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+router.get('/:sid/ApiKeys', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const {sid} = req.params;
+  try {
+    await validateRetrieve(req);
+    let results = await ApiKey.retrieveAllForSP(sid);
+    if (req.user.hasScope('account')) {
+      results = results.filter((r) => r.account_sid === req.user.account_sid);
+    }
+    res.status(200).json(results);
+    await ApiKey.updateLastUsed(sid);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
 /* add */
 router.post('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-
+    validateAdd(req);
     // create webhooks if provided
     const obj = Object.assign({}, req.body);
     for (const prop of ['registration_hook']) {
@@ -43,6 +228,12 @@ router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
     const results = await ServiceProvider.retrieveAll();
+    logger.debug({results, user: req.user}, 'ServiceProvider.retrieveAll');
+    if (req.user.hasScope('service_provider') || req.user.hasScope('account')) {
+      logger.debug(`Filtering results for ${req.user.service_provider_sid}`);
+      return res.status(200).json(results.filter((e) => req.user.service_provider_sid === e.service_provider_sid));
+    }
+
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
@@ -53,7 +244,9 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const results = await ServiceProvider.retrieve(req.params.sid);
+    await validateRetrieve(req);
+    const sid = parseServiceProviderSid(req);
+    const results = await ServiceProvider.retrieve(sid);
     if (results.length === 0) return res.status(404).end();
     return res.status(200).json(results[0]);
   }
@@ -64,9 +257,11 @@ router.get('/:sid', async(req, res) => {
 
 /* update */
 router.put('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    validateUpdate(req);
+    const sid = parseServiceProviderSid(req);
+
     // create webhooks if provided
     const obj = Object.assign({}, req.body);
     for (const prop of ['registration_hook']) {
@@ -75,15 +270,14 @@ router.put('/:sid', async(req, res) => {
           const sid = obj[prop]['webhook_sid'];
           delete obj[prop]['webhook_sid'];
           await Webhook.update(sid, obj[prop]);
-        }
-        else {
+        } else {
           const sid = await Webhook.make(obj[prop]);
           obj[`${prop}_sid`] = sid;
         }
-      }
-      else {
+      } else {
         obj[`${prop}_sid`] = null;
       }
+
       delete obj[prop];
     }
 
@@ -91,6 +285,7 @@ router.put('/:sid', async(req, res) => {
     if (rowsAffected === 0) {
       return res.status(404).end();
     }
+
     res.status(204).end();
   } catch (err) {
     sysError(logger, res, err);