fixes from initial load testing

lib/utils/install-srf-locals.js

@@ -1,3 +1,4 @@
+const Mrf = require('drachtio-fsmrf');
 const ip = require('ip');
 const localIp = ip.address();
 const PORT = process.env.HTTP_PORT || 3000;
@@ -5,19 +6,90 @@ const assert = require('assert');
 
 function installSrfLocals(srf, logger) {
   assert(!srf.locals.dbHelpers);
-
   const {getSBC, getSrf} = require('./sbc-pinger')(logger);
-  const freeswitch = process.env.JAMBONES_FREESWITCH
-    .split(',')
-    .map((fs) => {
-      const arr = /^(.*):(.*):(.*)/.exec(fs);
-      if (arr) return {address: arr[1], port: arr[2], secret: arr[3]};
-    });
-  logger.info({freeswitch}, 'freeswitch inventory');
-
   const StatsCollector = require('jambonz-stats-collector');
   const stats = srf.locals.stats = new StatsCollector(logger);
 
+  // freeswitch connections (typically we connect to only one)
+  const mrf = new Mrf(srf);
+  const mediaservers = [];
+  let idxStart = 0;
+
+  (async function() {
+    const fsInventory = process.env.JAMBONES_FREESWITCH
+      .split(',')
+      .map((fs) => {
+        const arr = /^(.*):(.*):(.*)/.exec(fs);
+        assert.ok(arr, `Invalid syntax JAMBONES_FREESWITCH: ${process.env.JAMBONES_FREESWITCH}`);
+        return {address: arr[1], port: arr[2], secret: arr[3]};
+      });
+    logger.info({fsInventory}, 'freeswitch inventory');
+
+    for (const fs of fsInventory) {
+      const val = {opts: fs, active: false, connects: 0};
+      mediaservers.push(val);
+      try {
+        const ms = await mrf.connect(fs);
+        Object.assign(val, {ms, active: true, connects: 1});
+        logger.info(`connected to freeswitch at ${fs.address}`);
+
+        ms.conn
+          .on('esl::end', () => {
+            val.active = false;
+            logger.info(`lost connection to freeswitch at ${fs.address}`);
+          })
+          .on('esl::ready', () => {
+            if (val.connects > 0) {
+              logger.info(`connected to freeswitch at ${fs.address}`);
+            }
+            val.connects = 1;
+            val.active = true;
+          });
+      }
+      catch (err) {
+        logger.info(`failed connecting to freeswitch at ${fs.address}, will retry shortly`);
+      }
+    }
+    // retry to connect to any that were initially offline
+    setInterval(async() => {
+      for (const val of mediaservers) {
+        if (val.connect === 0) {
+          try {
+            const ms = await mrf.connect(val.opts);
+            val.ms = ms;
+          } catch (err) {
+            logger.info(`failed connecting to freeswitch at ${val.opts.address}, will retry shortly`);
+          }
+        }
+      }
+    }, 3000);
+
+    // if we have a single freeswitch (as is typical) report stats periodically
+    if (mediaservers.length === 1) {
+      const ms = mediaservers[0].ms;
+      setInterval(() => {
+        try {
+          stats.gauge('fs.media.channels.in_use', ms.currentSessions);
+          stats.gauge('fs.media.channels.free', ms.maxSessions - ms.currentSessions);
+          stats.gauge('fs.media.calls_per_second', ms.cps);
+          stats.gauge('fs.media.cpu_idle', ms.cpuIdle);
+        }
+        catch (err) {
+          logger.info(err, 'Error sending media server metrics');
+        }
+      }, 30000);
+    }
+  })();
+
+  /**
+   * return an active media server
+   */
+  function getFreeswitch() {
+    const active = mediaservers.filter((mediaserver) => mediaserver.active);
+    if (active.length === 0) return null;
+    return active[idxStart++ % active.length].ms;
+  }
+
   const {
     lookupAppByPhoneNumber,
     lookupAppBySid,
@@ -54,11 +126,9 @@ function installSrfLocals(srf, logger) {
     serviceUrl: `http://${localIp}:${PORT}`,
     getSBC,
     getSrf,
-    getFreeswitch: () => freeswitch[0],
+    getFreeswitch,
     stats: stats
   });
-
-  logger.debug({locals: srf.locals}, 'srf.locals installed');
 }
 
 module.exports = installSrfLocals;