Feat/884: Capture system_alert when feature-server is online or offline (#950)

* writing alerts during startup and shutdown of feature-server

* feat/884: created constants for system component name and state

* feat/88: added 0.2.11 version of time-series

* feat/884: renamed constant, and added GracefulShutdownInProgress system alert

app.js

@@ -25,10 +25,22 @@ const opts = {
 };
 const pino = require('pino');
 const logger = pino(opts, pino.destination({sync: false}));
-const {LifeCycleEvents, FS_UUID_SET_NAME} = require('./lib/utils/constants');
+const {LifeCycleEvents, FS_UUID_SET_NAME, SystemState, FEATURE_SERVER} = require('./lib/utils/constants');
 const installSrfLocals = require('./lib/utils/install-srf-locals');
 installSrfLocals(srf, logger);
 
+const writeSystemAlerts = srf.locals?.writeSystemAlerts;
+if (writeSystemAlerts) {
+  writeSystemAlerts({
+    system_component: FEATURE_SERVER,
+    state : SystemState.Online,
+    fields : {
+      detail: `feature-server with process_id ${process.pid} started`,
+      host: srf.locals?.ipv4
+    }
+  });
+}
+
 const {
   initLocals,
   createRootSpan,
@@ -124,13 +136,25 @@ const disconnect = () => {
     srf.locals.mediaservers?.forEach((ms) => ms.disconnect());
   });
 };
-
 process.on('SIGTERM', handle);
+process.on('SIGINT', handle);
 
-function handle(signal) {
+async function handle(signal) {
   const {removeFromSet} = srf.locals.dbHelpers;
   srf.locals.disabled = true;
   logger.info(`got signal ${signal}`);
+  const writeSystemAlerts = srf.locals?.writeSystemAlerts;
+  if (writeSystemAlerts) {
+    // it has to be synchronous call, or else by the time system saves the app terminates
+    await writeSystemAlerts({
+      system_component: FEATURE_SERVER,
+      state : SystemState.Offline,
+      fields : {
+        detail: `feature-server with process_id ${process.pid} stopped, signal ${signal}`,
+        host: srf.locals?.ipv4
+      }
+    });
+  }
   const setName = `${(JAMBONES_CLUSTER_ID || 'default')}:active-fs`;
   const fsServiceUrlSetName = `${(JAMBONES_CLUSTER_ID || 'default')}:fs-service-url`;
   if (setName && srf.locals.localSipAddress) {

lib/utils/constants.json

@@ -228,5 +228,11 @@
   },
   "MAX_SIMRINGS": 10,
   "BONG_TONE": "tone_stream://v=-7;%(100,0,941.0,1477.0);v=-7;>=2;+=.1;%(1400,0,350,440)",
-  "FS_UUID_SET_NAME": "fsUUIDs"
+  "FS_UUID_SET_NAME": "fsUUIDs",
+  "SystemState" : {
+    "Online": "ONLINE",
+    "Offline": "OFFLINE",
+    "GracefulShutdownInProgress":"SHUTDOWN_IN_PROGRESS"
+  },
+  "FEATURE_SERVER" : "feature-server"
 }

lib/utils/install-srf-locals.js

@@ -199,7 +199,8 @@ function installSrfLocals(srf, logger) {
   } = require('@jambonz/speech-utils')({}, logger);
   const {
     writeAlerts,
-    AlertType
+    AlertType,
+    writeSystemAlerts
   } = require('@jambonz/time-series')(logger, {
     host: JAMBONES_TIME_SERIES_HOST,
     commitSize: 50,
@@ -269,7 +270,8 @@ function installSrfLocals(srf, logger) {
     getFreeswitch,
     stats: stats,
     writeAlerts,
-    AlertType
+    AlertType,
+    writeSystemAlerts
   };
 
   if (localIp) {

lib/utils/sbc-pinger.js

@@ -46,12 +46,24 @@ module.exports = (logger) => {
             const {srf} = require('../..');
             srf.locals.publicIp = publicIp;
           })
-          .on(LifeCycleEvents.ScaleIn, () => {
+          .on(LifeCycleEvents.ScaleIn, async() => {
             logger.info('AWS scale-in notification: begin drying up calls');
             dryUpCalls = true;
             lifecycleEmitter.operationalState = LifeCycleEvents.ScaleIn;
 
             const {srf} = require('../..');
+            const {writeSystemAlerts} = srf.locals;
+            if (writeSystemAlerts) {
+              const {SystemState, FEATURE_SERVER} = require('./constants');
+              await writeSystemAlerts({
+                system_component: FEATURE_SERVER,
+                state : SystemState.GracefulShutdownInProgress,
+                fields : {
+                  detail: `feature-server with process_id ${process.pid} shutdown in progress`,
+                  host: srf.locals?.ipv4
+                }
+              });
+            }
             pingProxies(srf);
 
             // if we have zero calls, we can complete the scale-in right