Feat/cli remove fs (#116)

* support draining feature server manually

* support draining feature server manually

* support CLI to add or remove feature server

* wip

* wip

* wip

* add redis key for feature server and integration test

* wip

README.md

@@ -29,6 +29,50 @@ Configuration is provided via environment variables:
 |JAMBONES_REGBOT_DEFAULT_EXPIRES_INTERVAL| default expire value for outbound registration in seconds (default 3600) |no|
 |JAMBONES_REGBOT_MIN_EXPIRES_INTERVAL| minimum expire value for outbound registration in seconds (default 30) |no|
 
+## CLI Management
+
+The SBC provides a CLI tool for runtime management of feature servers. Use the CLI to drain/undrain feature servers during maintenance or scaling operations.
+
+### Installation
+```bash
+npm install
+```
+
+### Usage
+```bash
+# Show all commands
+npm run cli
+
+# Drain a feature server (remove from active pool)
+npm run cli fs drain 192.168.1.10
+
+# Undrain a feature server (add back to active pool)  
+npm run cli fs undrain 192.168.1.10
+
+# List currently drained servers
+npm run cli fs drained
+
+# List all available feature servers
+npm run cli fs active
+
+# Show all servers with status
+npm run cli fs list
+```
+
+### Examples
+```bash
+# During maintenance - drain server before updates
+npm run cli fs drain 10.0.1.5
+
+# After maintenance - bring server back online
+npm run cli fs undrain 10.0.1.5
+
+# Check which servers are available for draining
+npm run cli fs active
+```
+
+The CLI connects via Unix socket (`/tmp/sbc-sip-sidecar.sock`) and requires admin access to the server.
+
 ## Registrar database
 
 A redis database is used to hold active registrations. When a register request arrives and is authenticated, the following values are parsed from the request:

app.js

@@ -17,6 +17,7 @@ const {
   NODE_ENV,
   SBC_PUBLIC_ADDRESS_KEEP_ALIVE_IN_MILISECOND
 } = require('./lib/config');
+
 assert.ok(JAMBONES_MYSQL_HOST &&
   JAMBONES_MYSQL_USER &&
   JAMBONES_MYSQL_PASSWORD &&
@@ -260,6 +261,9 @@ srf.use('options', [
 srf.register(require('./lib/register')({logger}));
 srf.options(require('./lib/options')({srf, logger}));
 
+// Start CLI runtime config server with access to srf.locals
+require('./lib/cli/runtime-config').initialize(srf.locals, logger);
+
 setInterval(async() => {
   const count = await srf.locals.registrar.getCountOfUsers();
   debug(`count of registered users: ${count}`);

lib/cli/cli.js

@@ -0,0 +1,239 @@
+#!/usr/bin/env node
+
+const net = require('net');
+
+const SOCKET_PATH = process.env.SBC_SOCKET_PATH || '/tmp/sbc-sip-sidecar.sock';
+
+function connectAndSend(command) {
+  return new Promise((resolve, reject) => {
+    const socket = net.createConnection(SOCKET_PATH);
+
+    socket.on('connect', () => {
+      socket.write(JSON.stringify(command) + '\n');
+    });
+
+    socket.on('data', (data) => {
+      try {
+        const response = JSON.parse(data.toString().trim());
+        socket.end();
+        resolve(response);
+      } catch {
+        socket.end();
+        reject(new Error('Invalid response from server'));
+      }
+    });
+
+    socket.on('error', (err) => {
+      reject(new Error(`Connection failed: ${err.message}`));
+    });
+
+    socket.setTimeout(5000, () => {
+      socket.end();
+      reject(new Error('Command timed out'));
+    });
+  });
+}
+
+function exitWithError(message) {
+  console.error('Error:', message);
+  process.exit(1);
+}
+
+function validateArgs(args, minCount, usage) {
+  if (args.length < minCount) {
+    exitWithError(`${usage}`);
+  }
+}
+
+async function handleFeatureServerCommand(args) {
+  const [action, server] = args;
+
+  switch (action) {
+    case 'drain':
+      validateArgs(args, 2, 'drain requires server IP address');
+      const drainResult = await connectAndSend({ action: 'fs-drain', server });
+
+      if (!drainResult.success) {
+        exitWithError(drainResult.error || 'Failed to drain server');
+      }
+
+      console.log(`✓ Drained ${drainResult.server}`);
+      console.log(`Drained servers: [${drainResult.drained.join(', ')}]`);
+      break;
+
+    case 'undrain':
+      validateArgs(args, 2, 'undrain requires server IP address');
+      const undrainResult = await connectAndSend({ action: 'fs-undrain', server });
+
+      if (!undrainResult.success) {
+        exitWithError(undrainResult.error || 'Failed to undrain server');
+      }
+
+      console.log(`✓ Undrained ${undrainResult.server}`);
+      console.log(`Drained servers: [${undrainResult.drained.join(', ')}]`);
+      break;
+
+    case 'drained':
+      const drainedResult = await connectAndSend({ action: 'fs-drained' });
+
+      if (drainedResult.error) {
+        exitWithError(drainedResult.error);
+      }
+
+      if (drainedResult.drained.length === 0) {
+        console.log('No servers are currently drained');
+      } else {
+        console.log('Drained servers:');
+        drainedResult.drained.forEach((server) => console.log(`  🔴 ${server}`));
+      }
+      break;
+
+    case 'active':
+      const activeResult = await connectAndSend({ action: 'fs-available' });
+
+      if (!activeResult.success) {
+        exitWithError(activeResult.error || 'Failed to get active servers');
+      }
+
+      if (activeResult.available.length === 0) {
+        console.log('No active feature servers found');
+      } else {
+        console.log('Available feature servers:');
+        activeResult.available.forEach((server) => console.log(`  🟢 ${server}`));
+      }
+      break;
+
+    case 'list':
+      const listResult = await connectAndSend({ action: 'fs-list' });
+
+      if (listResult.error) {
+        exitWithError(listResult.error);
+      }
+
+      if (listResult.servers.length === 0) {
+        console.log('No servers configured');
+        if (listResult.drained.length > 0) {
+          console.log('Orphaned drained servers:');
+          listResult.drained.forEach((server) => console.log(`  🔴 ${server} (orphaned)`));
+        }
+      } else {
+        console.log('Feature servers:');
+        listResult.servers.forEach(({ server, status }) => {
+          const icon = status === 'drained' ? '🔴' : '🟢';
+          console.log(`  ${icon} ${server} (${status})`);
+        });
+      }
+      break;
+
+    default:
+      exitWithError(`Unknown fs command: ${action}. Use: drain, undrain, drained, active, list`);
+  }
+}
+
+function showHelp() {
+  console.log('SBC Runtime CLI');
+  console.log('');
+  console.log('Usage:');
+  console.log('  npm run cli <command> [options]');
+  console.log('');
+  console.log('Feature Server Commands:');
+  console.log('  npm run cli fs drain <ip>      Drain server (remove from pool)');
+  console.log('  npm run cli fs undrain <ip>    Undrain server (add back to pool)');
+  console.log('  npm run cli fs drained         Show drained servers');
+  console.log('  npm run cli fs active          Show available servers');
+  console.log('  npm run cli fs list            Show all servers with status');
+  console.log('');
+  console.log('Configuration Commands:');
+  console.log('  npm run cli set <key> <value>  Set runtime config');
+  console.log('  npm run cli get <key>          Get runtime config');
+  console.log('  npm run cli list               Show all runtime config');
+  console.log('');
+  console.log('Examples:');
+  console.log('  npm run cli fs drain 192.168.1.10');
+  console.log('  npm run cli fs active');
+}
+
+async function handleConfigCommand(action, args) {
+  switch (action) {
+    case 'set':
+      validateArgs(args, 2, 'set requires key and value');
+      const [key, value] = args;
+      const setResult = await connectAndSend({ action: 'set', key, value });
+
+      if (setResult.error) {
+        exitWithError(setResult.error);
+      }
+
+      console.log(`✓ ${setResult.key} = ${JSON.stringify(setResult.value)}`);
+      break;
+
+    case 'get':
+      validateArgs(args, 1, 'get requires key');
+      const getResult = await connectAndSend({ action: 'get', key: args[0] });
+
+      if (getResult.error) {
+        exitWithError(getResult.error);
+      }
+
+      console.log(`${getResult.key} = ${JSON.stringify(getResult.value)}`);
+      break;
+
+    case 'list':
+      const listResult = await connectAndSend({ action: 'list' });
+
+      if (listResult.error) {
+        exitWithError(listResult.error);
+      }
+
+      console.log('Runtime Configuration:');
+      const entries = Object.entries(listResult.config);
+      if (entries.length === 0) {
+        console.log('  (no configuration set)');
+      } else {
+        entries.forEach(([key, value]) => {
+          console.log(`  ${key} = ${JSON.stringify(value)}`);
+        });
+      }
+      break;
+
+    default:
+      exitWithError(`Unknown config command: ${action}. Use: set, get, list`);
+  }
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+
+  if (args.length === 0) {
+    showHelp();
+    return;
+  }
+
+  const [command, ...remainingArgs] = args;
+
+  try {
+    switch (command) {
+      case 'fs':
+        validateArgs(remainingArgs, 1, 'fs command requires an action');
+        await handleFeatureServerCommand(remainingArgs);
+        break;
+
+      case 'set':
+      case 'get':
+      case 'list':
+        await handleConfigCommand(command, remainingArgs);
+        break;
+
+      default:
+        exitWithError(`Unknown command: ${command}. Run without arguments for help.`);
+    }
+  } catch (error) {
+    exitWithError(error.message);
+  }
+}
+
+if (require.main === module) {
+  main().catch((error) => {
+    exitWithError(error.message);
+  });
+}

lib/cli/feature-server-config.js

@@ -0,0 +1,81 @@
+const runtimeConfigModule = require('./runtime-config');
+
+function isValidIP(ip) {
+  if (!ip || typeof ip !== 'string') return false;
+
+  // IPv4 check
+  const ipv4 = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
+  if (ipv4.test(ip)) return true;
+
+  // IPv6 check (basic patterns)
+  const ipv6 = /^(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$|^::1$|^::$/;
+  if (ipv6.test(ip)) return true;
+
+  // IPv6 compressed
+  const ipv6Short = new RegExp([
+    '^([0-9a-fA-F]{1,4}:)*::([0-9a-fA-F]{1,4}:)*[0-9a-fA-F]{1,4}$',
+    '^([0-9a-fA-F]{1,4}:)*::[0-9a-fA-F]{1,4}$',
+    '^[0-9a-fA-F]{1,4}::([0-9a-fA-F]{1,4}:)*[0-9a-fA-F]{1,4}$'
+  ].join('|'));
+  if (ipv6Short.test(ip)) return true;
+
+  return false;
+}
+
+async function isDrained(serverIP) {
+  if (!isValidIP(serverIP)) {
+    return false;
+  }
+
+  try {
+    const runtimeConfig = runtimeConfigModule.getInstance();
+    return await runtimeConfig.isServerDrained(serverIP);
+  } catch {
+    return false;
+  }
+}
+
+async function getDrainedServers() {
+  try {
+    const runtimeConfig = runtimeConfigModule.getInstance();
+    return await runtimeConfig.getDrainedFeatureServers();
+  } catch {
+    return [];
+  }
+}
+
+async function getActiveServers() {
+  try {
+    const runtimeConfig = runtimeConfigModule.getInstance();
+    return await runtimeConfig.getActiveFeatureServers();
+  } catch {
+    return [];
+  }
+}
+
+async function getAvailableServers() {
+  try {
+    const runtimeConfig = runtimeConfigModule.getInstance();
+    return await runtimeConfig.getAvailableFeatureServers();
+  } catch {
+    return [];
+  }
+}
+
+async function getAllServersWithStatus() {
+  try {
+    const runtimeConfig = runtimeConfigModule.getInstance();
+    return await runtimeConfig.getAllFeatureServersWithStatus();
+  } catch {
+    return { servers: [], drained: [] };
+  }
+}
+
+module.exports = {
+  isValidIP,
+  isDrained,
+  getDrainedServers,
+  getActiveServers,
+  getAvailableServers,
+  getAllServersWithStatus
+};

lib/cli/runtime-config.js

@@ -0,0 +1,427 @@
+const net = require('net');
+
+const config = new Map();
+const queue = [];
+let processing = false;
+let logger;
+
+async function runOperation(operation) {
+  return new Promise((resolve, reject) => {
+    queue.push({ operation, resolve, reject });
+    processQueue();
+  });
+}
+
+async function processQueue() {
+  if (processing || queue.length === 0) return;
+
+  processing = true;
+
+  while (queue.length > 0) {
+    const { operation, resolve, reject } = queue.shift();
+    try {
+      const result = await operation();
+      resolve(result);
+    } catch (err) {
+      reject(err);
+    }
+  }
+
+  processing = false;
+}
+
+class RuntimeConfig {
+  constructor(srfLocals = null, appLogger = null) {
+    this.server = null;
+    this.socketPath = process.env.SBC_SOCKET_PATH || '/tmp/sbc-sip-sidecar.sock';
+    this.srfLocals = srfLocals;
+    if (appLogger) {
+      logger = appLogger;
+    } else if (!logger) {
+      throw new Error('Logger is required for RuntimeConfig');
+    }
+    this.startServer();
+  }
+
+  async set(key, value) {
+    return runOperation(() => {
+      config.set(key, value);
+      logger.info({ key, value }, 'Config updated');
+      return { key, value };
+    });
+  }
+
+  async get(key, defaultValue) {
+    return runOperation(() => {
+      return config.has(key) ? config.get(key) : defaultValue;
+    });
+  }
+
+  async addToArray(key, item) {
+    return runOperation(() => {
+      let arr = config.get(key) || [];
+
+      if (typeof arr === 'string') {
+        arr = arr.split(',').map((s) => s.trim()).filter((s) => s.length > 0);
+      }
+
+      if (!Array.isArray(arr)) arr = [];
+
+      const exists = arr.includes(item);
+
+      if (!exists) {
+        arr.push(item);
+        config.set(key, arr);
+        logger.info({ key, item, array: arr }, 'Added to array');
+      }
+
+      return { key, item, array: arr, added: !exists };
+    });
+  }
+
+  async removeFromArray(key, item) {
+    return runOperation(() => {
+      let arr = config.get(key) || [];
+
+      if (typeof arr === 'string') {
+        arr = arr.split(',').map((s) => s.trim()).filter((s) => s.length > 0);
+      }
+
+      if (!Array.isArray(arr)) arr = [];
+
+      const originalLength = arr.length;
+      arr = arr.filter((existing) => existing !== item);
+
+      if (arr.length !== originalLength) {
+        config.set(key, arr);
+        logger.info({ key, item, array: arr }, 'Removed from array');
+      }
+
+      return { key, item, array: arr, removed: arr.length !== originalLength };
+    });
+  }
+
+  async getAll() {
+    return runOperation(() => Object.fromEntries(config));
+  }
+
+  // Helper methods for cleaner code
+  checkRedisConnection(requiredMethod) {
+    return this.srfLocals && this.srfLocals[requiredMethod];
+  }
+
+  sendError(socket, message) {
+    socket.write(JSON.stringify({ success: false, error: message }) + '\n');
+  }
+
+  sendSuccess(socket, data) {
+    socket.write(JSON.stringify({ success: true, ...data }) + '\n');
+  }
+
+  getSetNames() {
+    const JAMBONES_CLUSTER_ID = process.env.JAMBONES_CLUSTER_ID;
+    return {
+      activeFs: `${(JAMBONES_CLUSTER_ID || 'default')}:active-fs`,
+      drainedFs: `${(JAMBONES_CLUSTER_ID || 'default')}:drained-fs`
+    };
+  }
+
+  // Feature server utility methods
+  async isServerDrained(serverIP) {
+    if (!this.checkRedisConnection('isMemberOfSet')) {
+      return false;
+    }
+
+    try {
+      const { drainedFs } = this.getSetNames();
+      return await this.srfLocals.isMemberOfSet(drainedFs, serverIP);
+    } catch {
+      return false;
+    }
+  }
+
+  async getDrainedFeatureServers() {
+    if (!this.checkRedisConnection('retrieveSet')) {
+      return [];
+    }
+
+    try {
+      const { drainedFs } = this.getSetNames();
+      const servers = await this.srfLocals.retrieveSet(drainedFs);
+      return servers || [];
+    } catch {
+      return [];
+    }
+  }
+
+  async getActiveFeatureServers() {
+    if (!this.checkRedisConnection('retrieveSet')) {
+      return [];
+    }
+
+    try {
+      const { activeFs } = this.getSetNames();
+      const servers = await this.srfLocals.retrieveSet(activeFs);
+      return servers || [];
+    } catch {
+      return [];
+    }
+  }
+
+  async getAvailableFeatureServers() {
+    try {
+      const [active, drained] = await Promise.all([
+        this.getActiveFeatureServers(),
+        this.getDrainedFeatureServers()
+      ]);
+
+      const drainedSet = new Set(drained);
+      return active.filter((server) => !drainedSet.has(server));
+    } catch {
+      return [];
+    }
+  }
+
+  async getAllFeatureServersWithStatus() {
+    try {
+      const [active, drained] = await Promise.all([
+        this.getActiveFeatureServers(),
+        this.getDrainedFeatureServers()
+      ]);
+
+      const drainedSet = new Set(drained);
+      const servers = active.map((server) => ({
+        server,
+        status: drainedSet.has(server) ? 'drained' : 'active'
+      }));
+
+      return { servers, drained };
+    } catch {
+      return { servers: [], drained: [] };
+    }
+  }
+
+  async handleFeatureServerDrain(socket, server) {
+    if (!this.checkRedisConnection('addToSet')) {
+      return this.sendError(socket, 'Redis connection not available');
+    }
+
+    const { isValidIP } = require('./feature-server-config');
+    if (!isValidIP(server)) {
+      return this.sendError(socket, `Invalid IP address: ${server}`);
+    }
+
+    try {
+      const { drainedFs } = this.getSetNames();
+      await this.srfLocals.addToSet(drainedFs, server);
+      const drainedServers = await this.srfLocals.retrieveSet(drainedFs);
+      this.sendSuccess(socket, {
+        action: 'drain',
+        server,
+        drained: drainedServers || []
+      });
+    } catch (err) {
+      logger.error({ err }, 'Error draining server');
+      this.sendError(socket, 'Failed to drain server');
+    }
+  }
+
+  async handleFeatureServerUndrain(socket, server) {
+    if (!this.checkRedisConnection('removeFromSet')) {
+      return this.sendError(socket, 'Redis connection not available');
+    }
+
+    const { isValidIP } = require('./feature-server-config');
+    if (!isValidIP(server)) {
+      return this.sendError(socket, `Invalid IP address: ${server}`);
+    }
+
+    try {
+      const { drainedFs } = this.getSetNames();
+      await this.srfLocals.removeFromSet(drainedFs, server);
+      const drainedServers = await this.srfLocals.retrieveSet(drainedFs);
+      this.sendSuccess(socket, {
+        action: 'undrain',
+        server,
+        drained: drainedServers || []
+      });
+    } catch (err) {
+      logger.error({ err }, 'Error undraining server');
+      this.sendError(socket, 'Failed to undrain server');
+    }
+  }
+
+  async handleFeatureServerDrained(socket) {
+    try {
+      const drainedServers = await this.getDrainedFeatureServers();
+      this.sendSuccess(socket, { drained: drainedServers });
+    } catch (err) {
+      logger.error({ err }, 'Error retrieving drained servers');
+      this.sendError(socket, 'Failed to retrieve drained servers');
+    }
+  }
+
+  async handleFeatureServerList(socket) {
+    try {
+      const result = await this.getAllFeatureServersWithStatus();
+      this.sendSuccess(socket, result);
+    } catch (err) {
+      logger.error({ err }, 'Error listing servers');
+      this.sendError(socket, 'Failed to list servers');
+    }
+  }
+
+  async handleFeatureServerAvailable(socket) {
+    try {
+      const availableServers = await this.getAvailableFeatureServers();
+      this.sendSuccess(socket, { available: availableServers });
+    } catch (err) {
+      logger.error({ err }, 'Error retrieving available servers');
+      this.sendError(socket, 'Failed to retrieve available servers');
+    }
+  }
+
+  startServer() {
+    try {
+      require('fs').unlinkSync(this.socketPath);
+    } catch {
+      // socket file doesn't exist, that's fine
+    }
+
+    this.server = net.createServer((socket) => {
+      let buffer = '';
+
+      socket.on('data', (data) => {
+        buffer += data.toString();
+
+        const lines = buffer.split('\n');
+        buffer = lines.pop();
+
+        for (const line of lines) {
+          if (line.trim()) {
+            this.handleCommand(socket, line.trim());
+          }
+        }
+      });
+
+      socket.on('error', (err) => {
+        logger.error({ err }, 'Socket error');
+      });
+    });
+
+    this.server.listen(this.socketPath, () => {
+      logger.info({ socketPath: this.socketPath }, 'CLI server started');
+      require('fs').chmodSync(this.socketPath, 0o600);
+    });
+
+    this.server.on('error', (err) => {
+      logger.error({ err }, 'Server error');
+    });
+  }
+
+  async handleCommand(socket, jsonString) {
+    try {
+      const cmd = JSON.parse(jsonString);
+
+      switch (cmd.action) {
+        case 'set':
+          const setResult = await this.set(cmd.key, cmd.value);
+          this.sendSuccess(socket, setResult);
+          break;
+
+        case 'get':
+          const value = await this.get(cmd.key);
+          this.sendSuccess(socket, { key: cmd.key, value });
+          break;
+
+        case 'add':
+          const addResult = await this.addToArray(cmd.key, cmd.item);
+          this.sendSuccess(socket, addResult);
+          break;
+
+        case 'remove':
+          const removeResult = await this.removeFromArray(cmd.key, cmd.item);
+          this.sendSuccess(socket, removeResult);
+          break;
+
+        case 'list':
+          const allConfig = await this.getAll();
+          this.sendSuccess(socket, { config: allConfig });
+          break;
+
+        case 'fs-drain':
+          await this.handleFeatureServerDrain(socket, cmd.server);
+          break;
+
+        case 'fs-undrain':
+          await this.handleFeatureServerUndrain(socket, cmd.server);
+          break;
+
+        case 'fs-drained':
+          await this.handleFeatureServerDrained(socket);
+          break;
+
+        case 'fs-list':
+          await this.handleFeatureServerList(socket);
+          break;
+
+        case 'fs-available':
+          await this.handleFeatureServerAvailable(socket);
+          break;
+
+        default:
+          this.sendError(socket, 'Unknown action');
+      }
+    } catch (err) {
+      logger.error({ err, command: jsonString }, 'Command error');
+      this.sendError(socket, 'Invalid command');
+    }
+  }
+
+  shutdown() {
+    if (this.server) {
+      this.server.close();
+      try {
+        require('fs').unlinkSync(this.socketPath);
+      } catch {
+        // ignore
+      }
+    }
+  }
+}
+
+let runtimeConfig = null;
+
+function createInstance(srfLocals = null, appLogger = null) {
+  const instance = new RuntimeConfig(srfLocals, appLogger);
+  process.on('SIGINT', () => instance.shutdown());
+  process.on('SIGTERM', () => instance.shutdown());
+  return instance;
+}
+
+function initialize(srfLocals, appLogger) {
+  if (!appLogger) {
+    throw new Error('Logger is required for RuntimeConfig initialization');
+  }
+  if (!runtimeConfig) {
+    runtimeConfig = createInstance(srfLocals, appLogger);
+  } else {
+    // Update existing instance with srfLocals and logger
+    runtimeConfig.srfLocals = srfLocals;
+    logger = appLogger;
+  }
+
+  return runtimeConfig;
+}
+
+function getInstance() {
+  if (!runtimeConfig) {
+    throw new Error('RuntimeConfig not initialized. Call initialize() first with logger.');
+  }
+  return runtimeConfig;
+}
+
+module.exports = {
+  initialize,
+  getInstance
+};

lib/options.js

@@ -1,4 +1,5 @@
 const debug = require('debug')('jambonz:sbc-options-handler');
+const { isDrained } = require('./cli/feature-server-config');
 const {
   EXPIRES_INTERVAL,
   CHECK_EXPIRES_INTERVAL,
@@ -132,6 +133,15 @@ module.exports = ({srf, logger}) => {
 
         status = req.get(`${prefix}-Status`);
 
+        // If feature server is drained, force status to closed
+        if (status === 'open' && !isRtpServer) {
+          const fsIP = req.source_address;
+          if (await isDrained(fsIP)) {
+            logger.warn({fsIP}, 'drained feature server attempted to check in - rejecting');
+            status = 'closed';
+          }
+        }
+
         countOfMembers = await _addToCache(map, status, setName, key);
         if (fsServiceUrlKey) {
           await _addToCache(fsServiceUrls, status, setNameFsSeriveUrl, fsServiceUrlKey);

package.json

@@ -12,6 +12,7 @@
   ],
   "scripts": {
     "start": "node app",
+    "cli": "node lib/cli/cli.js",
     "test": "NODE_ENV=test JAMBONES_HOSTING=1 HTTP_POOL=1 JWT_SECRET=foobarbazzle DRACHTIO_HOST=127.0.0.1 DRACHTIO_PORT=9022 DRACHTIO_SECRET=cymru JAMBONES_MYSQL_HOST=127.0.0.1 JAMBONES_MYSQL_PORT=3306 JAMBONES_MYSQL_USER=jambones_test JAMBONES_MYSQL_PASSWORD=jambones_test JAMBONES_MYSQL_DATABASE=jambones_test JAMBONES_REDIS_HOST=127.0.0.1 JAMBONES_REDIS_PORT=16379 JAMBONES_LOGLEVEL=debug ENABLE_METRICS=0 HTTP_PORT=3000 JAMBONES_SBCS=172.39.0.10 JAMBONES_FREESWITCH=127.0.0.1:8022:ClueCon:docker-host JAMBONES_TIME_SERIES_HOST=127.0.0.1 JAMBONES_NETWORK_CIDR=172.39.0.0/16 node test/ ",
     "coverage": "./node_modules/.bin/nyc --reporter html --report-dir ./coverage npm run test",
     "jslint": "eslint app.js lib"

test/cli-tests.js

@@ -0,0 +1,442 @@
+const test = require('tape');
+const {
+  JAMBONES_REDIS_HOST,
+  JAMBONES_REDIS_PORT,
+  JAMBONES_LOGLEVEL,
+  JAMBONES_CLUSTER_ID,
+} = require('../lib/config');
+const clearModule = require('clear-module');
+const exec = require('child_process').exec;
+const opts = Object.assign({
+  timestamp: () => { return `, "time": "${new Date().toISOString()}"`; }
+}, { level: JAMBONES_LOGLEVEL || 'info' });
+const logger = require('pino')(opts);
+const {
+  addToSet,
+  removeFromSet,
+  retrieveSet
+} = require('@jambonz/realtimedb-helpers')({
+  host: JAMBONES_REDIS_HOST || 'localhost',
+  port: JAMBONES_REDIS_PORT || 6379
+}, logger);
+
+const { isValidIP } = require('../lib/cli/feature-server-config');
+
+const activeSetName = `${(JAMBONES_CLUSTER_ID || 'default')}:active-fs`;
+const drainedSetName = `${(JAMBONES_CLUSTER_ID || 'default')}:drained-fs`;
+
+process.on('unhandledRejection', (reason, p) => {
+  console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
+});
+
+function connect(connectable) {
+  return new Promise((resolve, reject) => {
+    connectable.on('connect', () => {
+      return resolve();
+    });
+  });
+}
+
+const wait = (duration) => {
+  return new Promise((resolve) => {
+    setTimeout(resolve, duration);
+  });
+};
+
+function runCli(command) {
+  return new Promise((resolve, reject) => {
+    exec(`npm run cli ${command}`, { cwd: __dirname + '/..' }, (err, stdout, stderr) => {
+      if (err) {
+        return reject({ err, stderr, stdout });
+      }
+      resolve({ stdout, stderr });
+    });
+  });
+}
+
+// Setup test data
+test('setup feature server test data', async (t) => {
+  try {
+    // Clear any existing test data
+    const existing = await retrieveSet(drainedSetName);
+    if (existing && existing.length > 0) {
+      for (const server of existing) {
+        await removeFromSet(drainedSetName, server);
+      }
+    }
+
+    // Add some test feature servers to active set
+    await addToSet(activeSetName, '192.168.1.10');
+    await addToSet(activeSetName, '192.168.1.11');
+    await addToSet(activeSetName, '192.168.1.12');
+    
+    // Add one server to drained set initially
+    await addToSet(drainedSetName, '192.168.1.11');
+    
+    t.pass('test data setup complete');
+    t.end();
+  } catch (err) {
+    t.fail(`Failed to setup test data: ${err.message}`);
+    t.end();
+  }
+});
+
+test('CLI integration tests', (t) => {
+  clearModule.all();
+  const { srf } = require('../app');
+  t.timeoutAfter(30000);
+
+  connect(srf)
+    .then(() => wait(1000)) // Wait for CLI server to start
+    .then(async () => {
+      // Test CLI help
+      const helpResult = await runCli('');
+      t.ok(helpResult.stdout.includes('SBC Runtime CLI'), 'CLI help displays correctly');
+      t.pass('CLI help command works');
+
+      // Test fs active command
+      const activeResult = await runCli('fs active');
+      t.ok(activeResult.stdout.includes('Available feature servers'), 'fs active shows header');
+      t.ok(activeResult.stdout.includes('192.168.1.10'), 'fs active shows test server');
+      t.pass('fs active command works');
+
+      // Test fs drained command
+      const drainedResult = await runCli('fs drained');
+      t.ok(drainedResult.stdout.includes('192.168.1.11'), 'fs drained shows initially drained server');
+      t.pass('fs drained command works');
+
+      // Test drain a server
+      const drainResult = await runCli('fs drain 192.168.1.10');
+      t.ok(drainResult.stdout.includes('✓ Drained 192.168.1.10'), 'drain command shows success');
+      t.pass('fs drain command works');
+
+      // Verify server is drained
+      const drainedAfterResult = await runCli('fs drained');
+      t.ok(drainedAfterResult.stdout.includes('192.168.1.10'), 'drained server appears in list');
+      t.pass('drained server verification works');
+
+      // Test undrain a server
+      const undrainResult = await runCli('fs undrain 192.168.1.10');
+      t.ok(undrainResult.stdout.includes('✓ Undrained 192.168.1.10'), 'undrain command shows success');
+      t.pass('fs undrain command works');
+
+      // Test fs list command
+      const listResult = await runCli('fs list');
+      t.ok(listResult.stdout.includes('Feature servers'), 'fs list shows header');
+      t.ok(listResult.stdout.includes('192.168.1.10'), 'fs list shows servers');
+      t.pass('fs list command works');
+
+      return Promise.resolve();
+    })
+    .then(() => {
+      if (srf) srf.disconnect();
+      t.end();
+    })
+    .catch((err) => {
+      console.log('CLI test error:', err);
+      if (srf) srf.disconnect();
+      t.fail(`CLI test failed: ${err.err ? err.err.message : err.message}`);
+      if (err.stderr) console.log('stderr:', err.stderr);
+      if (err.stdout) console.log('stdout:', err.stdout);
+      t.end();
+    });
+});
+
+test('CLI error handling', (t) => {
+  clearModule.all();
+  const { srf } = require('../app');
+  t.timeoutAfter(15000);
+
+  connect(srf)
+    .then(() => wait(500))
+    .then(async () => {
+      // Test invalid IP for drain
+      try {
+        await runCli('fs drain invalid-ip');
+        t.fail('drain with invalid IP should fail');
+      } catch (err) {
+        t.ok(err.stderr.includes('Invalid IP') || err.stdout.includes('Invalid IP'), 'drain rejects invalid IP');
+        t.pass('invalid IP validation works');
+      }
+
+      // Test unknown command
+      try {
+        await runCli('unknown-command');
+        t.fail('unknown command should fail');
+      } catch (err) {
+        t.ok(err.stderr.includes('Unknown command') || err.stdout.includes('Unknown command'), 'unknown command rejected');
+        t.pass('unknown command handling works');
+      }
+
+      // Test missing server argument for drain
+      try {
+        await runCli('fs drain');
+        t.fail('drain without server should fail');
+      } catch (err) {
+        t.ok(err.stderr.includes('requires server') || err.stdout.includes('requires server'), 'drain requires server argument');
+        t.pass('missing argument validation works');
+      }
+
+      return Promise.resolve();
+    })
+    .then(() => {
+      if (srf) srf.disconnect();
+      t.end();
+    })
+    .catch((err) => {
+      console.log('CLI error test error:', err);
+      if (srf) srf.disconnect();
+      t.fail(`CLI error test failed: ${err.message}`);
+      t.end();
+    });
+});
+
+test('CLI config commands', (t) => {
+  clearModule.all();
+  const { srf } = require('../app');
+  t.timeoutAfter(15000);
+
+  connect(srf)
+    .then(() => wait(500))
+    .then(async () => {
+      // Test set command
+      const setResult = await runCli('set testKey testValue');
+      t.ok(setResult.stdout.includes('testKey = "testValue"'), 'set command works');
+      t.pass('CLI set command works');
+
+      // Test get command
+      const getResult = await runCli('get testKey');
+      t.ok(getResult.stdout.includes('testKey = "testValue"'), 'get command returns correct value');
+      t.pass('CLI get command works');
+
+      // Test list command
+      const listResult = await runCli('list');
+      t.ok(listResult.stdout.includes('Runtime Configuration'), 'list command shows header');
+      t.ok(listResult.stdout.includes('testKey'), 'list command shows set values');
+      t.pass('CLI list command works');
+
+      return Promise.resolve();
+    })
+    .then(() => {
+      if (srf) srf.disconnect();
+      t.end();
+    })
+    .catch((err) => {
+      console.log('CLI config test error:', err);
+      if (srf) srf.disconnect();
+      t.fail(`CLI config test failed: ${err.err ? err.err.message : err.message}`);
+      if (err.stderr) console.log('stderr:', err.stderr);
+      if (err.stdout) console.log('stdout:', err.stdout);
+      t.end();
+    });
+});
+
+// Cleanup test data
+test('cleanup feature server test data', async (t) => {
+  try {
+    // Clean up test data
+    const drainedServers = await retrieveSet(drainedSetName);
+    if (drainedServers && drainedServers.length > 0) {
+      for (const server of drainedServers) {
+        await removeFromSet(drainedSetName, server);
+      }
+    }
+
+    const activeServers = await retrieveSet(activeSetName);
+    if (activeServers && activeServers.length > 0) {
+      for (const server of ['192.168.1.10', '192.168.1.11', '192.168.1.12']) {
+        await removeFromSet(activeSetName, server);
+      }
+    }
+    
+    t.pass('test data cleanup complete');
+    t.end();
+  } catch (err) {
+    t.fail(`Failed to cleanup test data: ${err.message}`);
+    t.end();
+  }
+});
+
+test('IP validation tests', (t) => {
+  // Valid IPv4
+  t.ok(isValidIP('192.168.1.1'), 'valid IPv4 passes');
+  t.ok(isValidIP('10.0.0.1'), 'valid private IPv4 passes');
+  t.ok(isValidIP('255.255.255.255'), 'max IPv4 passes');
+
+  // Invalid IPv4
+  t.notOk(isValidIP('256.1.1.1'), 'invalid IPv4 fails');
+  t.notOk(isValidIP('192.168.1'), 'incomplete IPv4 fails');
+  t.notOk(isValidIP(''), 'empty string fails');
+  t.notOk(isValidIP(null), 'null fails');
+  t.notOk(isValidIP(undefined), 'undefined fails');
+  t.notOk(isValidIP('not-an-ip'), 'random string fails');
+
+  // Valid IPv6
+  t.ok(isValidIP('::1'), 'IPv6 loopback passes');
+  t.ok(isValidIP('2001:0db8:85a3:0000:0000:8a2e:0370:7334'), 'full IPv6 passes');
+
+  t.end();
+});
+
+test('Feature Server Config utility functions', (t) => {
+  clearModule.all();
+  const { srf } = require('../app');
+  t.timeoutAfter(30000);
+
+  connect(srf)
+    .then(() => wait(1000)) // Wait for CLI and Redis initialization
+    .then(async () => {
+      // Set up test data first
+      await addToSet(activeSetName, '192.168.1.10');
+      await addToSet(activeSetName, '192.168.1.11');
+      await addToSet(activeSetName, '192.168.1.12');
+      await addToSet(drainedSetName, '192.168.1.11');
+
+      const {
+        isDrained,
+        getDrainedServers,
+        getActiveServers,
+        getAvailableServers,
+        getAllServersWithStatus
+      } = require('../lib/cli/feature-server-config');
+
+      // Test initial state
+      let drainedList = await getDrainedServers();
+      t.ok(Array.isArray(drainedList), 'getDrainedServers returns array');
+      t.ok(drainedList.includes('192.168.1.11'), 'initially contains test drained server');
+      t.pass('getDrainedServers works correctly');
+
+      let activeList = await getActiveServers();
+      t.ok(Array.isArray(activeList), 'getActiveServers returns array');
+      t.ok(activeList.includes('192.168.1.10'), 'contains test active server');
+      t.ok(activeList.includes('192.168.1.12'), 'contains test active server');
+      t.pass('getActiveServers works correctly');
+
+      // Test isDrained function
+      let drained1 = await isDrained('192.168.1.11');
+      let drained2 = await isDrained('192.168.1.10');
+      let drainedInvalid = await isDrained('invalid-ip');
+      t.ok(drained1, 'isDrained correctly identifies drained server');
+      t.notOk(drained2, 'isDrained correctly identifies non-drained server');
+      t.notOk(drainedInvalid, 'isDrained returns false for invalid IP');
+      t.pass('isDrained works correctly');
+
+      // Test getAvailableServers (active servers that are not drained)
+      let availableList = await getAvailableServers();
+      t.ok(Array.isArray(availableList), 'getAvailableServers returns array');
+      t.ok(availableList.includes('192.168.1.10'), 'includes active non-drained server');
+      t.ok(availableList.includes('192.168.1.12'), 'includes active non-drained server');
+      t.notOk(availableList.includes('192.168.1.11'), 'excludes drained server');
+      t.pass('getAvailableServers works correctly');
+
+      // Test getAllServersWithStatus
+      let statusResult = await getAllServersWithStatus();
+      t.ok(statusResult && typeof statusResult === 'object', 'getAllServersWithStatus returns object');
+      t.ok(Array.isArray(statusResult.servers), 'result has servers array');
+      t.ok(Array.isArray(statusResult.drained), 'result has drained array');
+      
+      const server10 = statusResult.servers.find(s => s.server === '192.168.1.10');
+      const server11 = statusResult.servers.find(s => s.server === '192.168.1.11');
+      const server12 = statusResult.servers.find(s => s.server === '192.168.1.12');
+      
+      t.ok(server10 && server10.status === 'active', 'server 10 has active status');
+      t.ok(server11 && server11.status === 'drained', 'server 11 has drained status');
+      t.ok(server12 && server12.status === 'active', 'server 12 has active status');
+      t.pass('getAllServersWithStatus works correctly');
+
+      // Now test after draining a server via CLI
+      await runCli('fs drain 192.168.1.10');
+
+      // Re-test functions after draining
+      drainedList = await getDrainedServers();
+      t.ok(drainedList.includes('192.168.1.10'), 'getDrainedServers includes newly drained server');
+      t.ok(drainedList.includes('192.168.1.11'), 'getDrainedServers still includes previously drained server');
+      
+      drained1 = await isDrained('192.168.1.10');
+      drained2 = await isDrained('192.168.1.12');
+      t.ok(drained1, 'isDrained correctly identifies newly drained server');
+      t.notOk(drained2, 'isDrained correctly identifies still active server');
+
+      availableList = await getAvailableServers();
+      t.notOk(availableList.includes('192.168.1.10'), 'getAvailableServers excludes newly drained server');
+      t.notOk(availableList.includes('192.168.1.11'), 'getAvailableServers excludes previously drained server');
+      t.ok(availableList.includes('192.168.1.12'), 'getAvailableServers includes remaining active server');
+
+      statusResult = await getAllServersWithStatus();
+      const updatedServer10 = statusResult.servers.find(s => s.server === '192.168.1.10');
+      const updatedServer12 = statusResult.servers.find(s => s.server === '192.168.1.12');
+      t.ok(updatedServer10 && updatedServer10.status === 'drained', 'server 10 now has drained status');
+      t.ok(updatedServer12 && updatedServer12.status === 'active', 'server 12 still has active status');
+
+      // Test undraining
+      await runCli('fs undrain 192.168.1.10');
+
+      drained1 = await isDrained('192.168.1.10');
+      t.notOk(drained1, 'isDrained correctly identifies undrained server');
+
+      availableList = await getAvailableServers();
+      t.ok(availableList.includes('192.168.1.10'), 'getAvailableServers includes undrained server');
+
+      // Clean up test data
+      await removeFromSet(drainedSetName, '192.168.1.10');
+      await removeFromSet(drainedSetName, '192.168.1.11');
+      await removeFromSet(activeSetName, '192.168.1.10');
+      await removeFromSet(activeSetName, '192.168.1.11');
+      await removeFromSet(activeSetName, '192.168.1.12');
+
+      t.pass('All feature server config utility functions tested successfully');
+      return Promise.resolve();
+    })
+    .then(() => {
+      if (srf) srf.disconnect();
+      t.end();
+    })
+    .catch((err) => {
+      console.log('Feature server config test error:', err);
+      if (srf) srf.disconnect();
+      t.fail(`Feature server config test failed: ${err.message}`);
+      t.end();
+    });
+});
+
+test('Feature Server Config edge cases and error handling', (t) => {
+  clearModule.all();
+  
+  // Test functions when runtime config is not initialized (Redis not available)
+  const {
+    isDrained,
+    getDrainedServers,
+    getActiveServers,
+    getAvailableServers,
+    getAllServersWithStatus
+  } = require('../lib/cli/feature-server-config');
+
+  // These should return safe defaults when Redis is not available
+  Promise.all([
+    isDrained('192.168.1.1'),
+    getDrainedServers(),
+    getActiveServers(),
+    getAvailableServers(),
+    getAllServersWithStatus()
+  ]).then(([
+    drainedResult,
+    drainedList,
+    activeList,
+    availableList,
+    statusResult
+  ]) => {
+    // Should return safe defaults when Redis is not available
+    t.equal(drainedResult, false, 'isDrained returns false when Redis unavailable');
+    t.ok(Array.isArray(drainedList) && drainedList.length === 0, 'getDrainedServers returns empty array when Redis unavailable');
+    t.ok(Array.isArray(activeList) && activeList.length === 0, 'getActiveServers returns empty array when Redis unavailable');
+    t.ok(Array.isArray(availableList) && availableList.length === 0, 'getAvailableServers returns empty array when Redis unavailable');
+    t.ok(statusResult && Array.isArray(statusResult.servers) && statusResult.servers.length === 0, 'getAllServersWithStatus returns empty result when Redis unavailable');
+    t.ok(statusResult && Array.isArray(statusResult.drained) && statusResult.drained.length === 0, 'getAllServersWithStatus drained array is empty when Redis unavailable');
+
+    t.pass('All edge cases handled gracefully');
+    t.end();
+  }).catch((err) => {
+    t.fail(`Edge case test failed: ${err.message}`);
+    t.end();
+  });
+});

test/index.js

@@ -4,5 +4,6 @@ require('./regbot-tests');
 require('./regbot-unit-test');
 require('./sip-register-tests');
 require('./sip-options-tests');
+require('./cli-tests');
 require('./docker_stop');
 require('./utils');