Feat/cli remove fs (#116)

* support draining feature server manually

* support draining feature server manually

* support CLI to add or remove feature server

* wip

* wip

* wip

* add redis key for feature server and integration test

* wip
This commit is contained in:
Hoan Luu Huu
2025-10-29 06:26:24 +07:00
committed by GitHub
parent 658cb6ae7d
commit 55ace7820f
9 changed files with 1249 additions and 0 deletions

239
lib/cli/cli.js Normal file
View File

@@ -0,0 +1,239 @@
#!/usr/bin/env node
const net = require('net');
const SOCKET_PATH = process.env.SBC_SOCKET_PATH || '/tmp/sbc-sip-sidecar.sock';
function connectAndSend(command) {
return new Promise((resolve, reject) => {
const socket = net.createConnection(SOCKET_PATH);
socket.on('connect', () => {
socket.write(JSON.stringify(command) + '\n');
});
socket.on('data', (data) => {
try {
const response = JSON.parse(data.toString().trim());
socket.end();
resolve(response);
} catch {
socket.end();
reject(new Error('Invalid response from server'));
}
});
socket.on('error', (err) => {
reject(new Error(`Connection failed: ${err.message}`));
});
socket.setTimeout(5000, () => {
socket.end();
reject(new Error('Command timed out'));
});
});
}
function exitWithError(message) {
console.error('Error:', message);
process.exit(1);
}
function validateArgs(args, minCount, usage) {
if (args.length < minCount) {
exitWithError(`${usage}`);
}
}
async function handleFeatureServerCommand(args) {
const [action, server] = args;
switch (action) {
case 'drain':
validateArgs(args, 2, 'drain requires server IP address');
const drainResult = await connectAndSend({ action: 'fs-drain', server });
if (!drainResult.success) {
exitWithError(drainResult.error || 'Failed to drain server');
}
console.log(`✓ Drained ${drainResult.server}`);
console.log(`Drained servers: [${drainResult.drained.join(', ')}]`);
break;
case 'undrain':
validateArgs(args, 2, 'undrain requires server IP address');
const undrainResult = await connectAndSend({ action: 'fs-undrain', server });
if (!undrainResult.success) {
exitWithError(undrainResult.error || 'Failed to undrain server');
}
console.log(`✓ Undrained ${undrainResult.server}`);
console.log(`Drained servers: [${undrainResult.drained.join(', ')}]`);
break;
case 'drained':
const drainedResult = await connectAndSend({ action: 'fs-drained' });
if (drainedResult.error) {
exitWithError(drainedResult.error);
}
if (drainedResult.drained.length === 0) {
console.log('No servers are currently drained');
} else {
console.log('Drained servers:');
drainedResult.drained.forEach((server) => console.log(` 🔴 ${server}`));
}
break;
case 'active':
const activeResult = await connectAndSend({ action: 'fs-available' });
if (!activeResult.success) {
exitWithError(activeResult.error || 'Failed to get active servers');
}
if (activeResult.available.length === 0) {
console.log('No active feature servers found');
} else {
console.log('Available feature servers:');
activeResult.available.forEach((server) => console.log(` 🟢 ${server}`));
}
break;
case 'list':
const listResult = await connectAndSend({ action: 'fs-list' });
if (listResult.error) {
exitWithError(listResult.error);
}
if (listResult.servers.length === 0) {
console.log('No servers configured');
if (listResult.drained.length > 0) {
console.log('Orphaned drained servers:');
listResult.drained.forEach((server) => console.log(` 🔴 ${server} (orphaned)`));
}
} else {
console.log('Feature servers:');
listResult.servers.forEach(({ server, status }) => {
const icon = status === 'drained' ? '🔴' : '🟢';
console.log(` ${icon} ${server} (${status})`);
});
}
break;
default:
exitWithError(`Unknown fs command: ${action}. Use: drain, undrain, drained, active, list`);
}
}
function showHelp() {
console.log('SBC Runtime CLI');
console.log('');
console.log('Usage:');
console.log(' npm run cli <command> [options]');
console.log('');
console.log('Feature Server Commands:');
console.log(' npm run cli fs drain <ip> Drain server (remove from pool)');
console.log(' npm run cli fs undrain <ip> Undrain server (add back to pool)');
console.log(' npm run cli fs drained Show drained servers');
console.log(' npm run cli fs active Show available servers');
console.log(' npm run cli fs list Show all servers with status');
console.log('');
console.log('Configuration Commands:');
console.log(' npm run cli set <key> <value> Set runtime config');
console.log(' npm run cli get <key> Get runtime config');
console.log(' npm run cli list Show all runtime config');
console.log('');
console.log('Examples:');
console.log(' npm run cli fs drain 192.168.1.10');
console.log(' npm run cli fs active');
}
async function handleConfigCommand(action, args) {
switch (action) {
case 'set':
validateArgs(args, 2, 'set requires key and value');
const [key, value] = args;
const setResult = await connectAndSend({ action: 'set', key, value });
if (setResult.error) {
exitWithError(setResult.error);
}
console.log(`${setResult.key} = ${JSON.stringify(setResult.value)}`);
break;
case 'get':
validateArgs(args, 1, 'get requires key');
const getResult = await connectAndSend({ action: 'get', key: args[0] });
if (getResult.error) {
exitWithError(getResult.error);
}
console.log(`${getResult.key} = ${JSON.stringify(getResult.value)}`);
break;
case 'list':
const listResult = await connectAndSend({ action: 'list' });
if (listResult.error) {
exitWithError(listResult.error);
}
console.log('Runtime Configuration:');
const entries = Object.entries(listResult.config);
if (entries.length === 0) {
console.log(' (no configuration set)');
} else {
entries.forEach(([key, value]) => {
console.log(` ${key} = ${JSON.stringify(value)}`);
});
}
break;
default:
exitWithError(`Unknown config command: ${action}. Use: set, get, list`);
}
}
async function main() {
const args = process.argv.slice(2);
if (args.length === 0) {
showHelp();
return;
}
const [command, ...remainingArgs] = args;
try {
switch (command) {
case 'fs':
validateArgs(remainingArgs, 1, 'fs command requires an action');
await handleFeatureServerCommand(remainingArgs);
break;
case 'set':
case 'get':
case 'list':
await handleConfigCommand(command, remainingArgs);
break;
default:
exitWithError(`Unknown command: ${command}. Run without arguments for help.`);
}
} catch (error) {
exitWithError(error.message);
}
}
if (require.main === module) {
main().catch((error) => {
exitWithError(error.message);
});
}

View File

@@ -0,0 +1,81 @@
const runtimeConfigModule = require('./runtime-config');
function isValidIP(ip) {
if (!ip || typeof ip !== 'string') return false;
// IPv4 check
const ipv4 = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
if (ipv4.test(ip)) return true;
// IPv6 check (basic patterns)
const ipv6 = /^(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$|^::1$|^::$/;
if (ipv6.test(ip)) return true;
// IPv6 compressed
const ipv6Short = new RegExp([
'^([0-9a-fA-F]{1,4}:)*::([0-9a-fA-F]{1,4}:)*[0-9a-fA-F]{1,4}$',
'^([0-9a-fA-F]{1,4}:)*::[0-9a-fA-F]{1,4}$',
'^[0-9a-fA-F]{1,4}::([0-9a-fA-F]{1,4}:)*[0-9a-fA-F]{1,4}$'
].join('|'));
if (ipv6Short.test(ip)) return true;
return false;
}
async function isDrained(serverIP) {
if (!isValidIP(serverIP)) {
return false;
}
try {
const runtimeConfig = runtimeConfigModule.getInstance();
return await runtimeConfig.isServerDrained(serverIP);
} catch {
return false;
}
}
async function getDrainedServers() {
try {
const runtimeConfig = runtimeConfigModule.getInstance();
return await runtimeConfig.getDrainedFeatureServers();
} catch {
return [];
}
}
async function getActiveServers() {
try {
const runtimeConfig = runtimeConfigModule.getInstance();
return await runtimeConfig.getActiveFeatureServers();
} catch {
return [];
}
}
async function getAvailableServers() {
try {
const runtimeConfig = runtimeConfigModule.getInstance();
return await runtimeConfig.getAvailableFeatureServers();
} catch {
return [];
}
}
async function getAllServersWithStatus() {
try {
const runtimeConfig = runtimeConfigModule.getInstance();
return await runtimeConfig.getAllFeatureServersWithStatus();
} catch {
return { servers: [], drained: [] };
}
}
module.exports = {
isValidIP,
isDrained,
getDrainedServers,
getActiveServers,
getAvailableServers,
getAllServersWithStatus
};

427
lib/cli/runtime-config.js Normal file
View File

@@ -0,0 +1,427 @@
const net = require('net');
const config = new Map();
const queue = [];
let processing = false;
let logger;
async function runOperation(operation) {
return new Promise((resolve, reject) => {
queue.push({ operation, resolve, reject });
processQueue();
});
}
async function processQueue() {
if (processing || queue.length === 0) return;
processing = true;
while (queue.length > 0) {
const { operation, resolve, reject } = queue.shift();
try {
const result = await operation();
resolve(result);
} catch (err) {
reject(err);
}
}
processing = false;
}
class RuntimeConfig {
constructor(srfLocals = null, appLogger = null) {
this.server = null;
this.socketPath = process.env.SBC_SOCKET_PATH || '/tmp/sbc-sip-sidecar.sock';
this.srfLocals = srfLocals;
if (appLogger) {
logger = appLogger;
} else if (!logger) {
throw new Error('Logger is required for RuntimeConfig');
}
this.startServer();
}
async set(key, value) {
return runOperation(() => {
config.set(key, value);
logger.info({ key, value }, 'Config updated');
return { key, value };
});
}
async get(key, defaultValue) {
return runOperation(() => {
return config.has(key) ? config.get(key) : defaultValue;
});
}
async addToArray(key, item) {
return runOperation(() => {
let arr = config.get(key) || [];
if (typeof arr === 'string') {
arr = arr.split(',').map((s) => s.trim()).filter((s) => s.length > 0);
}
if (!Array.isArray(arr)) arr = [];
const exists = arr.includes(item);
if (!exists) {
arr.push(item);
config.set(key, arr);
logger.info({ key, item, array: arr }, 'Added to array');
}
return { key, item, array: arr, added: !exists };
});
}
async removeFromArray(key, item) {
return runOperation(() => {
let arr = config.get(key) || [];
if (typeof arr === 'string') {
arr = arr.split(',').map((s) => s.trim()).filter((s) => s.length > 0);
}
if (!Array.isArray(arr)) arr = [];
const originalLength = arr.length;
arr = arr.filter((existing) => existing !== item);
if (arr.length !== originalLength) {
config.set(key, arr);
logger.info({ key, item, array: arr }, 'Removed from array');
}
return { key, item, array: arr, removed: arr.length !== originalLength };
});
}
async getAll() {
return runOperation(() => Object.fromEntries(config));
}
// Helper methods for cleaner code
checkRedisConnection(requiredMethod) {
return this.srfLocals && this.srfLocals[requiredMethod];
}
sendError(socket, message) {
socket.write(JSON.stringify({ success: false, error: message }) + '\n');
}
sendSuccess(socket, data) {
socket.write(JSON.stringify({ success: true, ...data }) + '\n');
}
getSetNames() {
const JAMBONES_CLUSTER_ID = process.env.JAMBONES_CLUSTER_ID;
return {
activeFs: `${(JAMBONES_CLUSTER_ID || 'default')}:active-fs`,
drainedFs: `${(JAMBONES_CLUSTER_ID || 'default')}:drained-fs`
};
}
// Feature server utility methods
async isServerDrained(serverIP) {
if (!this.checkRedisConnection('isMemberOfSet')) {
return false;
}
try {
const { drainedFs } = this.getSetNames();
return await this.srfLocals.isMemberOfSet(drainedFs, serverIP);
} catch {
return false;
}
}
async getDrainedFeatureServers() {
if (!this.checkRedisConnection('retrieveSet')) {
return [];
}
try {
const { drainedFs } = this.getSetNames();
const servers = await this.srfLocals.retrieveSet(drainedFs);
return servers || [];
} catch {
return [];
}
}
async getActiveFeatureServers() {
if (!this.checkRedisConnection('retrieveSet')) {
return [];
}
try {
const { activeFs } = this.getSetNames();
const servers = await this.srfLocals.retrieveSet(activeFs);
return servers || [];
} catch {
return [];
}
}
async getAvailableFeatureServers() {
try {
const [active, drained] = await Promise.all([
this.getActiveFeatureServers(),
this.getDrainedFeatureServers()
]);
const drainedSet = new Set(drained);
return active.filter((server) => !drainedSet.has(server));
} catch {
return [];
}
}
async getAllFeatureServersWithStatus() {
try {
const [active, drained] = await Promise.all([
this.getActiveFeatureServers(),
this.getDrainedFeatureServers()
]);
const drainedSet = new Set(drained);
const servers = active.map((server) => ({
server,
status: drainedSet.has(server) ? 'drained' : 'active'
}));
return { servers, drained };
} catch {
return { servers: [], drained: [] };
}
}
async handleFeatureServerDrain(socket, server) {
if (!this.checkRedisConnection('addToSet')) {
return this.sendError(socket, 'Redis connection not available');
}
const { isValidIP } = require('./feature-server-config');
if (!isValidIP(server)) {
return this.sendError(socket, `Invalid IP address: ${server}`);
}
try {
const { drainedFs } = this.getSetNames();
await this.srfLocals.addToSet(drainedFs, server);
const drainedServers = await this.srfLocals.retrieveSet(drainedFs);
this.sendSuccess(socket, {
action: 'drain',
server,
drained: drainedServers || []
});
} catch (err) {
logger.error({ err }, 'Error draining server');
this.sendError(socket, 'Failed to drain server');
}
}
async handleFeatureServerUndrain(socket, server) {
if (!this.checkRedisConnection('removeFromSet')) {
return this.sendError(socket, 'Redis connection not available');
}
const { isValidIP } = require('./feature-server-config');
if (!isValidIP(server)) {
return this.sendError(socket, `Invalid IP address: ${server}`);
}
try {
const { drainedFs } = this.getSetNames();
await this.srfLocals.removeFromSet(drainedFs, server);
const drainedServers = await this.srfLocals.retrieveSet(drainedFs);
this.sendSuccess(socket, {
action: 'undrain',
server,
drained: drainedServers || []
});
} catch (err) {
logger.error({ err }, 'Error undraining server');
this.sendError(socket, 'Failed to undrain server');
}
}
async handleFeatureServerDrained(socket) {
try {
const drainedServers = await this.getDrainedFeatureServers();
this.sendSuccess(socket, { drained: drainedServers });
} catch (err) {
logger.error({ err }, 'Error retrieving drained servers');
this.sendError(socket, 'Failed to retrieve drained servers');
}
}
async handleFeatureServerList(socket) {
try {
const result = await this.getAllFeatureServersWithStatus();
this.sendSuccess(socket, result);
} catch (err) {
logger.error({ err }, 'Error listing servers');
this.sendError(socket, 'Failed to list servers');
}
}
async handleFeatureServerAvailable(socket) {
try {
const availableServers = await this.getAvailableFeatureServers();
this.sendSuccess(socket, { available: availableServers });
} catch (err) {
logger.error({ err }, 'Error retrieving available servers');
this.sendError(socket, 'Failed to retrieve available servers');
}
}
startServer() {
try {
require('fs').unlinkSync(this.socketPath);
} catch {
// socket file doesn't exist, that's fine
}
this.server = net.createServer((socket) => {
let buffer = '';
socket.on('data', (data) => {
buffer += data.toString();
const lines = buffer.split('\n');
buffer = lines.pop();
for (const line of lines) {
if (line.trim()) {
this.handleCommand(socket, line.trim());
}
}
});
socket.on('error', (err) => {
logger.error({ err }, 'Socket error');
});
});
this.server.listen(this.socketPath, () => {
logger.info({ socketPath: this.socketPath }, 'CLI server started');
require('fs').chmodSync(this.socketPath, 0o600);
});
this.server.on('error', (err) => {
logger.error({ err }, 'Server error');
});
}
async handleCommand(socket, jsonString) {
try {
const cmd = JSON.parse(jsonString);
switch (cmd.action) {
case 'set':
const setResult = await this.set(cmd.key, cmd.value);
this.sendSuccess(socket, setResult);
break;
case 'get':
const value = await this.get(cmd.key);
this.sendSuccess(socket, { key: cmd.key, value });
break;
case 'add':
const addResult = await this.addToArray(cmd.key, cmd.item);
this.sendSuccess(socket, addResult);
break;
case 'remove':
const removeResult = await this.removeFromArray(cmd.key, cmd.item);
this.sendSuccess(socket, removeResult);
break;
case 'list':
const allConfig = await this.getAll();
this.sendSuccess(socket, { config: allConfig });
break;
case 'fs-drain':
await this.handleFeatureServerDrain(socket, cmd.server);
break;
case 'fs-undrain':
await this.handleFeatureServerUndrain(socket, cmd.server);
break;
case 'fs-drained':
await this.handleFeatureServerDrained(socket);
break;
case 'fs-list':
await this.handleFeatureServerList(socket);
break;
case 'fs-available':
await this.handleFeatureServerAvailable(socket);
break;
default:
this.sendError(socket, 'Unknown action');
}
} catch (err) {
logger.error({ err, command: jsonString }, 'Command error');
this.sendError(socket, 'Invalid command');
}
}
shutdown() {
if (this.server) {
this.server.close();
try {
require('fs').unlinkSync(this.socketPath);
} catch {
// ignore
}
}
}
}
let runtimeConfig = null;
function createInstance(srfLocals = null, appLogger = null) {
const instance = new RuntimeConfig(srfLocals, appLogger);
process.on('SIGINT', () => instance.shutdown());
process.on('SIGTERM', () => instance.shutdown());
return instance;
}
function initialize(srfLocals, appLogger) {
if (!appLogger) {
throw new Error('Logger is required for RuntimeConfig initialization');
}
if (!runtimeConfig) {
runtimeConfig = createInstance(srfLocals, appLogger);
} else {
// Update existing instance with srfLocals and logger
runtimeConfig.srfLocals = srfLocals;
logger = appLogger;
}
return runtimeConfig;
}
function getInstance() {
if (!runtimeConfig) {
throw new Error('RuntimeConfig not initialized. Call initialize() first with logger.');
}
return runtimeConfig;
}
module.exports = {
initialize,
getInstance
};

View File

@@ -1,4 +1,5 @@
const debug = require('debug')('jambonz:sbc-options-handler');
const { isDrained } = require('./cli/feature-server-config');
const {
EXPIRES_INTERVAL,
CHECK_EXPIRES_INTERVAL,
@@ -132,6 +133,15 @@ module.exports = ({srf, logger}) => {
status = req.get(`${prefix}-Status`);
// If feature server is drained, force status to closed
if (status === 'open' && !isRtpServer) {
const fsIP = req.source_address;
if (await isDrained(fsIP)) {
logger.warn({fsIP}, 'drained feature server attempted to check in - rejecting');
status = 'closed';
}
}
countOfMembers = await _addToCache(map, status, setName, key);
if (fsServiceUrlKey) {
await _addToCache(fsServiceUrls, status, setNameFsSeriveUrl, fsServiceUrlKey);