ben.carrasco/jambonz-feature-server
1
0
Fork 0
mirror of https://github.com/jambonz/jambonz-feature-server.git synced 2025-12-20 08:40:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

add retry for http/ws requestor (#1210)

Browse Source 
* add retry for http requestor

* fix failing testcase

* wip

* update ws-requestor

* wip

* wip

* wip
This commit is contained in:
Hoan Luu Huu
2025-05-29 21:17:49 +07:00
committed by GitHub
parent f20d3dba2f
commit 4386df993c
13 changed files with 1906 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
lib/utils/ws-requestor.js
View File

@@ -1,6 +1,7 @@
const assert = require('assert');
const BaseRequestor = require('./base-requestor');
const short = require('short-uuid');
const parseUrl = require('parse-url');
const {HookMsgTypes, WS_CLOSE_CODES} = require('./constants.json');
const Websocket = require('ws');
const snakeCaseKeys = require('./snakecase-keys');
@@ -41,6 +42,19 @@ class WsRequestor extends BaseRequestor {
assert(this._isAbsoluteUrl(this.url));
const parsedUrl = parseUrl(this.url);
const hash = parsedUrl.hash || '';
const hashObj = hash ? this._parseHashParams(hash) : {};
// remove hash
this.cleanUrl = hash ? this.url.replace(`#${hash}`, '') : this.url;
// Retry policy: rp valid values: 4xx, 5xx, ct, rt, all, default is ct
// Retry count: rc valid values: 1-5, default is 5 for websockets
this.maxReconnects = Math.min(Math.abs(parseInt(hashObj.rc) || MAX_RECONNECTS), 5);
this.retryPolicy = hashObj.rp || 'ct';
this.retryPolicyValues = this.retryPolicy.split(',').map((v) => v.trim());
this.on('socket-closed', this._onSocketClosed.bind(this));
}
@@ -111,16 +125,65 @@ class WsRequestor extends BaseRequestor {
}
this.connectInProgress = true;
this.logger.debug(`WsRequestor:request(${this.id}) - connecting since we do not have a connection for ${type}`);
if (this.connections >= MAX_RECONNECTS) {
return Promise.reject(`max attempts connecting to ${this.url}`);
}
try {
const startAt = process.hrtime();
await this._connect();
const rtt = this._roundTrip(startAt);
this.stats.histogram('app.hook.connect_time', rtt, ['hook_type:app']);
let retryCount = 0;
let lastError = null;
while (retryCount <= this.maxReconnects) {
try {
this.logger.error({retryCount, maxReconnects: this.maxReconnects},
'WsRequestor:request - attempting connection');
// Ensure clean state before each connection attempt
if (this.ws) {
this.ws.removeAllListeners();
this.ws = null;
}
this.logger.error({retryCount}, 'WsRequestor:request - calling _connect()');
const startAt = process.hrtime();
await this._connect();
const rtt = this._roundTrip(startAt);
this.stats.histogram('app.hook.connect_time', rtt, ['hook_type:app']);
this.logger.error({retryCount}, 'WsRequestor:request - connection successful, exiting retry loop');
lastError = null;
break;
} catch (error) {
lastError = error;
retryCount++;
this.logger.error({error: error.message, retryCount, maxReconnects: this.maxReconnects},
'WsRequestor:request - connection attempt failed');
if (retryCount <= this.maxReconnects &&
this.retryPolicyValues?.length &&
this._shouldRetry(error, this.retryPolicyValues)) {
this.logger.error(
{url, error, retryCount, maxRetries: this.maxReconnects},
`WsRequestor:request - connection failed, retrying (${retryCount}/${this.maxReconnects})`
);
const delay = this.backoffMs;
this.backoffMs = this.backoffMs < 2000 ? this.backoffMs * 2 : (this.backoffMs + 2000);
this.logger.error({delay}, 'WsRequestor:request - waiting before retry');
await new Promise((resolve) => setTimeout(resolve, delay));
this.logger.error('WsRequestor:request - retry delay complete, attempting retry');
continue;
}
this.logger.error({lastError: lastError.message, retryCount, maxReconnects: this.maxReconnects},
'WsRequestor:request - throwing last error');
throw lastError;
}
}
// If we exit the loop without success, throw the last error
if (lastError) {
throw lastError;
}
} catch (err) {
this.logger.info({url, err}, 'WsRequestor:request - failed connecting');
this.logger.info({url, err, retryPolicy: this.retryPolicy},
'WsRequestor:request - all connection attempts failed');
this.connectInProgress = false;
return Promise.reject(err);
}
@@ -301,17 +364,23 @@ class WsRequestor extends BaseRequestor {
};
if (this.username && this.password) opts = {...opts, auth: `${this.username}:${this.password}`};
// Clean up any existing connection event listeners to prevent interference between retry attempts
this.removeAllListeners('ready');
this.removeAllListeners('not-ready');
this
.once('ready', (ws) => {
this.logger.error({retryCount: 'unknown'}, 'WsRequestor:_connect - ready event fired, resolving Promise');
this.removeAllListeners('not-ready');
if (this.connections > 1) this.request('session:reconnect', this.url);
resolve();
})
.once('not-ready', (err) => {
this.logger.error({err: err.message}, 'WsRequestor:_connect - not-ready event fired, rejecting Promise');
this.removeAllListeners('ready');
reject(err);
});
const ws = new Websocket(this.url, ['ws.jambonz.org'], opts);
const ws = new Websocket(this.cleanUrl, ['ws.jambonz.org'], opts);
this._setHandlers(ws);
});
}
@@ -335,10 +404,13 @@ class WsRequestor extends BaseRequestor {
}
_onError(err) {
if (this.connections > 0) {
this.logger.info({url: this.url, err}, 'WsRequestor:_onError');
if (this.connectInProgress) {
this.logger.info({url: this.url, err}, 'WsRequestor:_onError - emitting not-ready for connection attempt');
this.emit('not-ready', err);
}
else if (this.connections === 0) {
this.emit('not-ready', err);
}
else this.emit('not-ready', err);
}
_onOpen(ws) {
@@ -375,30 +447,44 @@ class WsRequestor extends BaseRequestor {
statusMessage: res.statusMessage
}, 'WsRequestor - unexpected response');
this.emit('connection-failure');
this.emit('not-ready', new Error(`${res.statusCode} ${res.statusMessage}`));
this.connections++;
const error = new Error(`${res.statusCode} ${res.statusMessage}`);
error.statusCode = res.statusCode;
this.connectInProgress = false;
this.emit('not-ready', error);
}
_onSocketClosed() {
this.ws = null;
this.emit('connection-dropped');
this._stopPingTimer();
if (this.connections > 0 && this.connections < MAX_RECONNECTS && !this.closedGracefully) {
if (this.connections > 0 && this.connections < this.maxReconnects && !this.closedGracefully) {
if (!this._initMsgId) this._clearPendingMessages();
this.logger.debug(`WsRequestor:_onSocketClosed waiting ${this.backoffMs} to reconnect`);
setTimeout(() => {
this._scheduleReconnect('_onSocketClosed');
}
}
_scheduleReconnect(source) {
this.logger.debug(`WsRequestor:_scheduleReconnect waiting ${this.backoffMs} to reconnect (${source})`);
setTimeout(() => {
this.logger.debug(
{haveWs: !!this.ws, connectInProgress: this.connectInProgress},
`WsRequestor:_scheduleReconnect time to reconnect (${source})`);
if (!this.ws && !this.connectInProgress) {
this.connectInProgress = true;
return this._connect()
.catch((err) => this.logger.error(`WsRequestor:${source} There is error while reconnect`, err))
.finally(() => this.connectInProgress = false);
} else {
this.logger.debug(
{haveWs: !!this.ws, connectInProgress: this.connectInProgress},
'WsRequestor:_onSocketClosed time to reconnect');
if (!this.ws && !this.connectInProgress) {
this.connectInProgress = true;
return this._connect()
.catch((err) => this.logger.error('WsRequestor:_onSocketClosed There is error while reconnect', err))
.finally(() => this.connectInProgress = false);
}
}, this.backoffMs);
this.backoffMs = this.backoffMs < 2000 ? this.backoffMs * 2 : (this.backoffMs + 2000);
}
`WsRequestor:_scheduleReconnect skipping reconnect attempt (${source}) - conditions not met`);
}
}, this.backoffMs);
this.backoffMs = this.backoffMs < 2000 ? this.backoffMs * 2 : (this.backoffMs + 2000);
}
_onMessage(content, isBinary) {