add retry for http/ws requestor (#1210)

* add retry for http requestor

* fix failing testcase

* wip

* update ws-requestor

* wip

* wip

* wip

lib/utils/http-requestor.js

@@ -43,6 +43,7 @@ class HttpRequestor extends BaseRequestor {
 
     this.method = hook.method || 'POST';
     this.authHeader = basicAuth(hook.username, hook.password);
+    this.backoffMs = 500;
 
     assert(this._isAbsoluteUrl(this.url));
     assert(['GET', 'POST'].includes(this.method));
@@ -136,25 +137,46 @@ class HttpRequestor extends BaseRequestor {
 
     let newClient;
     try {
+      this.backoffMs = 500;
+      // Parse URL and extract hash parameters for retry configuration
+      // Prepare request options - only do this once
+      const absUrl = this._isRelativeUrl(url) ? `${this.baseUrl}${url}` : url;
+      const parsedUrl = parseUrl(absUrl);
+      const hash = parsedUrl.hash || '';
+      const hashObj = hash ? this._parseHashParams(hash) : {};
+
+      // Retry policy: rp valid values: 4xx, 5xx, ct, rt, all, default is ct
+      // Retry count: rc valid values: 1-5, default is 0
+      // rc is the number of attempts we'll make AFTER the initial try
+      const rc = hash ? Math.min(Math.abs(parseInt(hashObj.rc || '0')), 5) : 0;
+      const rp = hashObj.rp || 'ct';
+      const rpValues = rp.split(',').map((v) => v.trim());
+      let retryCount = 0;
+
+      // Set up client, path and query parameters - only do this once
       let client, path, query;
       if (this._isRelativeUrl(url)) {
         client = this.client;
         path = url;
       }
       else {
-        const u = parseUrl(url);
-        if (u.resource === this._resource && u.port === this._port && u.protocol === this._protocol) {
+        if (parsedUrl.resource === this._resource &&
+          parsedUrl.port === this._port &&
+          parsedUrl.protocol === this._protocol) {
           client = this.client;
-          path = u.pathname;
-          query = u.query;
+          path = parsedUrl.pathname;
+          query = parsedUrl.query;
         }
         else {
-          if (u.port) client = newClient = new Client(`${u.protocol}://${u.resource}:${u.port}`);
-          else client = newClient = new Client(`${u.protocol}://${u.resource}`);
-          path = u.pathname;
-          query = u.query;
+          if (parsedUrl.port) {
+            client = newClient = new Client(`${parsedUrl.protocol}://${parsedUrl.resource}:${parsedUrl.port}`);
+          }
+          else client = newClient = new Client(`${parsedUrl.protocol}://${parsedUrl.resource}`);
+          path = parsedUrl.pathname;
+          query = parsedUrl.query;
         }
       }
+
       const sigHeader = this._generateSigHeader(payload, this.secret);
       const hdrs = {
         ...sigHeader,
@@ -162,20 +184,8 @@ class HttpRequestor extends BaseRequestor {
         ...httpHeaders,
         ...('POST' === method && {'Content-Type': 'application/json'})
       };
-      const absUrl = this._isRelativeUrl(url) ? `${this.baseUrl}${url}` : url;
-      this.logger.debug({url, absUrl, hdrs}, 'send webhook');
-      const {statusCode, headers, body} =  HTTP_PROXY_IP ? await request(
-        this.baseUrl,
-        {
-          path,
-          query,
-          method,
-          headers: hdrs,
-          ...('POST' === method && {body: JSON.stringify(payload)}),
-          timeout: HTTP_TIMEOUT,
-          followRedirects: false
-        }
-      ) : await client.request({
+
+      const requestOptions = {
         path,
         query,
         method,
@@ -183,14 +193,51 @@ class HttpRequestor extends BaseRequestor {
         ...('POST' === method && {body: JSON.stringify(payload)}),
         timeout: HTTP_TIMEOUT,
         followRedirects: false
-      });
-      if (![200, 202, 204].includes(statusCode)) {
-        const err = new HTTPResponseError(statusCode);
-        throw err;
-      }
-      if (headers['content-type']?.includes('application/json')) {
-        buf = await body.json();
+      };
+
+      // Simplified makeRequest function that just executes the HTTP request
+      const makeRequest = async() => {
+        this.logger.debug({url, absUrl, hdrs, retryCount},
+          `send webhook${retryCount > 0 ? ' (retry ' + retryCount + ')' : ''}`);
+
+        const {statusCode, headers, body} = HTTP_PROXY_IP ? await request(
+          this.baseUrl,
+          requestOptions
+        ) : await client.request(requestOptions);
+
+        if (![200, 202, 204].includes(statusCode)) {
+          const err = new HTTPResponseError(statusCode);
+          throw err;
+        }
+
+        if (headers['content-type']?.includes('application/json')) {
+          return await body.json();
+        }
+        return '';
+      };
+
+      while (true) {
+        try {
+          buf = await makeRequest();
+          break; // Success, exit the retry loop
+        } catch (err) {
+          retryCount++;
+
+          // Check if we should retry
+          if (retryCount <= rc && this._shouldRetry(err, rpValues)) {
+            this.logger.info(
+              {err, baseUrl: this.baseUrl, url, retryCount, maxRetries: rc},
+              `Retrying request (${retryCount}/${rc})`
+            );
+            const delay = this.backoffMs;
+            this.backoffMs = this.backoffMs < 2000 ? this.backoffMs * 2 : (this.backoffMs + 2000);
+            await new Promise((resolve) => setTimeout(resolve, delay));
+            continue;
+          }
+          throw err;
+        }
       }
+
       if (newClient) newClient.close();
     } catch (err) {
       if (err.statusCode) {
@@ -221,8 +268,8 @@ class HttpRequestor extends BaseRequestor {
 
     if (buf && (Array.isArray(buf) || type == 'llm:tool-call')) {
       this.logger.info({response: buf}, `HttpRequestor:request ${method} ${url} succeeded in ${rtt}ms`);
-      return buf;
     }
+    return buf;
   }
 }