[AZI-529] improve log forwarder retry behavior (#596)

* [AZI-529] add retry with delay

* [AZI-529] skip retrying 4xx responses

* [AZI-529] fix status code retry

* [AZI_529] change retry timeout

Author: parsons90

azure/activity_logs_monitoring/index.js

@@ -5,7 +5,7 @@
 
 var https = require('https');
 
-const VERSION = '0.5.6';
+const VERSION = '0.5.7';
 
 const STRING = 'string'; // example: 'some message'
 const STRING_ARRAY = 'string-array'; // example: ['one message', 'two message', ...]
@@ -29,6 +29,9 @@ const DD_SERVICE = process.env.DD_SERVICE || 'azure';
 const DD_SOURCE = process.env.DD_SOURCE || 'azure';
 const DD_SOURCE_CATEGORY = process.env.DD_SOURCE_CATEGORY || 'azure';
 
+const MAX_RETRIES = 4; // max number of times to retry a single http request
+const RETRY_INTERVAL = 250; // amount of time (milliseconds) to wait before retrying request, doubles after every retry
+
 /*
 To scrub PII from your logs, uncomment the applicable configs below. If you'd like to scrub more than just
 emails and IP addresses, add your own config to this map in the format
@@ -155,52 +158,70 @@ class HTTPClient {
         var batches = this.batcher.batch(records);
         var promises = [];
         for (var i = 0; i < batches.length; i++) {
-            promises.push(this.sendWithRetry(batches[i]));
+            promises.push(this.send(batches[i]));
         }
         return await Promise.all(
             promises.map(p => p.catch(e => this.context.log.error(e)))
         );
     }
 
-    sendWithRetry(record) {
-        return new Promise((resolve, reject) => {
-            return this.send(record)
-                .then(res => {
-                    resolve(true);
-                })
-                .catch(err => {
-                    this.send(record)
-                        .then(res => {
-                            resolve(true);
-                        })
-                        .catch(err => {
-                            reject(
-                                `unable to send request after 2 tries, err: ${err}`
-                            );
-                        });
-                });
-        });
+    isStatusCodeValid(statusCode) {
+        return statusCode >= 200 && statusCode <= 299;
+    }
+
+    shouldStatusCodeRetry(statusCode) {
+        // don't retry 4xx responses
+        return (
+            !this.isStatusCodeValid(statusCode) &&
+            (statusCode < 400 || statusCode > 499)
+        );
     }
 
     send(record) {
+        var numRetries = MAX_RETRIES;
+        var retryInterval = RETRY_INTERVAL;
         return new Promise((resolve, reject) => {
-            const req = https
-                .request(this.httpOptions, resp => {
-                    if (resp.statusCode < 200 || resp.statusCode > 299) {
-                        reject(`invalid status code ${resp.statusCode}`);
-                    } else {
-                        resolve(true);
+            const sendRequest = (options, record) => {
+                const retryRequest = errMsg => {
+                    if (numRetries === 0) {
+                        return reject(errMsg);
                     }
-                })
-                .on('error', error => {
-                    reject(error);
-                });
-            req.on('timeout', () => {
-                req.destroy();
-                reject(`request timed out after ${DD_REQUEST_TIMEOUT_MS}ms`);
-            })
-            req.write(this.scrubber.scrub(JSON.stringify(record)));
-            req.end();
+                    this.context.log.warn(
+                        `Unable to send request, with error: ${errMsg}. Retrying ${numRetries} more times`
+                    );
+                    numRetries--;
+                    retryInterval *= 2;
+                    setTimeout(() => {
+                        sendRequest(options, record);
+                    }, retryInterval);
+                };
+                const req = https
+                    .request(options, resp => {
+                        if (this.isStatusCodeValid(resp.statusCode)) {
+                            resolve(true);
+                        } else if (
+                            this.shouldStatusCodeRetry(resp.statusCode)
+                        ) {
+                            retryRequest(
+                                `invalid status code ${resp.statusCode}`
+                            );
+                        } else {
+                            reject(`invalid status code ${resp.statusCode}`);
+                        }
+                    })
+                    .on('error', error => {
+                        retryRequest(error.message);
+                    })
+                    .on('timeout', () => {
+                        req.destroy();
+                        retryRequest(
+                            `request timed out after ${DD_REQUEST_TIMEOUT_MS}ms`
+                        );
+                    });
+                req.write(this.scrubber.scrub(JSON.stringify(record)));
+                req.end();
+            };
+            sendRequest(this.httpOptions, record);
         });
     }
 }
@@ -219,9 +240,7 @@ class Scrubber {
                 );
             } catch {
                 context.log.error(
-                    `Regexp for rule ${name} pattern ${
-                        settings['pattern']
-                    } is malformed, skipping. Please update the pattern for this rule to be applied.`
+                    `Regexp for rule ${name} pattern ${settings['pattern']} is malformed, skipping. Please update the pattern for this rule to be applied.`
                 );
             }
         }