Changes for retrying in case of network issue

Author: SagarGaniga

bin/commands/runs.js

@@ -8,8 +8,7 @@ const archiver = require("../helpers/archiver"),
   Constants = require("../helpers/constants"),
   utils = require("../helpers/utils"),
   fileHelpers = require("../helpers/fileHelpers"),
-  syncRunner = require("../helpers/syncRunner"),
-  syncCliLogger = require("../helpers/logger").syncCliLogger;
+  syncRunner = require("../helpers/syncRunner");
 
 module.exports = function run(args) {
   let bsConfigPath = utils.getConfigPath(args.cf);
@@ -73,9 +72,7 @@ module.exports = function run(args) {
             if (args.sync) {
               syncRunner.pollBuildStatus(bsConfig, data).then((exitCode) => {
                 utils.sendUsageReport(bsConfig, args, `${message}\n${dashboardLink}`, Constants.messageTypes.SUCCESS, null);
-                syncCliLogger.info(Constants.userMessages.BUILD_REPORT_MESSAGE);
-                syncCliLogger.info(data.dashboard_url);
-                process.exit(exitCode);
+                utils.handleSyncExit(exitCode, data.dashboard_url)
               });
             }
 

bin/helpers/config.js

@@ -16,5 +16,7 @@ config.cypress_v1 = `${config.rails_host}/automate/cypress/v1`;
 config.buildUrl = `${config.cypress_v1}/builds/`;
 config.buildStopUrl = `${config.cypress_v1}/builds/stop/`;
 config.fileName = "tests.zip";
+config.retries = 5;
+config.networkErrorExitCode = 2;
 
 module.exports = config;

bin/helpers/sync/failedSpecsDetails.js

@@ -19,7 +19,8 @@ const tablePrinter = require('table'), // { table, getBorderCharacters }
 //
 let failedSpecsDetails = (data) => {
   return new Promise((resolve, reject) => {
-    data.exitCode = 0;
+    if (!data.exitCode) data.exitCode = 0;
+
     if (data.specs.length === 0) resolve(data); // return if no failed/skipped tests.
 
     let failedSpecs = false;
@@ -72,7 +73,7 @@ let failedSpecsDetails = (data) => {
     logger.info('\nFailed / skipped test report:');
     logger.info(result);
 
-    if (failedSpecs) data.exitCode = 1 ; // specs failed, send exitCode as 1
+    if (failedSpecs && data.exitCode !== 2) data.exitCode = 1 ; // specs failed, send exitCode as 1
     resolve(data); // No Specs failed, maybe skipped, but not failed, send exitCode as 0
   });
 }

bin/helpers/sync/syncSpecsLogs.js

@@ -8,7 +8,7 @@ const request = require("request"),
   tableStream = require('table').createStream,
   chalk = require('chalk');
 
-let whileLoop = true, whileTries = 40, options, timeout = 3000, n = 10, tableConfig, stream, startTime, endTime;
+let whileLoop = true, whileTries = config.retries, options, timeout = 3000, n = 10, tableConfig, stream, endTime, startTime = Date.now();
 let specSummary = {
   "specs": [],
   "duration": null
@@ -22,6 +22,7 @@ let  getOptions = (auth, build_id) => {
       user: auth.username,
       password: auth.access_key
     },
+    timeout: 10000,
     headers: {
       "Content-Type": "application/json",
       "User-Agent": utils.getUserAgent()
@@ -80,6 +81,7 @@ let printSpecsStatus = (bsConfig, buildDetails) => {
         whileProcess(callback)
       },
       function(err, result) { // when loop ends
+        logger.info("\n--------------------------------------------------------------------------------")
         specSummary.duration =  endTime - startTime
         resolve(specSummary)
       }
@@ -90,8 +92,24 @@ let printSpecsStatus = (bsConfig, buildDetails) => {
 let whileProcess = (whilstCallback) => {
   request.post(options, function(error, response, body) {
     if (error) {
-      return whilstCallback(error);
+      if (error.code === "ETIMEDOUT") {
+        whileTries -= 1;
+        if (whileTries === 0) {
+          whileLoop = false;
+          endTime = Date.now();
+          specSummary.exitCode = config.networkErrorExitCode;
+          return whilstCallback({ status: 504, message: "Tries limit reached" }); //Gateway Timeout
+        } else {
+          n = 2
+          return setTimeout(whilstCallback, timeout * n, null);
+        }
+      } else {
+        return whilstCallback(error);
+      }
     }
+
+    whileTries = config.retries; // reset to default after every successful request
+
     switch (response.statusCode) {
       case 202: // get data here and print it
         n = 2
@@ -104,7 +122,6 @@ let whileProcess = (whilstCallback) => {
         whileLoop = false;
         endTime = Date.now();
         showSpecsStatus(body);
-        logger.info("\n--------------------------------------------------------------------------------")
         return whilstCallback(null, body);
       default:
         whileLoop = false;
@@ -125,7 +142,6 @@ let showSpecsStatus = (data) => {
 }
 
 let printInitialLog = () => {
-  startTime = Date.now();
   logger.info(`\n${Constants.syncCLI.LOGS.INIT_LOG}`)
   logger.info("--------------------------------------------------------------------------------")
   n = Constants.syncCLI.INITIAL_DELAY_MULTIPLIER

bin/helpers/utils.js

@@ -5,7 +5,10 @@ const fs = require("fs");
 
 const usageReporting = require("./usageReporting"),
   logger = require("./logger").winstonLogger,
-  Constants = require("./constants");
+  Constants = require("./constants"),
+  chalk = require('chalk'),
+  syncCliLogger = require("../helpers/logger").syncCliLogger,
+  config = require("../helpers/config");
 
 exports.validateBstackJson = (bsConfigPath) => {
   return new Promise(function (resolve, reject) {
@@ -318,3 +321,20 @@ exports.setLocalIdentifier = (bsConfig) => {
 exports.capitalizeFirstLetter = (stringToCapitalize) => {
   return stringToCapitalize && (stringToCapitalize[0].toUpperCase() + stringToCapitalize.slice(1));
 };
+
+exports.handleSyncExit = (exitCode, dashboard_url) => {
+  if (exitCode === config.networkErrorExitCode) {
+    syncCliLogger.info(this.getNetworkErrorMessage(dashboard_url));
+  } else {
+    syncCliLogger.info(Constants.userMessages.BUILD_REPORT_MESSAGE);
+    syncCliLogger.info(dashboard_url);
+  }
+  process.exit(exitCode);
+}
+
+exports.getNetworkErrorMessage = (dashboard_url) => {
+  let message  = `fatal: unable to access '${config.buildUrl}': Could not resolve host: ${config.rails_host}` + '\n'
+                  + `Max retries exceeded trying to connect to the host (retries: ${config.retries})` + '\n'
+                  + `Please check the build status at: ${dashboard_url}`
+  return chalk.red(message)
+}

test/unit/bin/helpers/sync/syncSpecsLogs.js

@@ -96,7 +96,7 @@ describe("syncSpecsLogs", () => {
       expect(options.columns[1].alignment).to.equal('center');
       expect(options.columns[2].alignment).to.equal('left');
       expect(options.columns[1].width).to.equal(1);
-      expect(options.columns[2].width).to.equal(30);
+      expect(options.columns[2].width).to.equal(50);
       expect(options.columnCount).to.equal(3);
       expect(getBorderConfigStub.calledOnce).to.be.true;
     });
@@ -259,7 +259,60 @@ describe("syncSpecsLogs", () => {
   context("whileProcess", () => {
     const whileProcess = syncSpecsLogs.__get__("whileProcess");
 
-    it('Should break the loop if request has error', () => {
+    context('network issue', () => {
+      it('Should retry when error is because of network issue', () => {
+        let delayed_n = 2, timeout = 3000, n = 1;
+        let error = new Error("error");
+        error.code = "ETIMEDOUT";
+
+        let requestStub = sandbox.stub();
+
+        let postStub = sandbox
+          .stub(request, "post")
+          .yields(error, { statusCode: 502 }, JSON.stringify({}));
+
+        requestStub.post = postStub;
+
+        let setTimeout = sandbox.stub();
+        syncSpecsLogs.__set__('setTimeout', setTimeout);
+        syncSpecsLogs.__set__('n', n);
+        syncSpecsLogs.__set__('timeout', timeout);
+        syncSpecsLogs.__set__('request', requestStub);
+        syncSpecsLogs.__set__('whileTries', 5);
+
+        let whilstCallback = sandbox.stub();
+        whileProcess(whilstCallback);
+
+        sinon.assert.calledWith(setTimeout, whilstCallback, timeout * delayed_n, null);
+        expect(syncSpecsLogs.__get__("whileTries")).to.equal(4);
+      });
+
+      it('Should give-up on retry when error is b because of network issue after 5 retries and set proper exit code', () => {
+        let error = new Error("error"), requestStub = sandbox.stub();
+        error.code = "ETIMEDOUT";
+
+        let postStub = sandbox
+          .stub(request, "post")
+          .yields(error, { statusCode: 502 }, JSON.stringify({}));
+
+        requestStub.post = postStub;
+
+        syncSpecsLogs.__set__('request', requestStub);
+        syncSpecsLogs.__set__('whileTries', 1);
+        syncSpecsLogs.__set__('specSummary', {});
+        syncSpecsLogs.__set__('whileLoop', true);
+
+        let whilstCallback = sandbox.stub();
+        whileProcess(whilstCallback);
+
+        sinon.assert.calledWith(whilstCallback, { status: 504, message: "Tries limit reached" });
+        expect(syncSpecsLogs.__get__("whileTries")).to.equal(0);
+        expect(syncSpecsLogs.__get__("whileLoop")).to.equal(false);
+        expect(syncSpecsLogs.__get__("specSummary.exitCode")).to.equal(2);
+      });
+    });
+
+    it('Should break the loop if request has error other than network issue', () => {
       let error = new Error("error");
       let requestStub = sandbox.stub();
       let postStub = sandbox