Merge pull request #161 from browserstack/Change_Binary_Download_Distribution

Change Binary Download Distribution

Author: yashdsaraf

lib/Local.js

@@ -1,6 +1,7 @@
 var childProcess = require('child_process'),
   os = require('os'),
   fs = require('fs'),
+  util = require('util'),
   path = require('path'),
   running = require('is-running'),
   LocalBinary = require('./LocalBinary'),
@@ -17,7 +18,7 @@ function Local(){
   this.windows = os.platform().match(/mswin|msys|mingw|cygwin|bccwin|wince|emc|win32/i);
   this.pid = undefined;
   this.isProcessRunning = false;
-  this.retriesLeft = 5;
+  this.retriesLeft = 9;
   this.key = process.env.BROWSERSTACK_ACCESS_KEY;
   this.logfile = this.sanitizePath(path.join(process.cwd(), 'local.log'));
   this.opcode = 'start';
@@ -59,12 +60,15 @@ function Local(){
         return;
       }
     }catch(error){
-      console.error('Error while trying to execute binary', error);
+      const binaryDownloadErrorMessage = `Error while trying to execute binary: ${util.format(error)}`;
+      console.error(binaryDownloadErrorMessage);
       if(that.retriesLeft > 0) {
         console.log('Retrying Binary Download. Retries Left', that.retriesLeft);
         that.retriesLeft -= 1;
         fs.unlinkSync(that.binaryPath);
         delete(that.binaryPath);
+        process.env.BINARY_DOWNLOAD_ERROR_MESSAGE = binaryDownloadErrorMessage;
+        process.env.BINARY_DOWNLOAD_FALLBACK_ENABLED = true;
         return that.startSync(options);
       } else {
         throw new LocalError(error.toString());
@@ -87,12 +91,15 @@ function Local(){
       that.opcode = 'start';
       that.tunnel = childProcess.execFile(that.binaryPath, that.getBinaryArgs(), function(error, stdout, stderr){
         if(error) {
-          console.error('Error while trying to execute binary', error);
+          const binaryDownloadErrorMessage = `Error while trying to execute binary: ${util.format(error)}`;
+          console.error(binaryDownloadErrorMessage);
           if(that.retriesLeft > 0) {
             console.log('Retrying Binary Download. Retries Left', that.retriesLeft);
             that.retriesLeft -= 1;
             fs.unlinkSync(that.binaryPath);
             delete(that.binaryPath);
+            process.env.BINARY_DOWNLOAD_ERROR_MESSAGE = binaryDownloadErrorMessage;
+            process.env.BINARY_DOWNLOAD_FALLBACK_ENABLED = true;
             that.start(options, callback);
             return;
           } else {
@@ -254,9 +261,9 @@ function Local(){
         conf.useCaCertificate = this.useCaCertificate;
       }
       if(!callback) {
-        return this.binary.binaryPath(conf);
+        return this.binary.binaryPath(conf, this.key, this.retriesLeft);
       }
-      this.binary.binaryPath(conf, callback);
+      this.binary.binaryPath(conf, this.key, this.retriesLeft, callback);
     } else {
       console.log('BINARY PATH IS DEFINED');
       if(!callback) {

lib/LocalBinary.js

@@ -3,6 +3,7 @@ var https = require('https'),
   fs = require('fs'),
   path = require('path'),
   os = require('os'),
+  util = require('util'),
   childProcess = require('child_process'),
   zlib = require('zlib'),
   HttpsProxyAgent = require('https-proxy-agent'),
@@ -14,9 +15,55 @@ const packageName = 'browserstack-local-nodejs';
 function LocalBinary(){
   this.hostOS = process.platform;
   this.is64bits = process.arch == 'x64';
+  this.baseRetries = 9;
+  this.sourceURL = null;
+  this.downloadErrorMessage = null;
+
+  this.getSourceUrl = function(conf, retries) {
+    /* Request for an endpoint to download the local binary from Rails no more than twice with 5 retries each */
+    if (![4, 9].includes(retries) && this.sourceURL != null) {
+      return this.sourceURL;
+    }
+
+    if (process.env.BINARY_DOWNLOAD_SOURCE_URL !== undefined && process.env.BINARY_DOWNLOAD_FALLBACK_ENABLED == 'true' && this.parentRetries != 4) {
+      /* This is triggered from Local.js if there's an error executing the downloaded binary */
+      return process.env.BINARY_DOWNLOAD_SOURCE_URL;
+    }
+
+    let cmd, opts;
+    cmd = 'node';
+    opts = [path.join(__dirname, 'fetchDownloadSourceUrl.js'), this.key];
+
+    if (retries == 4 || (process.env.BINARY_DOWNLOAD_FALLBACK_ENABLED == 'true' && this.parentRetries == 4)) {
+      opts.push(true, this.downloadErrorMessage || process.env.BINARY_DOWNLOAD_ERROR_MESSAGE);
+    } else {
+      opts.push(false, null);
+    }
+
+    if(conf.proxyHost && conf.proxyPort) {
+      opts.push(conf.proxyHost, conf.proxyPort);
+      if (conf.useCaCertificate) {
+        opts.push(conf.useCaCertificate);
+      }
+    } else if (conf.useCaCertificate) {
+      opts.push(undefined, undefined, conf.useCaCertificate);
+    }
+
+    const userAgent = [packageName, version].join('/');
+    const env = Object.assign({ 'USER_AGENT': userAgent }, process.env);
+    const obj = childProcess.spawnSync(cmd, opts, { env: env });
+    if(obj.stdout.length > 0) {
+      this.sourceURL = obj.stdout.toString().replace(/\n+$/, '');
+      process.env.BINARY_DOWNLOAD_SOURCE_URL = this.sourceURL;
+      return this.sourceURL;
+    } else if(obj.stderr.length > 0) {
+      let output = Buffer.from(JSON.parse(JSON.stringify(obj.stderr)).data).toString();
+      throw(output);
+    }
+  };
 
-  this.getDownloadPath = function () {
-    let sourceURL = 'https://www.browserstack.com/local-testing/downloads/binaries/';
+  this.getDownloadPath = function (conf, retries) {
+    let sourceURL = this.getSourceUrl(conf, retries) + '/';
 
     if(this.hostOS.match(/darwin|mac os/i)){
       return sourceURL + 'BrowserStackLocal-darwin-x64';
@@ -43,9 +90,10 @@ function LocalBinary(){
     }
   };
 
-  this.httpPath = this.getDownloadPath();
-
-
+  this.binaryDownloadError = function(errorMessagePrefix, errorMessage) {
+    console.error(errorMessagePrefix, errorMessage);
+    this.downloadErrorMessage = errorMessagePrefix + ' : ' + errorMessage;
+  };
 
   this.retryBinaryDownload = function(conf, destParentDir, callback, retries, binaryPath) {
     var that = this;
@@ -66,6 +114,12 @@ function LocalBinary(){
   };
 
   this.downloadSync = function(conf, destParentDir, retries) {
+    try {
+      this.httpPath = this.getDownloadPath(conf, retries);
+    } catch (e) {
+      return console.error(`Unable to fetch the source url to download the binary with error: ${e}`);
+    }
+
     console.log('Downloading in sync');
     var that = this;
     if(!this.checkPath(destParentDir))
@@ -96,21 +150,27 @@ function LocalBinary(){
           fs.chmodSync(binaryPath, '0755');
           return binaryPath;
         }else{
-          console.log('failed to download');
+          that.binaryDownloadError('failed to download');
           return that.retryBinaryDownload(conf, destParentDir, null, retries, binaryPath);
         }
       } else if(obj.stderr.length > 0) {
         output = Buffer.from(JSON.parse(JSON.stringify(obj.stderr)).data).toString();
-        console.error(output);
+        that.binaryDownloadError(output);
         return that.retryBinaryDownload(conf, destParentDir, null, retries, binaryPath);
       }
     } catch(err) {
-      console.error('Download failed with error', err);
+      that.binaryDownloadError('Download failed with error', util.format(err));
       return that.retryBinaryDownload(conf, destParentDir, null, retries, binaryPath);
     }
   };
 
   this.download = function(conf, destParentDir, callback, retries){
+    try {
+      this.httpPath = this.getDownloadPath(conf, retries);
+    } catch (e) {
+      return console.error(`Unable to fetch the source url to download the binary with error: ${e}`);
+    }
+
     var that = this;
     if(!this.checkPath(destParentDir))
       fs.mkdirSync(destParentDir);
@@ -152,11 +212,11 @@ function LocalBinary(){
       }
 
       response.on('error', function(err) {
-        console.error('Got Error in binary download response', err);
+        that.binaryDownloadError('Got Error in binary download response', util.format(err));
         that.retryBinaryDownload(conf, destParentDir, callback, retries, binaryPath);
       });
       fileStream.on('error', function (err) {
-        console.error('Got Error while downloading binary file', err);
+        that.binaryDownloadError('Got Error while downloading binary file', util.format(err));
         that.retryBinaryDownload(conf, destParentDir, callback, retries, binaryPath);
       });
       fileStream.on('close', function () {
@@ -165,12 +225,14 @@ function LocalBinary(){
         });
       });
     }).on('error', function(err) {
-      console.error('Got Error in binary downloading request', err);
+      that.binaryDownloadError('Got Error in binary downloading request', util.format(err));
       that.retryBinaryDownload(conf, destParentDir, callback, retries, binaryPath);
     });
   };
 
-  this.binaryPath = function(conf, callback){
+  this.binaryPath = function(conf, key, parentRetries, callback){
+    this.key = key;
+    this.parentRetries = parentRetries;
     var destParentDir = this.getAvailableDirs();
     var destBinaryName = (this.windows) ? 'BrowserStackLocal.exe' : 'BrowserStackLocal';
     var binaryPath = path.join(destParentDir, destBinaryName);
@@ -180,10 +242,11 @@ function LocalBinary(){
       }
       callback(binaryPath);
     } else {
+      let retries = this.baseRetries;
       if(!callback) {
-        return this.downloadSync(conf, destParentDir, 5);
+        return this.downloadSync(conf, destParentDir, retries);
       }
-      this.download(conf, destParentDir, callback, 5);
+      this.download(conf, destParentDir, callback, retries);
     }
   };
 

lib/fetchDownloadSourceUrl.js

@@ -0,0 +1,61 @@
+const https = require('https'),
+  fs = require('fs'),
+  HttpsProxyAgent = require('https-proxy-agent');
+
+const authToken = process.argv[2], proxyHost = process.argv[5], proxyPort = process.argv[6], useCaCertificate = process.argv[7], downloadFallback = process.argv[3], downloadErrorMessage = process.argv[4];
+
+let body = '', data = {'auth_token': authToken};
+const options = {
+  hostname: 'local.browserstack.com',
+  port: 443,
+  path: '/binary/api/v1/endpoint',
+  method: 'POST',
+  headers: {
+    'Content-Type': 'application/json',
+    'user-agent': process.env.USER_AGENT
+  }
+};
+if (downloadFallback == 'true') {
+  options.headers['X-Local-Fallback-Cloudflare'] = true;
+  data['error_message'] = downloadErrorMessage;
+}
+
+if(proxyHost && proxyPort) {
+  options.agent = new HttpsProxyAgent({
+    host: proxyHost,
+    port: proxyPort
+  });
+}
+if (useCaCertificate) {
+  try {
+    options.ca = fs.readFileSync(useCaCertificate);
+  } catch(err) {
+    console.log('failed to read cert file', err);
+  }
+}
+
+const req = https.request(options, res => {
+  res.on('data', d => {
+    body += d;
+  });
+  res.on('end', () => {
+    try {
+      const reqBody = JSON.parse(body);
+      if(reqBody.error) {
+        throw reqBody.error;
+      }
+      console.log(reqBody.data.endpoint);
+    } catch (e) {
+      console.error(e);
+    }
+  });
+  res.on('error', (err) => {
+    console.error(err);
+  });
+});
+req.on('error', e => {
+  console.error(e);
+});
+req.write(JSON.stringify(data));
+req.end();
+

test/local.js

@@ -280,7 +280,7 @@ describe('LocalBinary', function () {
       // ensure that we have a valid binary downloaded
 
       // removeIfInvalid();
-      (new LocalBinary()).binaryPath({}, function(binaryPath) {
+      (new LocalBinary()).binaryPath({}, 'abc', 9, function(binaryPath) {
         defaultBinaryPath = binaryPath;
         tempfs.mkdir({
           recursive: true
@@ -313,7 +313,7 @@ describe('LocalBinary', function () {
         fs.writeFile(defaultBinaryPath, 'Random String', function() {
           fs.chmod(defaultBinaryPath, '0755', function() {
             localBinary.binaryPath({
-            }, function(binaryPath) {
+            }, 'abc', 9, function(binaryPath) {
               expect(downloadStub.called).to.be.true;
               done();
             });
@@ -331,7 +331,7 @@ describe('LocalBinary', function () {
         });
 
         localBinary.binaryPath({
-        }, function(binaryPath) {
+        }, 'abc', 9, function(binaryPath) {
           expect(downloadStub.called).to.be.true;
           done();
         });