Merge pull request #475 from browserstack/package-diff

🎨 added instrumentation for package diff and pushing package.json

Author: agrawalsaurabhs

bin/commands/runs.js

@@ -19,7 +19,8 @@ const archiver = require("../helpers/archiver"),
   downloadBuildArtifacts = require('../helpers/buildArtifacts').downloadBuildArtifacts,
   downloadBuildStacktrace = require('../helpers/downloadBuildStacktrace').downloadBuildStacktrace,
   updateNotifier = require('update-notifier'),
-  pkg = require('../../package.json');
+  pkg = require('../../package.json'),
+  packageDiff = require('../helpers/package-diff');
 const { getStackTraceUrl } = require('../helpers/sync/syncSpecsLogs');
 
 module.exports = function run(args, rawArgs) {
@@ -158,13 +159,21 @@ module.exports = function run(args, rawArgs) {
           // Archive the spec files
           logger.debug("Started archiving test suite");
           markBlockStart('zip.archive');
-          return archiver.archive(bsConfig.run_settings, config.fileName, args.exclude, md5data).then(function (data) {
+          return archiver.archive(bsConfig.run_settings, config.fileName, args.exclude, md5data).then(async function (data) {
             logger.debug("Completed archiving test suite");
             markBlockEnd('zip.archive');
 
             let test_zip_size = utils.fetchZipSize(path.join(process.cwd(), config.fileName));
             let npm_zip_size = utils.fetchZipSize(path.join(process.cwd(), config.packageFileName));
-
+            let node_modules_size = await utils.fetchFolderSize(path.join(process.cwd(), "node_modules"))            
+            
+            //Package diff
+            let isPackageDiff = false;
+            if(!md5data.zipUrlPresent){
+              isPackageDiff = packageDiff.run(`package.json`, `${config.packageDirName}/package.json`);
+              logger.debug(`Package difference was ${isPackageDiff ? `found` : `not found`}`);
+            }
+            
             // Uploaded zip file
             logger.debug("Started uploading the test suite zip");
             logger.debug("Started uploading the node_module zip");
@@ -267,9 +276,13 @@ module.exports = function run(args, rawArgs) {
                   build_id: data.build_id,
                   test_zip_size: test_zip_size,
                   npm_zip_size: npm_zip_size,
+                  node_modules_size: node_modules_size,
                   test_suite_zip_upload: md5data.zipUrlPresent ? 0 : 1,
                   package_zip_upload: md5data.packageUrlPresent ? 0 : 1
                 };
+                if(dataToSend.test_suite_zip_upload === 1 ){
+                  dataToSend['is_package_diff'] = isPackageDiff;
+                }
 
                 if (!md5data.zipUrlPresent && zip.tests_upload_time) {
                   dataToSend.test_suite_zip_size = parseFloat((test_zip_size / 1024).toFixed(2));

bin/helpers/archiver.js

@@ -1,4 +1,5 @@
 'use strict';
+const config = require('./config.js');
 const fs = require("fs"),
     path = require("path");
 
@@ -76,6 +77,14 @@ const archiveSpecs = (runSettings, filePath, excludeFiles, md5data) => {
       archive.append(packageJSONString, {name: `${cypressAppendFilesZipLocation}browserstack-package.json`});
     }
 
+    //Create copy of package.json
+    if(fs.existsSync('package.json')){
+      let originalPackageJson = JSON.parse(fs.readFileSync('package.json'));
+      let originalPackageJsonString = JSON.stringify(originalPackageJson, null, 4);
+      archive.append(originalPackageJsonString, {name: `${cypressAppendFilesZipLocation}userPackage.json`});
+      logger.debug(`Created copy of package.json in ${config.packageDirName} folder`)
+    }
+
     // do not add cypress.json if arg provided is false
     if (
       runSettings.cypress_config_file &&

bin/helpers/package-diff.js

@@ -0,0 +1,93 @@
+"use strict";
+const fs = require("fs");
+const path = require("path");
+const logger = require("./logger").winstonLogger;
+
+exports.run = (basePath, comparePath) => {
+  if (!basePath || !comparePath) {
+    logger.debug("Skipping package difference check.");
+  }
+
+  let base;
+  let compare;
+  let isDiff = false;
+  try {
+    base = readModules(basePath);
+    compare = readModules(comparePath);
+  } catch (error) {
+    logger.debug('Unable to process package difference');
+    return isDiff;
+  }
+  
+  Object.keys(base.deps).forEach((baseKey) => {
+    if (baseKey in compare.deps) {
+      if (base.deps[baseKey] !== compare.deps[baseKey]) {
+        isDiff = true;
+        return;
+      }
+    } else {
+      isDiff = true;
+      return;
+    }
+  });
+  return isDiff;
+};
+const readModules = (location) => {
+  const table = {};
+
+  // Resolve package dependencies
+  if (location.indexOf("package.json") !== -1) {
+    const data = fs.readFileSync(location.replace(":dev", ""), "utf-8");
+    let parsed;
+    try {
+      parsed = JSON.parse(data);
+    } catch (e) {
+      parsed = false;
+    }
+    if (!parsed) {
+      return;
+    }
+
+    const depsKey =
+      location.indexOf(":dev") !== -1 ? "devDependencies" : "dependencies";
+    const deps = parsed[depsKey]
+      ? parsed[depsKey]
+      : parsed.dependencies || parsed.devDependencies;
+
+    Object.keys(deps).forEach((key) => {
+      deps[key] = deps[key].replace(/\^|~/g, "");
+    });
+    return {
+      name: `${location} {${depsKey}}`,
+      deps,
+    };
+  }
+
+  fs.readdirSync(location)
+    .filter((name) => name !== ".bin")
+    .map((name) => {
+      const pkg = path.join(location, name, "package.json");
+      const exists = fs.existsSync(pkg);
+      if (!exists) {
+        return;
+      }
+
+      const data = fs.readFileSync(pkg, "utf-8");
+      let parsed;
+
+      try {
+        parsed = JSON.parse(data);
+      } catch (e) {
+        parsed = false;
+      }
+      if (!parsed) {
+        return;
+      }
+
+      table[name] = parsed.version;
+    });
+  return {
+    name: location,
+    deps: table,
+  };
+};

bin/helpers/utils.js

@@ -8,6 +8,9 @@ const { v4: uuidv4 } = require('uuid');
 const browserstack = require('browserstack-local');
 const crypto = require('crypto');
 const util = require('util');
+const { promisify } = require('util');
+const readdir = promisify(fs.readdir);
+const stat = promisify(fs.stat);
 
 const usageReporting = require("./usageReporting"),
   logger = require("./logger").winstonLogger,
@@ -1278,6 +1281,34 @@ exports.fetchZipSize = (fileName) => {
   }
 }
 
+const getDirectorySize = async function(dir) {
+  try{
+    const subdirs = (await readdir(dir));
+    const files = await Promise.all(subdirs.map(async (subdir) => {
+      const res = path.resolve(dir, subdir);
+      const s = (await stat(res));
+      return s.isDirectory() ? getDirectorySize(res) : (s.size);
+    }));
+    return files.reduce((a, f) => a+f, 0);
+  }catch(e){
+    console.log(`Error ${e}`)
+    logger.debug('Failed to get file or directory.');
+    return 0;
+  }
+};
+
+exports.fetchFolderSize = async (dir) => {
+  try {
+    if(fs.existsSync(dir)){
+      return (await getDirectorySize(dir) / 1024 / 1024);
+    }
+    return 0;
+  } catch (error) {
+    logger.debug(`Failed to get directory size.`);
+    return 0;
+  }
+}
+
 exports.getVideoConfig = (cypressConfig) => {
   let conf = {
     video: true,

test/unit/bin/helpers/utils.js

@@ -1042,6 +1042,13 @@ describe('utils', () => {
     });
   });
 
+  describe('getDirectorySize', () => {
+    it('should return size of directory', async() => {
+      expect(await utils.fetchFolderSize('/absolute/path')).to
+        .be.equal(0);
+    });
+  });
+
   describe('getLocalFlag', () => {
     it('should return false if connectionSettings is undefined', () => {
       expect(utils.getLocalFlag(undefined)).to.be.false;
@@ -1834,7 +1841,7 @@ describe('utils', () => {
   describe('setCypressTestSuiteType', () => {
 
     it('sets correct cypressTestSuiteType when cypress.json is the cypress config file ', () => {
-      bsConfig = {
+      let bsConfig = {
         run_settings: {
           cypressConfigFilePath: 'cypress.json',
         },
@@ -1846,7 +1853,7 @@ describe('utils', () => {
     });
 
     it('sets correct cypressTestSuiteType when cypress.config.js|.ts|.cjs|.mjs is the cypress config file ', () => {
-      bsConfig = {
+      let bsConfig = {
         run_settings: {
           cypressConfigFilePath: 'cypress.config.js',
         },
@@ -1880,7 +1887,7 @@ describe('utils', () => {
     });
 
     it('by default assumes that CYPRESS_V9_AND_OLDER_TYPE is the test suite type', () => {
-      bsConfig = {
+      let bsConfig = {
         run_settings: {},
       };
       utils.setCypressTestSuiteType(bsConfig);