Asynchronously build shortIndex

- refactoring cache crawler to use concurrent async approach

Author: vladimyr

lib/index.js

@@ -106,7 +106,9 @@ function readShortPagesIndex() {
     })
     .catch(() => {
       // File is not present; we need to create the index.
-      let idx = buildShortPagesIndex();
+      return buildShortPagesIndex();
+    })
+    .then((idx) => {
       if (Object.keys(idx).length <= 0) {
         return idx;
       }
@@ -118,23 +120,24 @@ function readShortPagesIndex() {
 }
 
 function buildShortPagesIndex() {
-  try {
-    let files = utils.walkSync(pagesPath);
-    files = files.filter(utils.isPage);
-    let reducer = (index, file) => {
-      let os = utils.parsePlatform(file);
-      let page = utils.parsePagename(file);
-      if (index[page]) {
-        index[page].push(os);
-      } else {
-        index[page] = [os];
-      }
-      return index;
-    };
-    return files.reduce(reducer, {});
-  } catch (e) {
-    return {};
-  }
+  return utils.walk(pagesPath)
+    .then((files) => {
+      files = files.filter(utils.isPage);
+      let reducer = (index, file) => {
+        let os = utils.parsePlatform(file);
+        let page = utils.parsePagename(file);
+        if (index[page]) {
+          index[page].push(os);
+        } else {
+          index[page] = [os];
+        }
+        return index;
+      };
+      return files.reduce(reducer, {});
+    })
+    .catch(() => {
+      return {};
+    });
 }
 
 module.exports = {

lib/utils.js

@@ -1,6 +1,13 @@
-var fs = require('fs-extra');
-var path = require('path');
-var glob = require('glob');
+const fs = require('fs-extra');
+const path = require('path');
+const glob = require('glob');
+const promiseMap = require('p-map');
+
+const flatten = (arr) => {
+  return arr.reduce((acc, it) => {
+    return acc.concat(it);
+  }, []);
+};
 
 module.exports = {
   parsePlatform(pagefile) {
@@ -23,20 +30,24 @@ module.exports = {
     };
   },
 
-  walkSync(dir, filelist) {
-    var files = fs.readdirSync(dir);
-    filelist = filelist || [];
-    files.forEach((file) => {
-      if (fs.statSync(path.join(dir, file)).isDirectory()) {
-        filelist = this.walkSync(path.join(dir, file), filelist);
-      }
-      else {
-        filelist.push(path.join(path.basename(dir), file));
-      }
-    });
-    return filelist;
+  walk(dir) {
+    return fs.readdir(dir)
+      .then((items) => {
+        return promiseMap(items, (item) => {
+          const itemPath = path.join(dir, item);
+          return fs.stat(itemPath).then((stat) => {
+            if (stat.isDirectory()) {
+              return this.walk(itemPath);
+            }
+            return path.join(path.basename(dir), item);
+          });
+        });
+      })
+      .then((paths) => {
+        return flatten(paths);
+      });
   },
-  
+
   glob(string,options) {
     return new Promise((resolve, reject) => {
       glob(string, options, (err, data) => {

package-lock.json

@@ -57,6 +57,7 @@
     "natural": "^0.5.4",
     "ora": "^2.0.0",
     "os-homedir": "^1.0.1",
+    "p-map": "^1.2.0",
     "request": "^2.75.0",
     "unzip2": "^0.2.5"
   },

package.json

@@ -32,7 +32,7 @@ describe('Index building', () => {
 
   describe('failure', () => {
     before(() => {
-      sinon.stub(utils, 'walkSync').throws('dummy error');
+      sinon.stub(utils, 'walk').rejects('dummy error');
     });
 
     it('shortIndex should not be created', () => {
@@ -43,7 +43,7 @@ describe('Index building', () => {
 
   describe('success', () => {
     before(() => {
-      sinon.stub(utils, 'walkSync').returns(pages);
+      sinon.stub(utils, 'walk').resolves(pages);
     });
 
     it('correct shortIndex should be created', () => {
@@ -53,7 +53,7 @@ describe('Index building', () => {
   });
 
   afterEach(() => {
-    utils.walkSync.restore();
+    utils.walk.restore();
     fs.readJson.restore();
     fs.writeJson.restore();
   });
@@ -62,13 +62,13 @@ describe('Index building', () => {
 describe('Index', () => {
   beforeEach(() => {
     index.clearRuntimeIndex();
-    sinon.stub(utils, 'walkSync').returns(pages);
+    sinon.stub(utils, 'walk').resolves(pages);
     sinon.stub(fs, 'readJson').rejects('dummy error');
     sinon.stub(fs, 'writeJson').resolves('');
   });
 
   afterEach(() => {
-    utils.walkSync.restore();
+    utils.walk.restore();
     fs.readJson.restore();
     fs.writeJson.restore();
   });