parallelize filling directory entries

fixes #4307

Author: vladak

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Configuration.java

@@ -269,6 +269,12 @@ public final class Configuration {
      */
     private int MaxRevisionThreadCount;
 
+    /**
+     * Upper bound for number of threads used for getting directory entries.
+     * This is total for the whole webapp.
+     */
+    private int MaxDirectoryListingThreadCount;
+
     /**
      * If false, do not display listing or projects/repositories on the index page.
      */
@@ -586,6 +592,7 @@ public Configuration() {
         //mandoc is default(String)
         setMaxSearchThreadCount(2 * Runtime.getRuntime().availableProcessors());
         setMaxRevisionThreadCount(Runtime.getRuntime().availableProcessors());
+        setMaxDirectoryListingThreadCount(Runtime.getRuntime().availableProcessors());
         setMergeCommitsEnabled(true);
         setMessageLimit(500);
         setNavigateWindowEnabled(false);
@@ -1353,6 +1360,14 @@ public void setMaxRevisionThreadCount(int count) {
         this.MaxRevisionThreadCount = count;
     }
 
+    public int getMaxDirectoryListingThreadCount() {
+        return MaxDirectoryListingThreadCount;
+    }
+
+    public void setMaxDirectoryListingThreadCount(int count) {
+        this.MaxDirectoryListingThreadCount = count;
+    }
+
     public boolean isProjectsEnabled() {
         return projectsEnabled;
     }

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java

@@ -103,6 +103,7 @@ public final class RuntimeEnvironment {
     private final LazilyInstantiate<IndexerParallelizer> lzIndexerParallelizer;
     private final LazilyInstantiate<ExecutorService> lzSearchExecutor;
     private final LazilyInstantiate<ExecutorService> lzRevisionExecutor;
+    private final LazilyInstantiate<ExecutorService> lzDirectoryListingExecutor;
     private static final RuntimeEnvironment instance = new RuntimeEnvironment();
 
     private final LazilyInstantiate<SuperIndexSearcherFactory> lzSuperIndexSearcherFactory;
@@ -164,6 +165,7 @@ private RuntimeEnvironment() {
         lzIndexerParallelizer = LazilyInstantiate.using(() -> new IndexerParallelizer(this));
         lzSearchExecutor = LazilyInstantiate.using(this::newSearchExecutor);
         lzRevisionExecutor = LazilyInstantiate.using(this::newRevisionExecutor);
+        lzDirectoryListingExecutor = LazilyInstantiate.using(this::newDirectoryListingExecutor);
         lzSuperIndexSearcherFactory = LazilyInstantiate.using(this::newSuperIndexSearcherFactory);
         lzIndexSearcherFactory = LazilyInstantiate.using(this::newIndexSearcherFactory);
     }
@@ -227,6 +229,20 @@ public void shutdownRevisionExecutor() throws InterruptedException {
         getRevisionExecutor().awaitTermination(getIndexerCommandTimeout(), TimeUnit.SECONDS);
     }
 
+    public ExecutorService getDirectoryListingExecutor() {
+        return lzDirectoryListingExecutor.get();
+    }
+
+    private ExecutorService newDirectoryListingExecutor() {
+        return Executors.newFixedThreadPool(this.getMaxDirectoryListingThreadCount(),
+                new OpenGrokThreadFactory("directory-listing"));
+    }
+
+    public void shutdownDirectoryListingExecutor() throws InterruptedException {
+        getDirectoryListingExecutor().shutdownNow();
+        getDirectoryListingExecutor().awaitTermination(getWebappStartCommandTimeout(), TimeUnit.SECONDS);
+    }
+
     private SuperIndexSearcherFactory newSuperIndexSearcherFactory() {
         return new SuperIndexSearcherFactory();
     }
@@ -1388,6 +1404,14 @@ public int getMaxRevisionThreadCount() {
         return syncReadConfiguration(Configuration::getMaxRevisionThreadCount);
     }
 
+    public int getMaxDirectoryListingThreadCount() {
+        return syncReadConfiguration(Configuration::getMaxDirectoryListingThreadCount);
+    }
+
+    public void setMaxDirectoryListingThreadCount(int threadCount) {
+        syncWriteConfiguration(threadCount, Configuration::setMaxDirectoryListingThreadCount);
+    }
+
     public int getCurrentIndexedCollapseThreshold() {
         return syncReadConfiguration(Configuration::getCurrentIndexedCollapseThreshold);
     }

opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java

@@ -37,11 +37,14 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -766,32 +769,51 @@ public boolean fillLastHistoryEntries(List<DirectoryEntry> entries) {
 
         boolean ret = true;
 
+        Statistics statistics = new Statistics();
+
+        final ExecutorService executor = env.getDirectoryListingExecutor();
+        Set<Future<Boolean>> futures = new HashSet<>();
         for (DirectoryEntry directoryEntry : entries) {
-            try {
-                File file = directoryEntry.getFile();
-                if (file.isDirectory()) {
-                    directoryEntry.setDescription("-");
-                    directoryEntry.setDate(null);
-                    continue;
+            futures.add(executor.submit(() -> {
+                try {
+                    File file = directoryEntry.getFile();
+                    if (file.isDirectory()) {
+                        directoryEntry.setDescription("-");
+                        directoryEntry.setDate(null);
+                        return true;
+                    }
+
+                    HistoryEntry historyEntry = getLastHistoryEntry(file);
+                    if (historyEntry != null && historyEntry.getDate() != null) {
+                        directoryEntry.setDescription(historyEntry.getDescription());
+                        directoryEntry.setDate(historyEntry.getDate());
+                    } else {
+                        LOGGER.log(Level.FINE, "cannot get last history entry for ''{0}''",
+                                directoryEntry.getFile());
+                        return false;
+                    }
+                } catch (CacheException e) {
+                    LOGGER.log(Level.FINER, "cannot get last history entry for ''{0}''", directoryEntry.getFile());
+                    return false;
                 }
+                return true;
+            }));
+        }
 
-                HistoryEntry historyEntry = getLastHistoryEntry(file);
-                if (historyEntry != null && historyEntry.getDate() != null) {
-                    directoryEntry.setDescription(historyEntry.getDescription());
-                    directoryEntry.setDate(historyEntry.getDate());
-                } else {
-                    LOGGER.log(Level.FINE, "cannot get last history entry for ''{0}''",
-                            directoryEntry.getFile());
+        for (Future<Boolean> future : futures) {
+            try {
+                if (!future.get()) {
                     ret = false;
                     break;
                 }
-            } catch (CacheException e) {
-                LOGGER.log(Level.FINER, "cannot get last history entry for ''{0}''", directoryEntry.getFile());
+            } catch (Exception e) {
                 ret = false;
                 break;
             }
         }
 
+        statistics.report(LOGGER, Level.FINER, "done filling directory entries");
+
         // Enforce the all-or-nothing semantics.
         if (!ret) {
             entries.forEach(e -> e.setDate(null));

opengrok-web/src/main/java/org/opengrok/web/WebappListener.java

@@ -204,6 +204,7 @@ public void contextDestroyed(final ServletContextEvent servletContextEvent) {
         try {
             env.shutdownRevisionExecutor();
             env.shutdownSearchExecutor();
+            env.shutdownDirectoryListingExecutor();
         } catch (InterruptedException e) {
             LOGGER.log(Level.WARNING, "Could not shutdown revision executor", e);
         }