make getDiffData() parallel (#2955)

fixes #2953

Author: Vladimir Kotal

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Configuration.java

@@ -266,10 +266,16 @@ public final class Configuration {
 
     /**
      * Upper bound for number of threads used for performing multi-project
-     * searches. This is total for the whole webapp/CLI utility.
+     * searches. This is total for the whole webapp.
      */
     private int MaxSearchThreadCount;
 
+    /**
+     * Upper bound for number of threads used for getting revision contents.
+     * This is total for the whole webapp.
+     */
+    private int MaxRevisionThreadCount;
+
     /**
      * If false, do not display listing or projects/repositories on the index page.
      */
@@ -470,6 +476,7 @@ public Configuration() {
         //luceneLocking default is OFF
         //mandoc is default(String)
         setMaxSearchThreadCount(2 * Runtime.getRuntime().availableProcessors());
+        setMaxRevisionThreadCount(Runtime.getRuntime().availableProcessors());
         setMessageLimit(500);
         setNavigateWindowEnabled(false);
         setOptimizeDatabase(true);
@@ -1222,7 +1229,15 @@ public int getMaxSearchThreadCount() {
     public void setMaxSearchThreadCount(int count) {
         this.MaxSearchThreadCount = count;
     }
-    
+
+    public int getMaxRevisionThreadCount() {
+        return MaxRevisionThreadCount;
+    }
+
+    public void setMaxRevisionThreadCount(int count) {
+        this.MaxRevisionThreadCount = count;
+    }
+
     public boolean isProjectsEnabled() {
         return projectsEnabled;
     }

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java

@@ -43,6 +43,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.logging.Level;
@@ -57,6 +58,7 @@
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.NamedThreadFactory;
 import org.opengrok.indexer.authorization.AuthorizationFramework;
 import org.opengrok.indexer.authorization.AuthorizationStack;
 import org.opengrok.indexer.history.HistoryGuru;
@@ -91,6 +93,7 @@ public final class RuntimeEnvironment {
     private final ReentrantReadWriteLock configLock;
     private final LazilyInstantiate<IndexerParallelizer> lzIndexerParallelizer;
     private final LazilyInstantiate<ExecutorService> lzSearchExecutor;
+    private final LazilyInstantiate<ExecutorService> lzRevisionExecutor;
     private static final RuntimeEnvironment instance = new RuntimeEnvironment();
 
     private final Map<Project, List<RepositoryInfo>> repository_map = new ConcurrentHashMap<>();
@@ -129,6 +132,7 @@ private RuntimeEnvironment() {
         lzIndexerParallelizer = LazilyInstantiate.using(() ->
                 new IndexerParallelizer(this));
         lzSearchExecutor = LazilyInstantiate.using(() -> newSearchExecutor());
+        lzRevisionExecutor = LazilyInstantiate.using(() -> newRevisionExecutor());
     }
 
     // Instance of authorization framework and its lock.
@@ -153,6 +157,20 @@ public Thread newThread(Runnable runnable) {
             });
     }
 
+    public ExecutorService getRevisionExecutor() {
+        return lzRevisionExecutor.get();
+    }
+
+    private ExecutorService newRevisionExecutor() {
+        return Executors.newFixedThreadPool(this.getMaxRevisionThreadCount(),
+                new NamedThreadFactory("get-revision"));
+    }
+
+    public void shutdownRevisionExecutor() throws InterruptedException {
+        getRevisionExecutor().shutdownNow();
+        getRevisionExecutor().awaitTermination(getCommandTimeout(), TimeUnit.SECONDS);
+    }
+
     /**
      * Get the one and only instance of the RuntimeEnvironment.
      *
@@ -1298,6 +1316,14 @@ public int getMaxSearchThreadCount() {
         return (int) getConfigurationValue("maxSearchThreadCount");
     }
 
+    public void setMaxRevisionThreadCount(int count) {
+        setConfigurationValue("maxRevisionThreadCount", count);
+    }
+
+    public int getMaxRevisionThreadCount() {
+        return (int) getConfigurationValue("maxRevisionThreadCount");
+    }
+
     public int getCurrentIndexedCollapseThreshold() {
         return (int) getConfigurationValue("currentIndexedCollapseThreshold");
     }

opengrok-indexer/src/main/java/org/opengrok/indexer/web/PageConfig.java

@@ -49,6 +49,8 @@
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.regex.Pattern;
@@ -58,6 +60,7 @@
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import javax.ws.rs.core.HttpHeaders;
+
 import org.opengrok.indexer.Info;
 import org.opengrok.indexer.analysis.AbstractAnalyzer;
 import org.opengrok.indexer.analysis.AnalyzerGuru;
@@ -148,6 +151,8 @@ public final class PageConfig {
     private static final String ATTR_NAME = PageConfig.class.getCanonicalName();
     private HttpServletRequest req;
 
+    private ExecutorService executor;
+
     /**
      * Sets current request's attribute.
      * 
@@ -255,9 +260,18 @@ public DiffData getDiffData() {
             InputStream[] in = new InputStream[2];
             try {
                 // Get input stream for both older and newer file.
+                ExecutorService executor = this.executor;
+                Future<?>[] future = new Future<?>[2];
                 for (int i = 0; i < 2; i++) {
                     File f = new File(srcRoot + filepath[i]);
-                    in[i] = HistoryGuru.getInstance().getRevision(f.getParent(), f.getName(), data.rev[i]);
+                    final String revision = data.rev[i];
+                    future[i] = executor.submit(() -> HistoryGuru.getInstance().
+                            getRevision(f.getParent(), f.getName(), revision));
+                }
+
+                for (int i = 0; i < 2; i++) {
+                    // The Executor used by given repository will enforce the timeout.
+                    in[i] = (InputStream) future[i].get();
                     if (in[i] == null) {
                         data.errorMsg = "Unable to get revision "
                                 + Util.htmlize(data.rev[i]) + " for file: "
@@ -287,7 +301,7 @@ public DiffData getDiffData() {
                 ArrayList<String> lines = new ArrayList<>();
                 Project p = getProject();
                 for (int i = 0; i < 2; i++) {
-                    // SRCROOT is read with UTF-8 as a default.
+                    // All files under source root are read with UTF-8 as a default.
                     try (BufferedReader br = new BufferedReader(
                         ExpandTabsReader.wrap(IOUtils.createBOMStrippedReader(
                         in[i], StandardCharsets.UTF_8.name()), p))) {
@@ -1530,6 +1544,7 @@ public static PageConfig get(HttpServletRequest request) {
     private PageConfig(HttpServletRequest req) {
         this.req = req;
         this.authFramework = RuntimeEnvironment.getInstance().getAuthorizationFramework();
+        this.executor = RuntimeEnvironment.getInstance().getRevisionExecutor();
     }
 
     /**

opengrok-web/src/main/java/org/opengrok/web/WebappListener.java

@@ -114,6 +114,11 @@ public void contextDestroyed(final ServletContextEvent servletContextEvent) {
         env.getIndexerParallelizer().bounce();
         env.watchDog.stop();
         env.stopExpirationTimer();
+        try {
+            env.shutdownRevisionExecutor();
+        } catch (InterruptedException e) {
+            LOGGER.log(Level.WARNING, "Could not shutdown revision executor", e);
+        }
         try {
             saveStatistics();
         } catch (IOException ex) {