add option to turn off per partes history cache

fixes #3660

Author: Vladimir Kotal

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Configuration.java

@@ -309,6 +309,7 @@ public final class Configuration {
     private boolean allowInsecureTokens;
 
     private int historyChunkCount;
+    private boolean historyCachePerPartesEnabled = true;
 
     /*
      * types of handling history for remote SCM repositories:
@@ -1385,6 +1386,14 @@ public void setHistoryChunkCount(int historyChunkCount) {
         this.historyChunkCount = historyChunkCount;
     }
 
+    public boolean isHistoryCachePerPartesEnabled() {
+        return historyCachePerPartesEnabled;
+    }
+
+    public void setHistoryCachePerPartesEnabled(boolean historyCachePerPartesEnabled) {
+        this.historyCachePerPartesEnabled = historyCachePerPartesEnabled;
+    }
+
     /**
      * Write the current configuration to a file.
      *

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java

@@ -1368,6 +1368,14 @@ public void setHistoryChunkCount(int chunkCount) {
         syncWriteConfiguration(chunkCount, Configuration::setHistoryChunkCount);
     }
 
+    public boolean isHistoryCachePerPartesEnabled() {
+        return syncReadConfiguration(Configuration::isHistoryCachePerPartesEnabled);
+    }
+
+    public void setHistoryCachePerPartesEnabled(boolean enabled) {
+        syncWriteConfiguration(enabled, Configuration::setHistoryCachePerPartesEnabled);
+    }
+
     public Set<String> getDisabledRepositories() {
         return syncReadConfiguration(Configuration::getDisabledRepositories);
     }

opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java

@@ -397,6 +397,7 @@ public void store(History history, Repository repository) throws HistoryExceptio
      * corresponding source file.
      *
      * @param history history object to process into per-file histories
+     * @param tillRevision end revision
      * @param repository repository object
      */
     @Override

opengrok-indexer/src/main/java/org/opengrok/indexer/history/HistoryCache.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2006, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2006, 2021, Oracle and/or its affiliates. All rights reserved.
  */
 package org.opengrok.indexer.history;
 
@@ -73,6 +73,14 @@ History get(File file, Repository repository, boolean withFiles)
      */
     void store(History history, Repository repository) throws HistoryException;
 
+    /**
+     * Store potentially partial history for a repository.
+     *
+     * @param history The history to store
+     * @param repository The repository whose history to store
+     * @param tillRevision end revision (can be null)
+     * @throws HistoryException if the history cannot be stored
+     */
     void store(History history, Repository repository, String tillRevision) throws HistoryException;
 
     /**

opengrok-indexer/src/main/java/org/opengrok/indexer/history/Repository.java

@@ -413,6 +413,13 @@ final void createCache(HistoryCache cache, String sinceRevision) throws HistoryE
         LOGGER.log(Level.FINE, "Done storing history cache for repository {0}", getDirectoryName());
     }
 
+    /**
+     * Actually store the history in history cache
+     * @param cache history cache object
+     * @param history history to store
+     * @param tillRevision end revision (matters only for renamed files), can be null
+     * @throws HistoryException on error
+     */
     void finishCreateCache(HistoryCache cache, History history, String tillRevision) throws HistoryException {
         // We need to refresh list of tags for incremental reindex.
         RuntimeEnvironment env = RuntimeEnvironment.getInstance();

opengrok-indexer/src/main/java/org/opengrok/indexer/history/RepositoryWithPerPartesHistory.java

@@ -22,6 +22,7 @@
  */
 package org.opengrok.indexer.history;
 
+import org.opengrok.indexer.configuration.RuntimeEnvironment;
 import org.opengrok.indexer.logger.LoggerFactory;
 import org.opengrok.indexer.util.Statistics;
 
@@ -70,6 +71,13 @@ public int getPerPartesCount() {
 
     @Override
     protected void doCreateCache(HistoryCache cache, String sinceRevision, File directory) throws HistoryException {
+        if (!RuntimeEnvironment.getInstance().isHistoryCachePerPartesEnabled()) {
+            LOGGER.log(Level.INFO, "repository {0} supports per partes history cache creation however " +
+                    "it is disabled in the configuration. Generating history cache as whole.", this);
+            finishCreateCache(cache, getHistory(directory, sinceRevision), null);
+            return;
+        }
+
         // For repositories that supports this, avoid storing complete History in memory
         // (which can be sizeable, at least for the initial indexing, esp. if merge changeset support is enabled),
         // by splitting the work into multiple chunks.

opengrok-indexer/src/test/java/org/opengrok/indexer/history/RepositoryWithPerPartesHistoryTest.java

@@ -44,6 +44,7 @@
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.isNull;
 import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 
 public class RepositoryWithPerPartesHistoryTest {
     private TestRepository repositories;
@@ -127,4 +128,17 @@ void testPseudoIncomingChangeset() throws Exception {
         assertEquals(1, cachedHistory.getHistoryEntries().size());
         assertEquals(historyEntries.get(0).getRevision(), cachedHistory.getHistoryEntries().get(0).getRevision());
     }
+
+    @Test
+    void testPerPartesOff() throws HistoryException {
+        FileHistoryCache cache = new FileHistoryCache();
+        FileHistoryCache spyCache = Mockito.spy(cache);
+        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
+        env.setHistoryCachePerPartesEnabled(false);
+        assertFalse(env.isHistoryCachePerPartesEnabled());
+        // Use non-null revision for better robustness (in case sinceRevision gets mixed up with tillRevision).
+        gitRepository.createCache(spyCache, "b6413947a59f481ddc0a05e0d181731233557f6e");
+        verify(spyCache, times(1)).store(any(), any(), isNull());
+        env.setHistoryCachePerPartesEnabled(true);
+    }
 }