add API endpoint for deleting history cache

Vladimir Kotal · Vladimir Kotal · commit 09dc2337b08d · 2019-02-05T20:27:51.000+01:00
fixes #2655
diff --git a/apiary.apib b/apiary.apib
@@ -156,6 +156,14 @@ This will also mark the project as not indexed.
 
 + Response 204
 
+## Project history cache management [/projects/{project}/historycache]
+
+### delete history cache for a project [DELETE]
+
+This will delete history cache for a project.
+
++ Response 204
+
 ## Project metadata management [/projects/{project}/indexed]
 
 This entry point is used by the Indexer once it finishes indexing given project.
diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java
@@ -55,7 +55,6 @@
 import org.opengrok.indexer.configuration.LuceneLockName;
 import org.opengrok.indexer.configuration.Project;
 import org.opengrok.indexer.configuration.RuntimeEnvironment;
-import org.opengrok.indexer.history.HistoryException;
 import org.opengrok.indexer.history.HistoryGuru;
 import org.opengrok.indexer.history.Repository;
 import org.opengrok.indexer.history.RepositoryFactory;
@@ -114,7 +113,6 @@ public final class Indexer {
     private static final Set<String> repositories = new HashSet<>();
     private static final HashSet<String> allowedSymlinks = new HashSet<>();
     private static final Set<String> defaultProjects = new TreeSet<>();
-    private static final ArrayList<String> zapCache = new ArrayList<>();
     private static RuntimeEnvironment env = null;
     private static String webappURI = null;
 
@@ -319,9 +317,9 @@ public static void main(String[] argv) {
             getInstance().prepareIndexer(env, searchRepositories, addProjects,
                     defaultProjects,
                     listFiles, createDict, runIndex, subFiles, new ArrayList<>(repositories),
-                    zapCache, listRepos);
+                    listRepos);
 
-            if (listRepos || !zapCache.isEmpty()) {
+            if (listRepos) {
                 return;
             }
 
@@ -498,12 +496,6 @@ public static String[] parseOptions(String[] argv) throws ParseException {
                 }
             );
 
-            parser.on("--deleteHistory", "=/path/to/repository",
-                "Delete the history cache for the given repository and exit.",
-                "Use '*' to delete the cache for all repositories.").Do(repo -> {
-                zapCache.add((String)repo);
-            });
-
             parser.on("--depth", "=number", Integer.class,
                 "Scanning depth for repositories in directory structure relative to",
                 "source root. Default is " + Configuration.defaultScanningDepth + ".").Do(depth -> {
@@ -899,10 +891,9 @@ public void prepareIndexer(RuntimeEnvironment env,
                                boolean createDict,
                                List<String> subFiles,
                                List<String> repositories,
-                               List<String> zapCache,
                                boolean listRepoPaths) throws IndexerException, IOException {
         prepareIndexer(env, searchRepositories, addProjects, defaultProjects, listFiles, createDict, true,
-                subFiles, repositories, zapCache, listRepoPaths);
+                subFiles, repositories, listRepoPaths);
     }
 
     /**
@@ -924,7 +915,6 @@ public void prepareIndexer(RuntimeEnvironment env,
      * @param createHistoryCache create history cache flag
      * @param subFiles list of directories
      * @param repositories list of repositories
-     * @param zapCache list of projects to remove history cache for
      * @param listRepoPaths print repository paths to standard output
      * @throws IndexerException indexer exception
      * @throws IOException I/O exception
@@ -939,7 +929,6 @@ public void prepareIndexer(RuntimeEnvironment env,
             boolean createHistoryCache,
             List<String> subFiles,
             List<String> repositories,
-            List<String> zapCache,
             boolean listRepoPaths) throws IndexerException, IOException {
 
         if (env.getDataRootPath() == null) {
@@ -950,12 +939,9 @@ public void prepareIndexer(RuntimeEnvironment env,
             throw new IndexerException("ERROR: please specify a SRC_ROOT with option -s !");
         }
 
-        if (zapCache.isEmpty() && !env.validateUniversalCtags()) {
+        if (!env.validateUniversalCtags()) {
             throw new IndexerException("Didn't find Universal Ctags");
         }
-        if (zapCache == null) {
-            throw new IndexerException("Internal error, zapCache shouldn't be null");
-        }
 
         // Projects need to be created first since when adding repositories below,
         // some of the project properties might be needed for that.
@@ -993,14 +979,14 @@ public void prepareIndexer(RuntimeEnvironment env,
             }
         }
         
-        if (searchRepositories || listRepoPaths || !zapCache.isEmpty()) {
+        if (searchRepositories || listRepoPaths) {
             LOGGER.log(Level.INFO, "Scanning for repositories...");
             Statistics stats = new Statistics();
             env.setRepositories(env.getSourceRootPath());
             stats.report(LOGGER, String.format("Done scanning for repositories, found %d repositories",
                     env.getRepositories().size()));
             
-            if (listRepoPaths || !zapCache.isEmpty()) {
+            if (listRepoPaths) {
                 List<RepositoryInfo> repos = env.getRepositories();
                 String prefix = env.getSourceRootPath();
                 if (listRepoPaths) {
@@ -1014,33 +1000,7 @@ public void prepareIndexer(RuntimeEnvironment env,
                         System.out.println(String.format("%s (%s)", dir.substring(prefix.length()), info.getType()));
                     }
                 }
-                if (!zapCache.isEmpty()) {
-                    HashSet<String> toZap = new HashSet<>(zapCache.size() << 1);
-                    boolean all = false;
-                    for (String repo : zapCache) {
-                        if ("*".equals(repo)) {
-                            all = true;
-                            break;
-                        }
-                        if (repo.startsWith(prefix)) {
-                            repo = repo.substring(prefix.length());
-                        }
-                        toZap.add(repo);
-                    }
-                    if (all) {
-                        toZap.clear();
-                        for (RepositoryInfo info : env.getRepositories()) {
-                            toZap.add(info.getDirectoryName()
-                                    .substring(prefix.length()));
-                        }
-                    }
-                    try {
-                        HistoryGuru.getInstance().removeCache(toZap);
-                    } catch (HistoryException e) {
-                        LOGGER.log(Level.WARNING, "Clearing history cache failed: {0}",
-                                e.getLocalizedMessage());
-                    }
-                }
+
                 return;
             }
         }
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java
@@ -85,7 +85,7 @@ public static void setUpClass() throws Exception {
         IndexChangedListener progress = new DefaultIndexChangedListener();
         Indexer.getInstance().prepareIndexer(env, true, true,
                 new TreeSet<>(Collections.singletonList("/c")),
-                false, false, null, null, new ArrayList<>(), false);
+                false, false, null, null, false);
 
         Indexer.getInstance().doIndexerExecution(true, null, progress);
 
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java
@@ -88,7 +88,7 @@ public static void setUpClass() throws Exception {
         Indexer indexer = Indexer.getInstance();
         indexer.prepareIndexer(
                 env, true, true, new TreeSet<>(Arrays.asList(new String[]{"/c"})),
-                false, false, null, null, new ArrayList<String>(), false);
+                false, false, null, null, false);
         indexer.doIndexerExecution(true, null, null);
     }
 
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexVersionTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexVersionTest.java
@@ -91,7 +91,7 @@ private void testIndexVersion(boolean projectsEnabled, List<String> subFiles) th
         env.setHistoryEnabled(false);
         env.setProjectsEnabled(projectsEnabled);
         Indexer.getInstance().prepareIndexer(env, true, true, null,
-                false, false, null, null, new ArrayList<>(), false);
+                false, false, null, null, false);
         Indexer.getInstance().doIndexerExecution(true, null, null);
 
         IndexVersion.check(subFiles);
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerRepoTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerRepoTest.java
@@ -143,7 +143,6 @@ private void testPerProjectHistory(boolean globalOn) throws IndexerException, IO
                 false, // don't create dictionary
                 null, // subFiles - not needed since we don't list files
                 null, // repositories - not needed when not refreshing history
-                new ArrayList<>(), // don't zap cache
                 false); // don't list repos
         
         File repoRoot = new File(env.getSourceRootFile(), "git");
@@ -209,7 +208,6 @@ public void testSymlinks() throws IndexerException, IOException {
                 false, // don't create dictionary
                 null, // subFiles - not needed since we don't list files
                 null, // repositories - not needed when not refreshing history
-                new ArrayList<>(), // don't zap cache
                 false); // don't list repos
 
         // Check the respository paths.
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java
@@ -121,7 +121,7 @@ public void testIndexGeneration() throws Exception {
         env.setDataRoot(repository.getDataRoot());
         env.setHistoryEnabled(false);
         Indexer.getInstance().prepareIndexer(env, true, true, new TreeSet<>(Collections.singletonList("/c")),
-                false, false, null, null, new ArrayList<>(), false);
+                false, false, null, null, false);
         Indexer.getInstance().doIndexerExecution(true, null, null);
     }
 
@@ -159,7 +159,6 @@ public void testRescanProjects() throws Exception {
                 false, // don't create dictionary
                 null, // subFiles - not needed since we don't list files
                 null, // repositories - not needed when not refreshing history
-                new ArrayList<>(), // don't zap cache
                 false); // don't list repos
 
         List<Project> newProjects = env.getProjectList();
@@ -487,7 +486,7 @@ public void testDefaultProjectsSingleProject() throws Exception {
         Indexer.getInstance().prepareIndexer(env, true, true, new TreeSet<>(Arrays.asList(new String[]{
                         Paths.get("/c").toString()
                 })),
-                false, false, null, null, new ArrayList<>(), false);
+                false, false, null, null,false);
         assertEquals(1, env.getDefaultProjects().size());
         assertEquals(new TreeSet<>(Arrays.asList(new String[]{"c"})),
                 env.getDefaultProjects().stream().map((Project p) -> p.getName()).collect(Collectors.toSet()));
@@ -513,7 +512,7 @@ public void testDefaultProjectsNonExistent() throws Exception {
                         Paths.get("/data").toString(),
                         Paths.get("/no-project-x32ds1").toString()
                 })),
-                false, false, null, null, new ArrayList<>(), false);
+                false, false, null, null, false);
         assertEquals(4, env.getDefaultProjects().size());
         assertEquals(new TreeSet<>(Arrays.asList(new String[]{"lisp", "pascal", "perl", "data"})),
                 env.getDefaultProjects().stream().map((Project p) -> p.getName()).collect(Collectors.toSet()));
@@ -537,7 +536,7 @@ public void testDefaultProjectsAll() throws Exception {
                         Paths.get("__all__").toString(),
                         Paths.get("/no-project-x32ds1").toString()
                 })),
-                false, false, null, null, new ArrayList<>(), false);
+                false, false, null, null, false);
         Set<String> projects = new TreeSet<>(Arrays.asList(new File(repository.getSourceRoot()).list()));
         assertEquals(projects.size(), env.getDefaultProjects().size());
         assertEquals(projects, env.getDefaultProjects().stream().map((Project p) -> p.getName()).collect(Collectors.toSet()));
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java
@@ -75,7 +75,7 @@ public static void setUpClass() throws Exception {
         env.setHistoryEnabled(false);
         Indexer.getInstance().prepareIndexer(env, true, true,
                 new TreeSet<>(Collections.singletonList("/c")),
-                false, false, null, null, new ArrayList<>(), false);
+                false, false, null, null, false);
         Indexer.getInstance().doIndexerExecution(true, null, null);
 
 
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java
@@ -89,7 +89,7 @@ public static void setUpClass() throws Exception {
         env.setHistoryEnabled(false);
         Indexer.getInstance().prepareIndexer(env, true, true,
                 new TreeSet<>(Collections.singletonList("/c")),
-                false, false, null, null, new ArrayList<>(), false);
+                false, false, null, null, false);
         Indexer.getInstance().doIndexerExecution(true, null, null);
 
         configFile = File.createTempFile("configuration", ".xml");
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java
@@ -128,7 +128,7 @@ public static void setUpClass() throws Exception {
         env.setHistoryEnabled(false);
         Indexer.getInstance().prepareIndexer(env, true, true,
                 new TreeSet<>(Collections.singletonList("/c")),
-                false, false, null, null, new ArrayList<>(), false);
+                false, false, null, null, false);
 
         Project proj1 = env.getProjects().get(SYMLINK1);
         assertNotNull("symlink1 project", proj1);
diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java
@@ -76,7 +76,7 @@ private void reindex() throws Exception {
 
         Indexer.getInstance().prepareIndexer(env, true, true,
             new TreeSet<>(Collections.singletonList("/c")),
-            false, false, null, null, new ArrayList<>(), false);
+            false, false, null, null, false);
         Indexer.getInstance().doIndexerExecution(true, null, null);
     }
 
diff --git a/opengrok-web/src/main/java/org/opengrok/web/api/v1/controller/ProjectsController.java b/opengrok-web/src/main/java/org/opengrok/web/api/v1/controller/ProjectsController.java
@@ -193,7 +193,22 @@ public void deleteProjectData(@PathParam("project") String projectName) throws H
             }
         }
 
-        // Delete history index.
+        deleteHistoryCache(projectName);
+
+        // Delete suggester data.
+        suggester.delete(projectName);
+    }
+
+    @DELETE
+    @Path("/{project}/historycache")
+    public void deleteHistoryCache(@PathParam("project") String projectName) throws HistoryException {
+
+        Project project = disableProject(projectName);
+        logger.log(Level.INFO, "deleting history cache for project {0}", projectName);
+
+        List<RepositoryInfo> repos = env.getProjectRepositoriesMap().get(project);
+
+        // Delete history cache data.
         HistoryGuru guru = HistoryGuru.getInstance();
         guru.removeCache(repos.stream().
                 map(x -> {
@@ -203,16 +218,13 @@ public void deleteProjectData(@PathParam("project") String projectName) throws H
                         logger.log(Level.FINER, e.getMessage());
                         return "";
                     } catch (IOException e) {
-                        logger.log(Level.INFO, "cannot remove files for repository {0}", x.getDirectoryName());
+                        logger.log(Level.WARNING, "cannot remove files for repository {0}", x.getDirectoryName());
                         // Empty output should not cause any harm
                         // since {@code getReposFromString()} inside
                         // {@code removeCache()} will return nothing.
                         return "";
                     }
                 }).filter(x -> !x.isEmpty()).collect(Collectors.toSet()));
-
-        // Delete suggester data.
-        suggester.delete(projectName);
     }
 
     @PUT
diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java
@@ -314,7 +314,6 @@ public void testDelete() throws Exception {
                 false, // don't create dictionary
                 subFiles, // subFiles - needed when refreshing history partially
                 repos, // repositories - needed when refreshing history partially
-                new ArrayList<>(), // don't zap cache
                 false); // don't list repos
         Indexer.getInstance().doIndexerExecution(true, null, null);
 
diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/RepositoriesControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/RepositoriesControllerTest.java
@@ -96,7 +96,7 @@ public void testGetRepositoryTypeOfNonExistenRepository() throws Exception {
 
     @Test
     public void testGetRepositoryType() throws Exception {
-        // Create subrepository.
+        // Create sub-repository.
         File mercurialRoot = new File(repository.getSourceRoot() + File.separator + "mercurial");
         MercurialRepositoryTest.runHgCommand(mercurialRoot,
                 "clone", mercurialRoot.getAbsolutePath(),
@@ -112,7 +112,6 @@ public void testGetRepositoryType() throws Exception {
                 false, // don't create dictionary
                 null, // subFiles - needed when refreshing history partially
                 null, // repositories - needed when refreshing history partially
-                new ArrayList<>(), // don't zap cache
                 false); // don't list repos
 
         assertEquals(Paths.get("/mercurial").toString() + ":Mercurial",
diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java
@@ -80,7 +80,7 @@ public static void setUpClass() throws Exception {
 
         Indexer.getInstance().prepareIndexer(env, true, true,
                 Collections.singleton("__all__"),
-                false, false, null, null, new ArrayList<>(), false);
+                false, false, null, null, false);
         Indexer.getInstance().doIndexerExecution(true, null, null);
 
         env.getSuggesterConfig().setRebuildCronConfig(null);
diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java
@@ -112,7 +112,7 @@ public static void setUpClass() throws Exception {
         env.setProjectsEnabled(true);
         Indexer.getInstance().prepareIndexer(env, true, true,
                 Collections.singleton("__all__"),
-                false, false, null, null, new ArrayList<>(), false);
+                false, false, null, null, false);
         Indexer.getInstance().doIndexerExecution(true, null, null);
 
         env.getSuggesterConfig().setRebuildCronConfig(null);

Original file line number	Diff line number	Diff line change
`@@ -88,7 +88,7 @@ public static void setUpClass() throws Exception {`
`88`	`88`	`Indexer indexer = Indexer.getInstance();`
`89`	`89`	`indexer.prepareIndexer(`
`90`	`90`	`env, true, true, new TreeSet<>(Arrays.asList(new String[]{"/c"})),`
`91`		`- false, false, null, null, new ArrayList<String>(), false);`
	`91`	`+ false, false, null, null, false);`
`92`	`92`	`indexer.doIndexerExecution(true, null, null);`
`93`	`93`	`}`
`94`	`94`