allow -S to specify search path

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java

@@ -32,6 +32,7 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashSet;
@@ -790,11 +791,11 @@ public void removeRepositories() {
      * Search through the directory for repositories and use the result to replace
      * the lists of repositories in both RuntimeEnvironment/Configuration and HistoryGuru.
      *
-     * @param dir the root directory to start the search in
+     * @param dir the directories to start the search in
      */
-    public void setRepositories(String dir) {
+    public void setRepositories(String... dir) {
         List<RepositoryInfo> repos = new ArrayList<>(HistoryGuru.getInstance().
-                addRepositories(new File[]{new File(dir)},
+                addRepositories(Arrays.stream(dir).map(File::new).toArray(File[]::new),
                     RuntimeEnvironment.getInstance().getIgnoredNames()));
         RuntimeEnvironment.getInstance().setRepositories(repos);
     }

opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java

@@ -35,6 +35,7 @@
 import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -118,6 +119,7 @@ public final class Indexer {
     private static int status = 0;
 
     private static final Set<String> repositories = new HashSet<>();
+    private static Set<String> searchPaths = new HashSet<>();
     private static final HashSet<String> allowedSymlinks = new HashSet<>();
     private static final HashSet<String> canonicalRoots = new HashSet<>();
     private static final Set<String> defaultProjects = new TreeSet<>();
@@ -335,7 +337,16 @@ public static void main(String[] argv) {
                     new Object[]{Info.getVersion(), Info.getRevision()});
 
             // Create history cache first.
-            getInstance().prepareIndexer(env, searchRepositories, addProjects,
+            if (searchRepositories) {
+                if (searchPaths.isEmpty()) {
+                    searchPaths.add(env.getSourceRootPath());
+                } else {
+                    searchPaths = searchPaths.stream().
+                            map(t -> Paths.get(env.getSourceRootPath(), t).toString()).
+                            collect(Collectors.toSet());
+                }
+            }
+            getInstance().prepareIndexer(env, searchPaths, addProjects,
                     createDict, runIndex, subFiles, new ArrayList<>(repositories));
 
             // prepareIndexer() populated the list of projects so now default projects can be set.
@@ -417,6 +428,8 @@ public static String[] parseOptions(String[] argv) throws ParseException {
             });
         });
 
+        searchPaths.clear();
+
         // Limit usage lines to 72 characters for concise formatting.
 
         optParser = OptionParser.Do(parser -> {
@@ -689,9 +702,16 @@ public static String[] parseOptions(String[] argv) throws ParseException {
                     "are history-eligible; using --repository limits to only those specified.",
                     "Option may be repeated.").Do(v -> repositories.add((String) v));
 
-            parser.on("-S", "--search",
-                    "Search for source repositories under -s,--source, and add them.").Do(v ->
-                    searchRepositories = true);
+            parser.on("-S", "--search", "=[path/to/repository]",
+                    "Search for source repositories under -s,--source, and add them." +
+                    "Path (relative to the source root) to repository is optional. " +
+                    "Option may be repeated.").Do(v -> {
+                        searchRepositories = true;
+                        String repoPath = (String) v;
+                        if (!repoPath.isEmpty()) {
+                            searchPaths.add(repoPath);
+                        }
+                    });
 
             parser.on("-s", "--source", "=/path/to/source/root",
                 "The root directory of the source tree.").
@@ -873,8 +893,9 @@ public void prepareIndexer(RuntimeEnvironment env,
                                List<String> subFiles,
                                List<String> repositories) throws IndexerException, IOException {
 
-        prepareIndexer(env, searchRepositories, addProjects, createDict, true,
-                subFiles, repositories);
+        prepareIndexer(env,
+                searchRepositories ? Collections.singleton(env.getSourceRootPath()) : Collections.emptySet(),
+                addProjects, createDict, true, subFiles, repositories);
     }
 
     /**
@@ -888,7 +909,7 @@ public void prepareIndexer(RuntimeEnvironment env,
      * for performance. We prefer clarity over performance here, so silence it.
      *
      * @param env runtime environment
-     * @param searchRepositories if true, search for repositories
+     * @param searchPaths list of paths in which to search for repositories
      * @param addProjects if true, add projects
      * @param createDict if true, create dictionary
      * @param createHistoryCache create history cache flag
@@ -899,7 +920,7 @@ public void prepareIndexer(RuntimeEnvironment env,
      */
     @SuppressWarnings("PMD.SimplifyStartsWith")
     public void prepareIndexer(RuntimeEnvironment env,
-            boolean searchRepositories,
+            Set<String> searchPaths,
             boolean addProjects,
             boolean createDict,
             boolean createHistoryCache,
@@ -954,10 +975,10 @@ public void prepareIndexer(RuntimeEnvironment env,
             }
         }
 
-        if (searchRepositories) {
-            LOGGER.log(Level.INFO, "Scanning for repositories...");
+        if (!searchPaths.isEmpty()) {
+            LOGGER.log(Level.INFO, "Scanning for repositories in {0}...", searchPaths);
             Statistics stats = new Statistics();
-            env.setRepositories(env.getSourceRootPath());
+            env.setRepositories(searchPaths.toArray(new String[0]));
             stats.report(LOGGER, String.format("Done scanning for repositories, found %d repositories",
                     env.getRepositories().size()));
         }

opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerMainTest.java

@@ -0,0 +1,96 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Portions Copyright (c) 2017-2019, Chris Fraire <cfraire@me.com>.
+ */
+
+package org.opengrok.indexer.index;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.opengrok.indexer.configuration.RuntimeEnvironment;
+import org.opengrok.indexer.history.HistoryGuru;
+import org.opengrok.indexer.util.TestRepository;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertFalse;
+
+public class IndexerMainTest {
+    private TestRepository repository;
+
+    @Before
+    public void setUp() throws IOException {
+        repository = new TestRepository();
+        // For these tests we need Mercurial repository with renamed files.
+        repository.create(HistoryGuru.class.getResourceAsStream("repositories.zip"));
+    }
+
+    @After
+    public void tearDown() {
+        repository.destroy();
+    }
+
+    private void checkNumberOfThreads() {
+        /*
+         * There should not be any threads in the renamed pool now.
+         * We need to check it like this since the test framework tears
+         * down the threads at the end of the test case run so any
+         * hangs due to executors not being shut down would not be visible.
+         */
+        ThreadGroup mainGroup = Thread.currentThread().getThreadGroup();
+        Thread[] threads = new Thread[mainGroup.activeCount()];
+        mainGroup.enumerate(threads);
+        for (int i = 0; i < threads.length; i++) {
+            if (threads[i] == null || threads[i].getName() == null) {
+                continue;
+            }
+            assertFalse(threads[i].getName().contains("renamed-handling"));
+        }
+    }
+
+    /**
+     * Test cleanup of renamed thread pool after indexing with -H.
+     */
+    @Test
+    public void testMainWithH() {
+        System.out.println("Generate index by using command line options with -H");
+        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
+        String[] argv = {"-S", "-H", "-s", repository.getSourceRoot(),
+                "-d", repository.getDataRoot(), "-v", "-c", env.getCtags()};
+        Indexer.main(argv);
+        checkNumberOfThreads();
+    }
+
+    /**
+     * Test cleanup of renamed thread pool after indexing without -H.
+     */
+    @Test
+    public void testMainWithoutH() {
+        System.out.println("Generate index by using command line options without -H");
+        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
+        String[] argv = {"-S", "-P", "-s", repository.getSourceRoot(),
+                "-d", repository.getDataRoot(), "-v", "-c", env.getCtags()};
+        Indexer.main(argv);
+        checkNumberOfThreads();
+    }
+}

opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerRepoTest.java

@@ -79,24 +79,6 @@ public void tearDown() {
         repository.destroy();
     }
 
-    private void checkNumberOfThreads() {
-        /*
-         * There should not be any threads in the renamed pool now.
-         * We need to check it like this since the test framework tears
-         * down the threads at the end of the test case run so any
-         * hangs due to executors not being shut down would not be visible.
-         */
-        ThreadGroup mainGroup = Thread.currentThread().getThreadGroup();
-        Thread[] threads = new Thread[mainGroup.activeCount()];
-        mainGroup.enumerate(threads);
-        for (int i = 0; i < threads.length; i++) {
-            if (threads[i] == null || threads[i].getName() == null) {
-                continue;
-            }
-            assertFalse(threads[i].getName().contains("renamed-handling"));
-        }
-    }
-
     /**
      * Test it is possible to disable history per project.
      */
@@ -222,30 +204,4 @@ public void testSymlinks() throws IndexerException, IOException {
         IOUtils.removeRecursive(realSource);
         IOUtils.removeRecursive(sourceRoot);
     }
-
-    /**
-     * Test cleanup of renamed thread pool after indexing with -H.
-     */
-    @Test
-    public void testMainWithH() {
-        System.out.println("Generate index by using command line options with -H");
-        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
-        String[] argv = {"-S", "-H", "-s", repository.getSourceRoot(),
-                "-d", repository.getDataRoot(), "-v", "-c", env.getCtags()};
-        Indexer.main(argv);
-        checkNumberOfThreads();
-    }
-
-    /**
-     * Test cleanup of renamed thread pool after indexing without -H.
-     */
-    @Test
-    public void testMainWithoutH() {
-        System.out.println("Generate index by using command line options without -H");
-        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
-        String[] argv = {"-S", "-P", "-s", repository.getSourceRoot(),
-                "-d", repository.getDataRoot(), "-v", "-c", env.getCtags()};
-        Indexer.main(argv);
-        checkNumberOfThreads();
-    }
 }

opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java

@@ -33,6 +33,7 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.StringWriter;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -350,6 +351,25 @@ public void testRemoveFileOnFileChange() throws Exception {
         testrepo.destroy();
     }
 
+    @Test
+    @ConditionalRun(RepositoryInstalled.MercurialInstalled.class)
+    @ConditionalRun(RepositoryInstalled.GitInstalled.class)
+    public void testSetRepositories() throws Exception {
+        RuntimeEnvironment env = RuntimeEnvironment.getInstance();
+
+        TestRepository testrepo = new TestRepository();
+        testrepo.create(HistoryGuru.class.getResourceAsStream("repositories.zip"));
+        env.setSourceRoot(testrepo.getSourceRoot());
+
+        env.setRepositories(testrepo.getSourceRoot());
+        assertEquals(9, env.getRepositories().size());
+
+        String[] repoNames = {"mercurial", "git"};
+        env.setRepositories(Arrays.stream(repoNames).
+                map(t -> Paths.get(env.getSourceRootPath(), t).toString()).distinct().toArray(String[]::new));
+        assertEquals(2, env.getRepositories().size());
+    }
+
     @Test
     public void testXref() throws IOException {
         List<File> files = new ArrayList<>();

opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java

@@ -291,7 +291,7 @@ public void testDelete() throws Exception {
                 filter(p -> p.getName().equals("git")).
                 collect(Collectors.toSet()).size());
 
-        // Run the indexer (ala 'indexpart') so that data directory is populated.
+        // Run the indexer so that data directory is populated.
         ArrayList<String> subFiles = new ArrayList<>();
         subFiles.add("/git");
         subFiles.add("/mercurial");
@@ -301,7 +301,7 @@ public void testDelete() throws Exception {
         repos.add("/mercurial");
         repos.add("/svn");
         // This is necessary so that repositories in HistoryGuru get populated.
-        // When 'indexpart' is run, this is called from setConfiguration() because
+        // For per project reindex this is called from setConfiguration() because
         // of the -R option is present.
         HistoryGuru.getInstance().invalidateRepositories(
                 env.getRepositories(), null, false);