use opengrok-sync

fixes #3221

Author: Vladimir Kotal

docker/README.md

@@ -36,6 +36,8 @@ The mirroring step works by going through all projects and attempting to
 synchronize all its repositories (e.g. it will do `git pull` for Git
 repositories).
 
+Projects are enabled in this setup and there is no way how to change that.
+
 ### Indexer logs
 
 The indexer/mirroring is set so that it does not log into files.

docker/start.sh

@@ -40,16 +40,24 @@ fi
 
 URI="http://localhost:8080/${URL_ROOT}"
 OPS=${INDEXER_FLAGS:='-H -P -S -G'}
-BODY_INCLUDE_FILE="/opengrok/data/body_include"
+
+OPENGROK_BASE_DIR="/opengrok"
+OPENGROK_DATA_ROOT="$OPENGROK_BASE_DIR/data"
+OPENGROK_SRC_ROOT="$OPENGROK_BASE_DIR/src"
+BODY_INCLUDE_FILE="$OPENGROK_DATA_ROOT/body_include"
+OPENGROK_CONFIG_FILE="$OPENGROK_BASE_DIR/etc/configuration.xml"
+
+export OPENGROK_INDEXER_OPTIONAL_ARGS=$INDEXER_OPT
+export OPENGROK_NO_MIRROR=$NOMIRROR
 
 function deploy {
 	if [[ ! -f "/usr/local/tomcat/webapps/${WAR_NAME}" ]]; then
 		date +"%F %T Deployment path does not exist. Deploying..."
 
 		# Delete old deployment and (re)deploy.
 		rm -rf /usr/local/tomcat/webapps/*
-		opengrok-deploy -c /opengrok/etc/configuration.xml \
-	            /opengrok/lib/source.war "/usr/local/tomcat/webapps/${WAR_NAME}"
+		opengrok-deploy -c "$OPENGROK_CONFIG_FILE" \
+	            "$OPENGROK_BASE_DIR/lib/source.war" "/usr/local/tomcat/webapps/${WAR_NAME}"
 
 		# Set up redirect from /source
 		mkdir "/usr/local/tomcat/webapps/source"
@@ -64,42 +72,62 @@ function bare_config {
 	fi
 	echo '<p><h1>Waiting on the initial reindex to finish.. Stay tuned !</h1></p>' > "$BODY_INCLUDE_FILE"
 
+  echo "Creating bare configuration"
 	opengrok-indexer \
 	    -a /opengrok/lib/opengrok.jar -- \
-	    -s /opengrok/src \
-	    -d /opengrok/data \
+	    -s "$OPENGROK_SRC_ROOT" \
+	    -d "$OPENGROK_DATA_ROOT" \
 	    --remote on \
-	    -W /opengrok/etc/configuration.xml \
+	    -P -H \
+	    -W "$OPENGROK_CONFIG_FILE" \
 	    --noIndex
+  echo "Done creating bare config"
 }
 
 function wait_for_tomcat {
-	# Wait for Tomcat startup.
-	date +"%F %T Waiting for Tomcat startup..."
-	while [ "`curl --silent --write-out '%{response_code}' -o /dev/null \"http://localhost:8080/${URL_ROOT}\"`" == "000" ]; do
+	# Wait for Tomcat startup and web app deploy.
+	date +"%F %T Waiting for Tomcat startup and web app deployment..."
+	while [ "`curl --silent --write-out '%{response_code}' -o /dev/null \"http://localhost:8080/${URL_ROOT}\"`" != "200" ]; do
 		sleep 1;
 	done
-	date +"%F %T Tomcat startup finished"
+	date +"%F %T Tomcat startup finished and web app deployed"
+}
+
+function save_config {
+  echo "Saving configuration"
+  # Note: URI ends with a slash
+  # TODO: check result of the API call and move only if successful
+  curl -s -o "$OPENGROK_CONFIG_FILE" -X GET "${URI}api/v1/configuration"
+  echo "Done saving configuration"
 }
 
+function add_projects {
+  # Add each directory under source root as a project.
+  # For https://github.com/oracle/opengrok/issues/3403 this should be replaced
+  # with query to get all projects and add only those that are not already present.
+  echo "Adding projects"
+  find $OPENGROK_SRC_ROOT/* -maxdepth 0 -type d -print | xargs -n 1 basename | \
+    while read dir; do
+      echo "Adding $dir"
+      # Note: URI ends with a slash
+      curl -s -X POST -H 'Content-Type: text/plain' -d "$dir" "${URI}api/v1/projects"
+    done
+  echo "Done adding projects"
+}
+
+# Perform mirroring and indexing of all projects.
 function data_sync {
-	if [ -z $NOMIRROR ]; then
-		date +"%F %T Mirroring starting"
-		opengrok-mirror --all --uri "$URI"
-		date +"%F %T Mirroring finished"
-	fi
+  add_projects
 
-	date +"%F %T Indexing starting"
-	opengrok-indexer \
-	    -a /opengrok/lib/opengrok.jar -- \
-	    -s /opengrok/src \
-	    -d /opengrok/data \
-	    --remote on \
-	    -W /opengrok/etc/configuration.xml \
-	    -U "$URI" \
-	    $OPS \
-	    $INDEXER_OPT "$@"
-	date +"%F %T Indexing finished"
+	date +"%F %T Sync starting"
+	# TODO: $URI vs sync.yml
+	opengrok-sync -U "$URI" --driveon --config /scripts/sync.yml
+
+	# Workaround for https://github.com/oracle/opengrok/issues/1670
+	touch $OPENGROK_DATA_ROOT/timestamp
+
+	date +"%F %T Sync finished"
+	save_config
 }
 
 function indexer {
@@ -108,18 +136,21 @@ function indexer {
 	while true; do
 		data_sync
 
-		# If this was a case of initial indexing, move the include away.
+		# If this was the case of initial indexing, move the include away.
 		rm -f "$BODY_INCLUDE_FILE"
 		if [[ -f $BODY_INCLUDE_FILE.orig ]]; then
 			mv "$BODY_INCLUDE_FILE.orig" "$BODY_INCLUDE_FILE"
 		fi
 
-		# Index every $REINDEX minutes.
+		# Want to reload the includes anyway in case the user modified them.
+		curl -s -X PUT "${URI}api/v1/system/includes/reload"
+
+		# Sync every $REINDEX minutes.
 		if [ "$REINDEX" == "0" ]; then
-			date +"%F %T Automatic reindexing disabled"
+			date +"%F %T Automatic sync disabled"
 			return
 		else
-			date +"%F %T Automatic reindexing in $REINDEX minutes..."
+			date +"%F %T Automatic sync in $REINDEX minutes..."
 		fi
 		sleep `expr 60 \* $REINDEX`
 	done
@@ -131,7 +162,7 @@ deploy
 # Create empty configuration to avoid the non existent file exception
 # during the first web app startup.
 #
-if [[ ! -f /opengrok/etc/configuration.xml ]]; then
+if [[ ! -f "$OPENGROK_CONFIG_FILE" ]]; then
 	bare_config
 fi
 

tools/src/main/python/opengrok_tools/indexer.py

@@ -95,7 +95,15 @@ def main():
     lock = FileLock(os.path.join(tempfile.gettempdir(), lockfile + ".lock"))
     try:
         with lock.acquire(timeout=0):
-            indexer = Indexer(args.options, logger=logger, java=args.java,
+            optional_args = os.environ.get("OPENGROK_INDEXER_OPTIONAL_ARGS")
+            options = args.options
+            if optional_args and len(optional_args) > 0 and options:
+                logger.debug("adding optional indexer arguments: {}".
+                             format(optional_args))
+                if options:
+                    options.extend(optional_args)
+
+            indexer = Indexer(options, logger=logger, java=args.java,
                               jar=args.jar, java_opts=args.java_opts,
                               env_vars=args.environment, doprint=doprint)
             indexer.execute()

tools/src/main/python/opengrok_tools/mirror.py

@@ -93,6 +93,8 @@ def main():
                         help='uri of the webapp with context path')
     parser.add_argument('-b', '--batch', action='store_true',
                         help='batch mode - will log into a file')
+    parser.add_argument('-L', '--logdir',
+                        help='log directory')
     parser.add_argument('-B', '--backupcount', default=8,
                         help='how many log files to keep around in batch mode')
     parser.add_argument('-I', '--check-changes', action='store_true',
@@ -110,6 +112,12 @@ def main():
 
     logger = get_console_logger(get_class_basename(), args.loglevel)
 
+    nomirror = os.environ.get("OPENGROK_NO_MIRROR")
+    if nomirror and len(nomirror) > 0:
+        logger.debug("skipping mirror based on the OPENGROK_NO_MIRROR " +
+                     "environment variable")
+        return SUCCESS_EXITVAL
+
     if len(args.project) > 0 and args.all:
         return fatal("Cannot use both project list and -a/--all", False)
 
@@ -156,10 +164,13 @@ def main():
     logdir = None
     # Log messages to dedicated log file if running in batch mode.
     if args.batch:
-        logdir = config.get(LOGDIR_PROPERTY)
-        if not logdir:
-            return fatal("The {} property is required in batch mode".
-                         format(LOGDIR_PROPERTY), False)
+        if args.logdir:
+            logdir = args.logdir
+        else:
+            logdir = config.get(LOGDIR_PROPERTY)
+            if not logdir:
+                return fatal("The {} property is required in batch mode".
+                             format(LOGDIR_PROPERTY), False)
 
     projects = args.project
     if len(projects) == 1:

tools/src/main/python/opengrok_tools/reindex_project.py

@@ -75,50 +75,61 @@ def get_config_file(logger, uri):
 def main():
     parser = argparse.ArgumentParser(description='OpenGrok indexer wrapper '
                                                  'for indexing single project',
-                                     parents=[get_java_parser()])
-    parser.add_argument('-t', '--template', required=True,
+                                     parents=[get_java_parser()],
+                                     prog=sys.argv[0])
+    parser.add_argument('-t', '--template',
                         help='Logging template file')
-    parser.add_argument('-p', '--pattern', required=True,
+    parser.add_argument('-p', '--pattern',
                         help='Pattern to substitute in logging template with'
                              'project name')
     parser.add_argument('-P', '--project', required=True,
                         help='Project name')
-    parser.add_argument('-d', '--directory', required=True,
+    parser.add_argument('-d', '--directory',
                         help='Logging directory')
     parser.add_argument('-U', '--uri', default='http://localhost:8080/source',
                         help='URI of the webapp with context path')
 
+    cmd_args = sys.argv[1:]
+    extra_opts = os.environ.get("OPENGROK_INDEXER_OPTIONAL_ARGS")
+    if extra_opts:
+        cmd_args.extend(extra_opts.split())
+
     try:
-        args = parser.parse_args()
+        args = parser.parse_args(cmd_args)
     except ValueError as e:
         fatal(e)
 
     logger = get_console_logger(get_class_basename(), args.loglevel)
 
     # Make sure the log directory exists.
-    if not os.path.isdir(args.directory):
-        os.makedirs(args.directory)
+    if args.directory:
+        if not os.path.isdir(args.directory):
+            os.makedirs(args.directory)
 
     # Get files needed for per-project reindex.
     conf_file = get_config_file(logger, args.uri)
-    logprop_file = get_logprop_file(logger, args.template, args.pattern,
-                                    args.project)
+    logprop_file = None
+    if args.template and args.pattern:
+        logprop_file = get_logprop_file(logger, args.template, args.pattern,
+                                        args.project)
 
     # Reindex with the modified logging.properties file and read-only config.
     command = ['-R', conf_file]
     command.extend(args.options)
     java_opts = []
     if args.java_opts:
         java_opts.extend(args.java_opts)
-    java_opts.append("-Djava.util.logging.config.file={}".
-                     format(logprop_file))
+    if logprop_file:
+        java_opts.append("-Djava.util.logging.config.file={}".
+                         format(logprop_file))
     indexer = Indexer(command, logger=logger, jar=args.jar,
                       java=args.java, java_opts=java_opts,
                       env_vars=args.environment, doprint=args.doprint)
     indexer.execute()
     ret = indexer.getretcode()
     os.remove(conf_file)
-    os.remove(logprop_file)
+    if logprop_file:
+        os.remove(logprop_file)
     if ret is None or ret != SUCCESS_EXITVAL:
         logger.error(indexer.getoutputstr())
         logger.error("Indexer command for project {} failed (return code {})".

tools/src/main/python/opengrok_tools/sync.py

@@ -90,13 +90,15 @@ def do_sync(args, commands, config, directory, dirs_to_process, ignore_errors,
         for entry in os.listdir(directory):
             if path.isdir(path.join(directory, entry)):
                 dirs_to_process.append(entry)
+
     logger.debug("to process: {}".format(dirs_to_process))
     cmds_base = []
     for d in dirs_to_process:
         cmd_base = CommandSequenceBase(d, commands, args.loglevel,
                                        config.get("cleanup"),
                                        args.driveon)
         cmds_base.append(cmd_base)
+
     # Map the commands into pool of workers so they can be processed.
     retval = SUCCESS_EXITVAL
     with Pool(processes=int(args.workers)) as pool:

tools/src/main/python/opengrok_tools/utils/commandsequence.py

@@ -149,11 +149,22 @@ def run(self):
                                               format(command, self.name))
                             continue
                     else:
-                        self.logger.error("command '{}' for project {} failed "
-                                          "with code {}, breaking".
-                                          format(command, self.name, retcode))
-                        self.failed = True
-                        self.run_cleanup()
+                        if self.driveon:
+                            do = "driving on"
+                            self.logger.debug("command '{}' for project '{}'' failed "
+                                              "with code {}, {}".
+                                              format(command, self.name, retcode, do))
+                        else:
+                            do = "breaking"
+                            self.logger.error("command '{}' for project '{}'' failed "
+                                              "with code {}, {}".
+                                              format(command, self.name, retcode, do))
+
+                        if self.driveon:
+                            continue
+                        else:
+                            self.failed = True
+                            self.run_cleanup()
 
                     break