Indexer should pass CLI cmds to use to webapp (Bugzilla #18104)

[vladak]

status NEW severity major in component webapp for ---
Reported in version unspecified on platform ANY/Generic
Assigned to: Lubos Kosco

Original attachment names and IDs:

• cliwebapp.patch (ID 4505)
• opengrok (ID 4506)

On 2011-03-31 05:17:22 +0000, Jens Elkner wrote:

Right now it is possible to specify the CLI cmds to use to access repository contents, however this information gets not passed to the webapp and thus history/annotations might be not available via web even so the information is already in the DB.

The attached patch takes this into account, makes ScmChecker obsolete, some
other small enhancements wrt. source code readability/lang usage and introduces 2 new CLI options:
-K ... list all repository pathes
-k repositoryPath ... clean the cache for the given repositories

The second attachment is an example script, how we currently use the indexer:
opengrok -c ~/gf/src/opengrok.rc -v -p deploy

On 2011-03-31 05:19:08 +0000, Jens Elkner wrote:

Created attachment 4505
CLI client config pass + misc enhancements

On 2011-03-31 05:21:04 +0000, Jens Elkner wrote:

Created attachment 4506
example indexer script (may help to understand the issue)

[tarzanek]

att 4506:

#!/bin/ksh93

# OpenGrok Wrapper (initial setup and cron job updating) for Glassfish
OPENGROK_DIR=$(_T=`pwd`; cd `dirname ${0}`; cd ..; pwd; cd $_T)
export LC_CTYPE="de_DE.UTF-8" LANG="$LC_CTYPE"

typeset -ft $(typeset +f)

# Initialize the environment for opengrok with reasonable defaults.
SetupEnv() {
    # If set and available the index_body.html in the webarchive (source.war)
    # gets replaced with the given file right before it gets deployed to GF.
    #INDEX_BODY_HTML=

    # REQUIRED: Glassfish domain directory to use
    GF_DOMAINDIR="${GF_DOMAINDIR:-/data/sites/glassfish/v3.1/src}"

    # The owner of the glassfish directory
    GF_OWNER="webservd:webservd"

    # Configuration Address (host:port in conf/web.xml)
    WEBAPP_CONFIG="localhost:2424"

    # REQUIRED: Full Path to Java Installation Root
    JAVA_HOME="${JAVA_HOME:-/usr/jdk/instances/jdk1.6.0}"
    # Enable remote debugging for the current indexer
    #JAVA_DEBUG="-agentlib:jdwp=transport=dt_socket,server=y,address=8010,suspend=y"
    #JAVA_OPTS="-server -Xmx2048m"
    JAVA_OPTS=${JAVA_OPTS:-"-Xmx2048m"}

    # REQUIRED: Full Path to Exuberant CTags
    EXUBERANT_CTAGS="${EXUBERANT_CTAGS:-/local/usr/bin/ctags}"

    # REQUIRED: Base Directory of the OpenGrok User Data Area
    OPENGROK_INSTANCE_BASE="${OPENGROK_INSTANCE_BASE:-/data/opengrok}"

    # file with read only configuration - a temp workaround for bug #327
    # see https://defect.opensolaris.org/bz/show_bug.cgi?id=327
    #READ_XML_CONFIGURATION=

    # How deep should scanning for repos go (default: 3 dirs from SRC_ROOT)
    #OPENGROK_SCAN_DEPTH=3

    # Enable Projects: every dir in SRC_ROOT is considered a separate project
    #OPENGROK_PROJECTS=1

    # Disable History Cache for (remote) Repositories (CVS or SVN).
    # Can be very time demanding, uncomment if not needed.
    #OPENGROK_NOREMOTE_REPOS=1

    # Disable Scan for repositories. Uncomment if needed.
    #OPENGROK_NOSCAN_REPOS=1

    # Set Maximum Indexed Words Per File (default: unlimited).
    # Note, that you might run out of memory, then either increase JVM memory
    # as noted in JAVA_OPTS, or set this limit (if you don't mind opengrok not
    # indexing the rest of the file, once the limit is reached)
    #OPENGROK_MAX_WORDS=100000

    # Store the history cache in Java DB (derby) @localhost:1527 per default,
    # instead of file system (gzipped xml files).
    #OPENGROK_DERBY=1

    # Disable wildcard prefix search query support. Uncomment if needed.
    #OPENGROK_NOWPREFIX=1

    # Ignore these patterns as names of files or directories.
    # Add for each pattern a '-i' and enclose patterns with special characters
    # like ! or * in single quotes and escape special shell characters.
    #OPENGROK_IGNORE="-i dummy -i dummy2"

    # Web Site Look & Feel {default|offwhite|polished}.
    #OPENGROK_SKIN="default"

    # Suppress Progress and Warnings Messages
    #OPENGROK_NON_INTERACTIVE=1

    # Enable Verbose Mode in opengrok.jar
    #OPENGROK_VERBOSE=1

    # Shows progress in % of working through project. It's good to have verbose
    # Mode enabled too, cost of this is one more traversal of the project 
    # before indexing it
    #OPENGROK_PROGRESS=1

    # ------ usually one does not need to touch remaining stuff ------

    # REQUIRED (deploy): Glassfish WAR Target Directory
    OPENGROK_WAR_TARGET="${OPENGROK_WAR_TARGET:-${GF_DOMAINDIR}/autodeploy}"

    # REQUIRED (deploy): Web Archive of OpenGrok (Distribution Location).
    OPENGROK_WAR_SOURCE="${OPENGROK_WAR_SOURCE:-${OPENGROK_DIR}/lib/source.war}"

    # REQUIRED: Source Code/Repository Root
    # (your source code or the root of all repositories)
    SRC_ROOT="${SRC_ROOT:-${OPENGROK_INSTANCE_BASE}/source}"

    # REQUIRED: OpenGrok Generate Data Root (for Lucene index and hypertext
    # cross-references). This area is rebuilt by "update".
    DATA_ROOT="${DATA_ROOT:-${OPENGROK_INSTANCE_BASE}/data}"

    # User Provided Source Path to Description Mapping (Tab Separated Value)
    # The user maintained source of the generated EftarFile file.
    PATH_DESC="${PATH_DESC:-${OPENGROK_INSTANCE_BASE}/etc/paths.tsv}"

    # REQUIRED: XML Configuration (the config used by Web/GUI interfaces)
    XML_CONFIGURATION="${XML_CONFIGURATION:-${OPENGROK_INSTANCE_BASE}/etc/configuration.xml}"

    # REQUIRED: Logger Configuration
    LOGGER_CONFIG_FILE="${LOGGER_CONFIG_FILE:-logging.properties}"
    LOGGER_CONFIG_PATH="${LOGGER_CONFIG_PATH:-${OPENGROK_INSTANCE_BASE}/${LOGGER_CONFIG_FILE}}"

    # template to use to create $LOGGER_CONFIG_PATH when it is n/a
    LOGGER_CONF_SOURCE="${LOGGER_CONF_SOURCE:-${OPENGROK_DIR}/doc/logging.properties}"

    # REQUIRED: Java Archive of OpenGrok (Installation Location)
    OPENGROK_JAR="${OPENGROK_JAR:-${OPENGROK_DIR}/lib/opengrok.jar}"

    # Full Path to History Utilities
    HG="${HG:-`Which hg`}"
    CVS="${CVS:-`Which cvs`}"
    SVN="${SVN:-`Which svn`}"
    SCCS="${SCCS:-`Which sccs`}"
    CLEARCASE="${CLEARCASE:-`Which cleartool`}"
    GIT="${GIT:-`Which git`}"
    P4="${P4:-`Which p4`}"
    MTN="${MTN:-`Which mtn`}"
    BZR="${BZR:-`Which bzr`}"

    # EftarFile == An Extremely Fast Tagged Attribute Read-only File System
    EFTAR_UPDATE="${EFTAR_UPDATE:-org.opensolaris.opengrok.web.EftarFile}"

    # HARDCODED: Generated EftarFile (See web/*.jsp)
    EFTAR_OUTPUT_FILE="${EFTAR_OUTPUT_FILE:-${DATA_ROOT}/index/dtags.eftar}"
}

#
# Helper Functions
#
Progress() {
    [ -z "${OPENGROK_NON_INTERACTIVE}" ] && print "${@}"
}

Warning() {
    [ -z "${OPENGROK_NON_INTERACTIVE}" ] && print -u2 "WARNING: ${@}"
}

Error() {
    print -u2 "ERROR: ${@}"
}

FatalError() {
    print -u2 "\nFATAL ERROR: ${@} - Aborting!"
    ${DO} exit 2
}

#
# Helper Functions - Autodetection of Runtime Environment
#

Which() {
    whence -p $1 2>/dev/null
}

LoadInstanceConfiguration() {
    if [ -n "$1" -a -r "$1" ]; then
        # Load the Local OpenGrok Configuration Environment
        Progress "Loading $1 ..."
        . "$1"
    else
        Usage
    fi
}

ValidateConfiguration() {
    # REQUIRED: Java Virtual Machine
    export JAVA_HOME
    if [ -z "$JAVA_HOME" ]; then
        if [ -n "$JAVA:" -a -x "$JAVA" ]; then
            :
        else
            JAVA=$(Which java)
        fi
        if [ -n "$JAVA:" -a -x "$JAVA" ]; then
            JAVA_HOME=${JAVA%/*}
            JAVA_HOME=${JAVA%/*}
        else
            FatalError "java not found."
        fi
    fi
    JAVA="${JAVA:-$JAVA_HOME/bin/java}"
    JAR="${JAR:-$JAVA_HOME/bin/jar}"

    VERBOSE="" ; QUIET="" ; PROGRESS="" ; DERBY_HISTORY_CACHE=""
    SCAN_DEPTH="" ; READ_XML_CONF="" ; ENABLE_PROJECTS="" ; SKIN=""
    IGNORE_PATTERNS="" ; MAX_INDEXED_WORDS=""

    [ -n "${OPENGROK_VERBOSE}" ] && VERBOSE="-v" && QUIET=""
    [ -n "${OPENGROK_PROGRESS}" ] && PROGRESS="-C"
    [ -n "${OPENGROK_DERBY}" ] && DERBY_HISTORY_CACHE="-D"
    [ -n "${OPENGROK_SCAN_DEPTH}" ] && SCAN_DEPTH="-z${OPENGROK_SCAN_DEPTH}"
    [ -n "${OPENGROK_PROJECTS}" ] && ENABLE_PROJECTS="-P"
    [ -n "${OPENGROK_IGNORE}" ] && IGNORE_PATTERNS="-i $OPENGROK_IGNORE"
    [ -n "${OPENGROK_MAX_WORDS}" ] && MAX_INDEXED_WORDS="-m $OPENGROK_MAX_WORDS"
    LOGGER_PROPERTIES="-Djava.util.logging.config.file=${LOGGER_CONFIG_PATH}"
    WEBAPP_CONFIG_ADDRESS="-U "${WEBAPP_CONFIG:-"localhost:2424"}

    # We should not set properties to the empty string
    PROPERTIES="\
${HG:+-Dorg.opensolaris.opengrok.history.Mercurial=$HG} \
${CVS:+-Dorg.opensolaris.opengrok.history.cvs=$CVS} \
${SVN:+-Dorg.opensolaris.opengrok.history.Subversion=$SVN} \
${SCCS:+-Dorg.opensolaris.opengrok.history.SCCS=$SCCS} \
${CLEARCASE:+-Dorg.opensolaris.opengrok.history.ClearCase=$CLEARCASE} \
${GIT:+-Dorg.opensolaris.opengrok.history.git=$GIT} \
${P4:+-Dorg.opensolaris.opengrok.history.Perforce=$P4} \
${MTN:+-Dorg.opensolaris.opengrok.history.Monotone=$MTN} \
${BZR:+-Dorg.opensolaris.opengrok.history.Bazaar=$BZR} \
"
    [ -n "$READ_XML_CONFIGURATION" -a -r "${READ_XML_CONFIGURATION}" ] && \
    READ_XML_CONF="-R ${READ_XML_CONFIGURATION}"

    LEADING_WILDCARD="-a on"
    [ -n "${OPENGROK_NOWPREFIX}" ] && LEADING_WILDCARD=""

    REMOTE_REPOSITORIES="-r on"
    [ -n "${OPENGROK_NOREMOTE_REPOS}" ] && REMOTE_REPOSITORIES=""

    SCAN_FOR_REPOSITORY="-S"
    [ -n "${OPENGROK_NOSCAN_REPOS}" ] && SCAN_FOR_REPOSITORY=""

    if [ -n "$OPENGROK_SKIN" ]; then
        case "$OPENGROK_SKIN" in
            default) SKIN="-L default" ;;
            offwhite) SKIN="-L offwhite" ;;
            polished) SKIN="-L polished" ;;
            *) Warning "Invalid OPENGROK_SKIN ignored!" ;;
        esac
    fi

    if [ ! -x "${EXUBERANT_CTAGS}" ]; then
        FatalError "Missing Dependent Application - Exuberant CTags"
    fi

    if [ ! -d "${SRC_ROOT}" ]; then
        FatalError "OpenGrok Source Path ${SRC_ROOT} doesn't exist"
    fi

    if [ -n "${QUIET}" -a -n "${VERBOSE}" ]; then
        Warning "Both Quiet and Verbose Mode Enabled - Choosing Verbose"
        QUIET=""
        VERBOSE="-v"
    fi

    if [ -n "${OPENGROK_DERBY}" ]; then
        if [ ! -r "$OPENGROK_DIR/lib/lib/derbyclient.jar" ]; then
            if [ -n "$AS_HOME" -a -r "$AS_HOME/javadb/lib/derbyclient.jar" ]; then
                ln -s "$AS_HOME/javadb/lib/derbyclient.jar" \
                    "$OPENGROK_DIR/lib/lib/derbyclient.jar"
            elif [ -r /opt/SUNWjavadb/lib/derbyclient.jar ]; then
                ln -s /opt/SUNWjavadb/lib/derbyclient.jar \
                    "$OPENGROK_DIR/lib/lib/derbyclient.jar"
            else
                FatalError "$OPENGROK_DIR/lib/lib/derbyclient.jar not found!"
            fi
        fi
    fi
}

CreateRuntimeRequirements() {
    if [ ! -d "${DATA_ROOT}" ]; then
        Warning  "OpenGrok generated data path ${DATA_ROOT} doesn't exist"
        Progress "  Attempting to create generated data directory ... "
        ${DO} mkdir -p "${DATA_ROOT}"
    fi
    if [ ! -d "${DATA_ROOT}" ]; then
        FatalError "OpenGrok data path ${DATA_ROOT} doesn't exist"
    fi

    if [ ! -d "${OPENGROK_INSTANCE_BASE}/etc" ]; then
        Warning  "OpenGrok generated etc path " \
            "${OPENGROK_INSTANCE_BASE}/etc doesn't exist"
        Progress "  Attempting to create generated etc directory ... "
        ${DO} mkdir -p "${OPENGROK_INSTANCE_BASE}/etc"
    fi
    if [ ! -d "${OPENGROK_INSTANCE_BASE}/etc" ]; then
        FatalError "OpenGrok etc path " \
            "${OPENGROK_INSTANCE_BASE}/etc doesn't exist"
    fi

    if [ -n "${LOGGER_CONFIG_PATH}" -a ! -f "${LOGGER_CONFIG_PATH}" ]; then
        Progress "  Creating default ${LOGGER_CONFIG_PATH} ... "
        if [ ! -f "${LOGGER_CONF_SOURCE}" ]; then
            Warning "Can't find distribution logging configuration"   \
                "(${LOGGER_CONF_SOURCE}) to install as default" \
                "logging configuration (${LOGGER_CONFIG_PATH})"
        else
            ${DO} grep -v java.util.logging.FileHandler.pattern \
                "${LOGGER_CONF_SOURCE}" > "${LOGGER_CONFIG_PATH}"
            ${DO} grep java.util.logging.FileHandler.pattern \
                "${LOGGER_CONF_SOURCE}" | \
                sed "s|opengrok%g.%u.log|${OPENGROK_INSTANCE_BASE}/log/opengrok%g.%u.log|g" >> "${LOGGER_CONFIG_PATH}"
            ${DO} mkdir ${OPENGROK_INSTANCE_BASE}/log
        fi
    fi

    if [ -n "${OPENGROK_DERBY}" ]; then
        if [ ! -d "${OPENGROK_INSTANCE_BASE}/derby" ]; then
            Warning  "OpenGrok generated derby path " \
                "${OPENGROK_INSTANCE_BASE}/derby doesn't exist"
            Progress "  Attempting to create generated derby directory ... "
            ${DO} mkdir -p ${OPENGROK_INSTANCE_BASE}/derby
        fi
    fi
}

StdInvocation() {
    ${DO} ${JAVA} ${JAVA_OPTS} ${PROPERTIES} \
        ${LOGGER_PROPERTIES} \
        ${JAVA_DEBUG} \
        -jar ${OPENGROK_JAR} \
        ${IGNORE_PATTERNS} ${ENABLE_PROJECTS} \
        ${DERBY_HISTORY_CACHE} \
        ${SCAN_FOR_REPOSITORY} ${REMOTE_REPOSITORIES} \
        ${SCAN_DEPTH} \
        ${VERBOSE} ${QUIET} \
        ${PROGRESS} \
        ${EXUBERANT_CTAGS:+-c} ${EXUBERANT_CTAGS} \
        ${MAX_INDEXED_WORDS} ${SKIN} ${LEADING_WILDCARD} \
        ${READ_XML_CONF} \
        -W ${XML_CONFIGURATION} \
        ${WEBAPP_CONFIG_ADDRESS} \
        -s ${SRC_ROOT} -d ${DATA_ROOT} \
        "${@}"
}

UpdateDescriptionCache() {
    # An update program for EftarFile
    # Usage: <class> inputFile [inputFile ...] outputFile
    if [ -n "${PATH_DESC}" -a -s "${PATH_DESC}" ]; then
        ${DO} ${JAVA} -classpath ${OPENGROK_JAR} \
            ${EFTAR_UPDATE} ${PATH_DESC} ${EFTAR_OUTPUT_FILE}
    fi
}

OpenGrokUsage() {
    print -u2 "Options for opengrok.jar:\n"
    ${DO} ${JAVA} ${JAVA_OPTS} -jar ${OPENGROK_JAR} '-?'
}

# Args: CurrentValue, PropertyName, PropertyValue
AppendEntry() {
    printf "%s <env-entry><env-entry-name>%s</env-entry-name><env-entry-value>%s</env-entry-value><env-entry-type>java.lang.String</env-entry-type></env-entry>" "$1" "$2" "$3"
}

DeployWar() {
    if [ ! -f "${OPENGROK_WAR_SOURCE}" ]; then
        FatalError "Missing Web Application Archive ${OPENGROK_WAR_SOURCE}"
    fi

    if [ ! -d "${OPENGROK_WAR_TARGET}" ]; then
        FatalError "Missing Deployment Directory ${OPENGROK_WAR_TARGET}"
    fi

    Progress "Installing ${OPENGROK_WAR_SOURCE} to ${OPENGROK_WAR_TARGET} ..."

    REPLACE_IDX="" ; REPLACE_OG=""
    [ -n "$INDEX_BODY_HTML" -a -r "$INDEX_BODY_HTML" ] && REPLACE_IDX="y"
    [ "$OPENGROK_INSTANCE_BASE" != "/var/opengrok" ] && REPLACE_OG="y"
    RES=
    if [ -n "$REPLACE_IDX" -o -n "$WENV" -o -n "$REPLACE_OG" ]; then
        ATMPDIR=$(/usr/bin/mktemp -d)
        if [ -z "$ATMPDIR" ]; then
            FatalError "Unable to create temp file"
        fi
        OLDPWD=$(pwd)
        ${DO} cp "${OPENGROK_WAR_SOURCE}" "$ATMPDIR/source.war"
        ${DO} cd "$ATMPDIR"
        FILES=""
        if [ -n "$WENV" -o "$REPLACE_OG" ]; then
            ${DO} ${JAR} xf source.war WEB-INF/web.xml 
            FILES="WEB-INF/web.xml"
            if [ -n "$REPLACE_OG" ]; then
                ${DO} ${JAR} xf source.war WEB-INF/lib/opengrok.jar
                ${DO} ${JAR} xf WEB-INF/lib/opengrok.jar \
                    org/opensolaris/opengrok/management/oga.properties
                ${DO} gsed -i -e "s|/var/opengrok|$OPENGROK_INSTANCE_BASE|" \
                    WEB-INF/web.xml \
                    org/opensolaris/opengrok/management/oga.properties
                ${DO} ${JAR} uf WEB-INF/lib/opengrok.jar \
                    org/opensolaris/opengrok/management/oga.properties
                FILES="$FILES WEB-INF/lib/opengrok.jar"
                cp WEB-INF/lib/opengrok.jar "$OPENGROK_JAR"
            fi
        fi
        if [ -n "$INDEX_BODY_HTML" -a -r "$INDEX_BODY_HTML" ]; then
            ${DO} cp "$INDEX_BODY_HTML" index_body.html
            FILES="$FILES index_body.html"
        fi
        ${DO} ${JAR} uf source.war $FILES
        chmod 644 source.war
        ${DO} cp source.war "${OPENGROK_WAR_TARGET}/source.war"
        RES=$?
        cd "$X"
        ${DO} rm -rf "$ATMPDIR"
    else
        ${DO} cp "${OPENGROK_WAR_SOURCE}" "${OPENGROK_WAR_TARGET}/source.war"
        RES=$?
    fi
    #chown $GF_OWNER "${OPENGROK_WAR_TARGET}/source.war"

    if [ $RES != 0 ]; then
        FatalError "Web Application Installation FAILED"
    fi

    Progress
    Progress "Start your Glassfish application server, if it is not already"
    Progress "running, or wait until it loads the just installed web application."
    Progress
    Progress "OpenGrok should be available on <HOST>:<PORT>/source"
    Progress "  where HOST and PORT are configured in Glassfish."
    Progress
}

# Main Program
Usage() {
    print -u2 "\nUsage: ${0} [-h] [-c file] [-q|v] [-p] [-s directory] cmd\n\n\
Options:\n\
    -h      .. print this help and exit.\n\
    -c file .. configuration file to read. Gets sourced in immediately and\n\
               it may overwrite other cmd line options or values configured\n\
               might be overwritten by cmd line options, which follow this\n\
               one. So cmd line option order is important. The config file\n\
               should obey bourne shell syntax and only overwrite intended\n\
                   environment variables.\n\
    -s dir  .. Use the given directory dir as SRC_ROOT.\n\
    -q      .. Run quietly (disables verbose).\n\
    -v      .. Run verbose.\n\
    -p      .. Show progress in percent.\n\
    cmd     .. deploy:\n\
                   Deploys the opengrok source.war to the Glassfish domain\n\
                   directory.\n\
               undeploy:\n\
                   Undeploy the opengrok source.war\n\
               derby:\n\
                   Enables the Java DB system service (smf).\n\
               update:\n\
                   Index the related source directories and update database.\n\
               usage:\n\
                   Display opengrok usage.\n\
               list:\n\
                   Display a list of all repository pathes.\n\
               zap:\n\
                   Clean the cache for the given repository pathes.\n\
"
    exit 1
}

while getopts "hc:qvps:" option ; do
    case "$option" in
        h) Usage ;;
        q) QUIET="-q" ; VERBOSE="" ;;
        c) LoadInstanceConfiguration "${OPTARG}" ;;
        s) SRC_ROOT="${OPTARG}" ;;
        v) OPENGROK_VERBOSE=1 ;;
        p) OPENGROK_PROGRESS=1 ;;
    esac
done
shift $((OPTIND-1))

[ $# -eq 0 -o $# -gt 1 ] && Usage

SetupEnv

case "${1}" in
    undeploy)
        ValidateConfiguration
        ${DO} rm -f "${OPENGROK_WAR_TARGET}/source.war"
        ;;
    deploy)
        ValidateConfiguration
        DeployWar
        ;;
    derby)
        ValidateConfiguration
        CreateRuntimeRequirements
        svcadm enable javadb
        ;;
    update)
        ValidateConfiguration
        CreateRuntimeRequirements
        StdInvocation -H
        UpdateDescriptionCache
        ;;
    list)
        ValidateConfiguration
        CreateRuntimeRequirements
        StdInvocation -K
        ;;
    zap)
        ValidateConfiguration
        CreateRuntimeRequirements
        StdInvocation -k '*'
        ;;
    usage)
        OpenGrokUsage
        ;;
    *)
        Usage
        ;;
esac

[tarzanek]

att 4505:
https://java.net/projects/opengrok/downloads/download/attachments/4505.patch

[vladak]

The shell wrapper is not used anymore, hence closing.