EZP-30463: As an Developer, I want to use Solr Cloud (#137)

ezsystems · Jun 21, 2019 · af22427 · af22427
1 parent 6053cbf
commit af22427
Show file tree

Hide file tree

Showing 15 changed files with 667 additions and 46 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -18,7 +18,8 @@ matrix:
           env: TEST_CONFIG="phpunit-integration-legacy-solr.xml" SOLR_VERSION="6.5.1"  CORES_SETUP="shared"
         - php: 7.2
           env: TEST_CONFIG="phpunit-integration-legacy-solr.xml" SOLR_VERSION="6.6.5"  CORES_SETUP="single" SOLR_CORES="collection1"
-
+        - php: 7.2
+          env: TEST_CONFIG="phpunit-integration-legacy-solr.xml" SOLR_VERSION="6.6.5"  CORES_SETUP="cloud" SOLR_CLOUD="yes"
 # test only master and stable branches (+ Pull requests against those)
 branches:
     only:

diff --git a/bin/.travis/init_solr.sh b/bin/.travis/init_solr.sh
@@ -10,16 +10,36 @@ default_cores[2]='core2'
 default_cores[3]='core3'
 default_cores[4]='core4'
 
+default_nodes=('node1:8983' 'node2:8984' 'node3:8985' 'node4:8986')
+default_shards=('shard0')
+
 SOLR_PORT=${SOLR_PORT:-8983}
-SOLR_VERSION=${SOLR_VERSION:-6.6.5}
+SOLR_DIR=${SOLR_DIR:-'__solr'}
+SOLR_VERSION=${SOLR_VERSION:-'6.6.5'}
+SOLR_INSTALL_DIR="${SOLR_DIR}/${SOLR_VERSION}"
 SOLR_DEBUG=${SOLR_DEBUG:-false}
-SOLR_HOME=${SOLR_HOME:-ez}
-SOLR_CONFIG=${SOLR_CONFIG:-${default_config_files[*]}}
+SOLR_HOME=${SOLR_HOME:-'ezcloud'}
+SOLR_CONFIG=("${SOLR_CONFIG[@]:-${default_config_files[*]}}")
 SOLR_CORES=${SOLR_CORES:-${default_cores[*]}}
-SOLR_DIR=${SOLR_DIR:-__solr}
-SOLR_INSTALL_DIR="${SOLR_DIR}/${SOLR_VERSION}"
+SOLR_SHARDS=("${SOLR_SHARDS[@]:-${default_shards[*]}}")
+SOLR_NODES=("${SOLR_NODES[@]:-${default_nodes[@]}}")
+SOLR_COLLECTION_NAME=${SOLR_COLLECTION_NAME:-'ezplatform'}
+SOLR_CONFIGURATION_NAME=${SOLR_CONFIGURATION_NAME:-'ezconfig'}
+SOLR_MAX_SHARDS_PER_NODE=${SOLR_MAX_SHARDS_PER_NODE:-'1'}
+SOLR_REPLICATION_FACTOR=${SOLR_REPLICATION_FACTOR:-'1'}
+SOLR_CLOUD=${SOLR_CLOUD:-'no'}
+
+INSTALL_DIR="${SOLR_DIR}/${SOLR_VERSION}"
+HOME_DIR="${INSTALL_DIR}/server/${SOLR_HOME}"
+TEMPLATE_DIR="${HOME_DIR}/template"
+START_SCRIPT="./${INSTALL_DIR}/bin/solr"
+ZOOKEEPER_CLI_SCRIPT="./${INSTALL_DIR}/server/scripts/cloud-scripts/zkcli.sh"
+ZOOKEEPER_HOST=""
 
 SCRIPT_DIR=`dirname $0`
+if [ "$1" = "--solr-cloud" ]; then
+    SOLR_CLOUD='yes'
+fi
 
 download() {
     case ${SOLR_VERSION} in
@@ -35,10 +55,10 @@ download() {
 
     create_dir ${SOLR_DIR}
 
-    archive_file_name="${SOLR_VERSION}.tgz"
-    installation_archive_file="${SOLR_DIR}/${archive_file_name}"
+    local archive_file_name="${SOLR_VERSION}.tgz"
+    local installation_archive_file="${SOLR_DIR}/${archive_file_name}"
 
-    if [ ! -d ${SOLR_INSTALL_DIR} ] ; then
+    if [ ! -d ${INSTALL_DIR} ] ; then
         echo "Installation ${SOLR_VERSION} does not exists"
 
         if [ ! -f ${installation_archive_file} ] ; then
@@ -49,30 +69,30 @@ download() {
         fi
 
         echo "Extracting from installation archive ${archive_file_name}..."
-        create_dir ${SOLR_INSTALL_DIR}
-        tar -zxf ${installation_archive_file} -C ${SOLR_INSTALL_DIR} --strip-components=1
+        create_dir ${INSTALL_DIR}
+        tar -zxf ${installation_archive_file} -C ${INSTALL_DIR} --strip-components=1
         echo 'Extracted'
     else
         echo "Found existing ${SOLR_VERSION} installation"
     fi
 }
 
 copy_files() {
-    destination_dir_name=$1
+    local destination_dir_name=$1
     shift
-    files=("$@")
+    local files="$@"
 
     for file in ${files} ; do
         copy_file ${file} ${destination_dir_name}
     done
 }
 
 copy_file() {
-    file=$1
-    destination_dir=$2
+    local file=$1
+    local destination_dir=$2
 
-    if [ -f "${file}" ] ; then
-        cp ${file} ${destination_dir}
+    if [ -f ${file} ] ; then
+        cp ${file} ${destination_dir} || exit_on_error "Couldn't copy file '${file}' to directory '${destination_dir}'"
         echo "Copied file '${file}' to directory '${destination_dir}'"
     else
         echo "${file} is not valid"
@@ -81,71 +101,194 @@ copy_file() {
 }
 
 create_dir() {
-    dir_name=$1
+    local dir_name=$1
 
     if [ ! -d ${dir_name} ] ; then
-        mkdir ${dir_name}
+        mkdir ${dir_name} || exit_on_error "Couldn't create directory '${dir_name}'"
         echo "Created directory '${dir_name}'"
     fi
 }
 
 exit_on_error() {
-    message=$1
+    local message=$1
 
     echo "ERROR: ${message}"
     exit 1
 }
 
+#reintroduced VL
 is_solr_up() {
-    address="http://localhost:${SOLR_PORT}/solr/admin/cores"
-    http_code=`echo $(curl -s -o /dev/null -w "%{http_code}" ${address})`
+    local address="http://localhost:${SOLR_PORT}/solr/admin/cores"
+    local http_code=`echo $(curl -s -o /dev/null -w "%{http_code}" ${address})`
     echo "Checking if Solr is up on ${address}"
     return `test ${http_code} = "200"`
 }
 
+#reintroduced VL
 wait_for_solr(){
     while ! is_solr_up; do
         sleep 3
     done
 }
 
 # Run for Solr 6
-run() {
+solr_run() {
     echo "Running with version ${SOLR_VERSION} in standalone mode"
     echo "Starting solr on port ${SOLR_PORT}..."
 
     ./${SOLR_INSTALL_DIR}/bin/solr -p ${SOLR_PORT} -s ${SOLR_HOME} || exit_on_error "Can't start Solr"
 
     echo "Started"
 
-    create_cores
+    solr_create_cores
 }
 
 # Create cores for Solr 6
-create_cores() {
+solr_create_cores() {
     home_dir="${SOLR_INSTALL_DIR}/server/${SOLR_HOME}"
     template_dir="${home_dir}/template"
 
     for solr_core in ${SOLR_CORES} ; do
         if [ ! -d "${home_dir}/${solr_core}" ] ; then
-            create_core ${solr_core} ${template_dir}
+            solr_create_core ${solr_core} ${template_dir}
         else
             echo "Core ${solr_core} already exists, skipping"
         fi
     done
 }
 
-create_core() {
+solr_create_core() {
     core_name=$1
     config_dir=$2
 
     ./${SOLR_INSTALL_DIR}/bin/solr create_core -c ${core_name} -d ${config_dir} || exit_on_error "Can't create core"
 }
 
+solr_cloud_configure_nodes() {
+    create_dir ${HOME_DIR}
+
+    for node in "${SOLR_NODES[@]}" ; do
+        IFS=':' read node_name node_port <<< "${node}"
+
+        if [ ! -d "${HOME_DIR}/${node_name}" ] ; then
+            solr_cloud_configure_node ${node_name} ${node_port}
+        else
+            echo "Node '${node_name}' already exists, skipping"
+        fi
+    done
+}
+
+solr_cloud_configure_node() {
+    local name=$1
+    local port=$2
+    local node_dir="${HOME_DIR}/${name}"
+
+    create_dir ${node_dir}
+    copy_file "${INSTALL_DIR}/server/solr/zoo.cfg" ${node_dir}
+    copy_file "${INSTALL_DIR}/server/solr/solr.xml" ${node_dir}
+
+    # define host name 'localhost'
+    sed -i.bak "s/\${host:}/\${host:localhost}/g" "${node_dir}/solr.xml" || exit_on_error "Can't modify file '${node_dir}/solr.xml'"
+    # define port
+    sed -i.bak "s/\${jetty.port:8983}/\${jetty.port:${port}}/g" "${node_dir}/solr.xml" || exit_on_error "Can't modify file '${node_dir}/solr.xml'"
+}
+
+solr_cloud_start_nodes() {
+    for node in "${SOLR_NODES[@]}" ; do
+        local IFS=':'; read node_name node_port <<< "${node}"
+        local node_dir="${HOME_DIR}/${node_name}"
+
+        if [[ ! ${ZOOKEEPER_HOST} ]] ; then
+            ${START_SCRIPT} start -cloud -s ${node_dir} -p ${node_port} -V || exit_on_error "Can't start node '${node_name}'"
+            # start script default
+            ZOOKEEPER_HOST="localhost:$((node_port+1000))"
+        else
+            ${START_SCRIPT} start -cloud -s ${node_dir} -p ${node_port} -z "${ZOOKEEPER_HOST}" -V || exit_on_error "Can't start node '${node_name}'"
+        fi
+    done
+}
+
+solr_cloud_configure_collection() {
+    if [ -d ${TEMPLATE_DIR} ] ; then
+        echo "Configuration template is already created, skipping..."
+        return 1
+    fi
+
+    create_dir ${HOME_DIR}
+    create_dir ${TEMPLATE_DIR}
+
+    local files=("${SOLR_CONFIG[@]}")
+    local config_dir="${INSTALL_DIR}/server/solr/configsets/basic_configs/conf"
+
+    files+=("${config_dir}/currency.xml")
+    files+=("${config_dir}/stopwords.txt")
+    files+=("${config_dir}/synonyms.txt")
+    files+=("${config_dir}/elevate.xml")
+    files+=("${config_dir}/solrconfig.xml")
+
+    copy_files ${TEMPLATE_DIR} "${files[*]}"
+
+    # modify solrconfig.xml to remove section that doesn't agree with our schema
+    sed -i.bak '/<updateRequestProcessorChain name="add-unknown-fields-to-the-schema">/,/<\/updateRequestProcessorChain>/d' "${TEMPLATE_DIR}/solrconfig.xml" || exit_on_error "Can't modify file '${TEMPLATE_DIR}/solrconfig.xml'"
+    # Adapt autoSoftCommit to have a recommended value
+    sed -i.bak2 's/${solr.autoSoftCommit.maxTime:-1}/${solr.autoSoftCommit.maxTime:20}/' "${TEMPLATE_DIR}/solrconfig.xml" || exit_on_error "Can't modify file '${TEMPLATE_DIR}/solrconfig.xml'"
+}
+
+solr_cloud_upload_collection_configuration() {
+    ${ZOOKEEPER_CLI_SCRIPT} -zkhost "${ZOOKEEPER_HOST}" -cmd upconfig -confname ${SOLR_CONFIGURATION_NAME} -confdir ${TEMPLATE_DIR} || exit_on_error "Can't upload configuration to Zookeeper"
+    echo "Uploaded configuration to Zookeeper"
+}
+
+solr_cloud_create_collections() {
+    for solr_core in ${SOLR_CORES} ; do
+        solr_cloud_create_collection "${solr_core}"
+    done
+}
+
+solr_cloud_create_collection() {
+    local collection_name=$1
+    local shards_array=($SOLR_SHARDS)
+    local shards_count="${#shards_array[@]}"
+    local shards="${SOLR_SHARDS[@]// /,}"
+    local nodes_tmp=("${SOLR_NODES[@]/*:/localhost:}")
+    local nodes="${nodes_tmp[@]/%/_solr}"
+    nodes="${nodes[@]// /,}"
+
+    local parameters=(
+        "name=${collection_name}"
+        "collection.configName=${SOLR_CONFIGURATION_NAME}"
+        "createNodeSet=${nodes}"
+        "shards=${shards}"
+        "maxShardsPerNode=${SOLR_MAX_SHARDS_PER_NODE}"
+        "replicationFactor=${SOLR_REPLICATION_FACTOR}"
+        "router.name=compositeId"
+        "numShards=${shards_count}"
+        "createNodeSet.shuffle=false"
+        "wt=json"
+        "indent=on"
+    )
+
+    echo "Creating collection with parameters:"
+    echo "$(IFS=$'\n'; echo "${parameters[*]}")"
+
+    local url="http://localhost:8983/solr/admin/collections?action=CREATE&$(IFS=$'&'; echo "${parameters[*]}")"
+
+    curl ${url} || exit_on_error "Couldn't create collection"
+}
+
 download
 
+if [ "$SOLR_CLOUD" = "no" ]; then
+    $SCRIPT_DIR/../generate-solr-config.sh \
+            --solr-install-dir="${SOLR_INSTALL_DIR}" \
+            --destination-dir="${SOLR_INSTALL_DIR}/server/${SOLR_HOME}/template"
+    solr_run
+else
+    solr_cloud_configure_nodes
+    solr_cloud_start_nodes
+    solr_cloud_configure_collection
+    solr_cloud_upload_collection_configuration
+    solr_cloud_create_collections
+fi
+
 
-$SCRIPT_DIR/../generate-solr-config.sh \
-        --solr-install-dir="${SOLR_INSTALL_DIR}" \
-        --destination-dir="${SOLR_INSTALL_DIR}/server/${SOLR_HOME}/template"
-run
diff --git a/bundle/DependencyInjection/Configuration.php b/bundle/DependencyInjection/Configuration.php
@@ -171,6 +171,10 @@ function ($v) {
                     )
                 ->end()
                 ->children()
+                    ->enumNode('distribution_strategy')
+                        ->values(['standalone', 'cloud'])
+                        ->defaultValue('standalone')
+                    ->end()
                     ->arrayNode('entry_endpoints')
                         ->info(
                             "A set of entry endpoint names.\n\n" .

diff --git a/bundle/DependencyInjection/EzSystemsEzPlatformSolrSearchEngineExtension.php b/bundle/DependencyInjection/EzSystemsEzPlatformSolrSearchEngineExtension.php
@@ -10,7 +10,7 @@
  */
 namespace EzSystems\EzPlatformSolrSearchEngineBundle\DependencyInjection;
 
-use EzSystems\EzPlatformSolrSearchEngine\FieldMapper\BoostFactorProvider;
+use Symfony\Component\DependencyInjection\ChildDefinition;
 use Symfony\Component\DependencyInjection\Definition;
 use Symfony\Component\DependencyInjection\DefinitionDecorator;
 use Symfony\Component\DependencyInjection\Reference;
@@ -74,6 +74,16 @@ class EzSystemsEzPlatformSolrSearchEngineExtension extends Extension
      */
     const BOOST_FACTOR_PROVIDER_ID = 'ezpublish.search.solr.field_mapper.boost_factor_provider';
 
+    /**
+     * @var string
+     */
+    const STANDALONE_DISTRIBUTION_STRATEGY_ID = 'ezpublish.search.solr.gateway.distribution_strategy.abstract_standalone';
+
+    /**
+     * @var string
+     */
+    const CLOUD_DISTRIBUTION_STRATEGY_ID = 'ezpublish.search.solr.gateway.distribution_strategy.abstract_cloud';
+
     public function getAlias()
     {
         return 'ez_search_engine_solr';
@@ -181,9 +191,28 @@ private function configureSearchServices(ContainerBuilder $container, $connectio
         $coreFilterId = "$alias.connection.$connectionName.core_filter_id";
         $container->setDefinition($coreFilterId, $coreFilterDefinition);
 
-        // Gateway
+        // Distribution Strategy
+        $distributionStrategyId = "$alias.connection.$connectionName.distribution_strategy";
+
+        switch ($connectionParams['distribution_strategy']) {
+            case 'standalone':
+                $distributionStrategyDefinition = new ChildDefinition(self::STANDALONE_DISTRIBUTION_STRATEGY_ID);
+                $distributionStrategyDefinition->setArgument(1, new Reference($endpointResolverId));
+                break;
+            case 'cloud':
+                $distributionStrategyDefinition = new ChildDefinition(self::CLOUD_DISTRIBUTION_STRATEGY_ID);
+                $distributionStrategyDefinition->setArgument(1, new Reference($endpointResolverId));
+                break;
+            default:
+                throw new \RuntimeException('Unknown distribution strategy');
+        }
+
+        $container->setDefinition($distributionStrategyId, $distributionStrategyDefinition);
+
         $gatewayDefinition = new DefinitionDecorator(self::GATEWAY_ID);
         $gatewayDefinition->replaceArgument(1, new Reference($endpointResolverId));
+        $gatewayDefinition->replaceArgument(6, new Reference($distributionStrategyId));
+
         $gatewayId = "$alias.connection.$connectionName.gateway_id";
         $container->setDefinition($gatewayId, $gatewayDefinition);
     }

diff --git a/bundle/Resources/config/services.yml b/bundle/Resources/config/services.yml
@@ -35,3 +35,15 @@ services:
     ezpublish.search.solr.field_mapper.indexing_depth_provider:
         class: "%ezpublish.search.solr.field_mapper.indexing_depth_provider.class%"
         factory: 'ezpublish.search.solr.field_mapper.indexing_depth_provider_factory:buildService'
+
+    ezpublish.search.solr.gateway.distribution_strategy.abstract_standalone:
+        abstract: true
+        class: EzSystems\EzPlatformSolrSearchEngine\Gateway\DistributionStrategy\StandaloneDistributionStrategy
+        arguments:
+            - "@ezpublish.search.solr.gateway.endpoint_registry"
+
+    ezpublish.search.solr.gateway.distribution_strategy.abstract_cloud:
+        abstract: true
+        class: EzSystems\EzPlatformSolrSearchEngine\Gateway\DistributionStrategy\CloudDistributionStrategy
+        arguments:
+            - "@ezpublish.search.solr.gateway.endpoint_registry"