STORM-2508:storm-solr enhancement: update solrj to 5.5, support custom SolrClientFactory and commit operation #2108
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| /* | ||
| * DefaultCloudSolrClientFactory.java | ||
|
Contributor
There was a problem hiding this comment. Please replace this license header with the Apache header (copy it from one of the other files) |
||
| * Copyright 2017 Qunhe Tech, all rights reserved. | ||
| * Qunhe PROPRIETARY/CONFIDENTIAL, any form of usage is subject to approval. | ||
| */ | ||
|
|
||
| package org.apache.storm.solr.client; | ||
|
|
||
| import org.apache.solr.client.solrj.SolrClient; | ||
| import org.apache.solr.client.solrj.impl.CloudSolrClient; | ||
|
|
||
| /** | ||
| * default implementation of {@link SolrClientFactory}, which takes a zookeeper host | ||
| * and produce {@link org.apache.solr.client.solrj.impl.CloudSolrClient} with the host. | ||
| * | ||
| * @author alei | ||
|
|
||
| */ | ||
| public class DefaultSolrClientFactory implements SolrClientFactory { | ||
|
|
||
| private String zkHostString; | ||
|
|
||
| public DefaultSolrClientFactory(String zkHostString) { | ||
| this.zkHostString = zkHostString; | ||
| } | ||
|
|
||
| @Override | ||
| public SolrClient getSolrClient() { | ||
| return new CloudSolrClient(zkHostString); | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| /* | ||
| * SerializableSolrClientFactory.java | ||
| * Copyright 2017 Qunhe Tech, all rights reserved. | ||
| * Qunhe PROPRIETARY/CONFIDENTIAL, any form of usage is subject to approval. | ||
| */ | ||
|
|
||
| package org.apache.storm.solr.client; | ||
|
|
||
| import org.apache.solr.client.solrj.SolrClient; | ||
|
|
||
| import java.io.Serializable; | ||
|
|
||
| /** | ||
| * this interface provide SolrClient for {@link org.apache.storm.solr.mapper.SolrMapper} | ||
| * | ||
| * @author alei | ||
| */ | ||
| public interface SolrClientFactory extends Serializable { | ||
|
|
||
| SolrClient getSolrClient(); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| /* | ||
| * OnCommit.java | ||
| * Copyright 2017 Qunhe Tech, all rights reserved. | ||
| * Qunhe PROPRIETARY/CONFIDENTIAL, any form of usage is subject to approval. | ||
| */ | ||
|
|
||
| package org.apache.storm.solr.config; | ||
|
|
||
| import org.apache.solr.client.solrj.SolrClient; | ||
| import org.apache.solr.client.solrj.SolrServerException; | ||
|
|
||
| import java.io.IOException; | ||
| import java.io.Serializable; | ||
|
|
||
| /** | ||
| * This callback interface is invoked on {@link SolrCommitStrategy} triggers or tick. Typically fire | ||
| * a "commit" request to solr, you can customize commit details by implementing this interface. | ||
| * @author alei | ||
| */ | ||
| public interface CommitCallback extends Serializable { | ||
|
|
||
| void process(SolrClient solrClient, String collection) throws SolrServerException, IOException; | ||
|
Contributor
There was a problem hiding this comment. Nit: Shouldn't this be named "commit"? |
||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| /* | ||
| * DefaultCommitOperation.java | ||
| * Copyright 2017 Qunhe Tech, all rights reserved. | ||
| * Qunhe PROPRIETARY/CONFIDENTIAL, any form of usage is subject to approval. | ||
| */ | ||
|
|
||
| package org.apache.storm.solr.config; | ||
|
|
||
| import org.apache.solr.client.solrj.SolrClient; | ||
| import org.apache.solr.client.solrj.SolrServerException; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| /** | ||
| * @author alei | ||
| */ | ||
| public class DefaultCommitCallback implements CommitCallback { | ||
|
|
||
| @Override | ||
| public void process(final SolrClient solrClient, final String collection) throws SolrServerException, IOException { | ||
| solrClient.commit(collection); | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| /* | ||
| * SolrClientSchemaBuilder.java | ||
| * Copyright 2017 Qunhe Tech, all rights reserved. | ||
| * Qunhe PROPRIETARY/CONFIDENTIAL, any form of usage is subject to approval. | ||
| */ | ||
|
|
||
| package org.apache.storm.solr.schema.builder; | ||
|
|
||
| import org.apache.solr.client.solrj.SolrServerException; | ||
| import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition; | ||
| import org.apache.solr.client.solrj.request.schema.SchemaRequest; | ||
| import org.apache.solr.client.solrj.response.schema.SchemaRepresentation; | ||
| import org.apache.solr.client.solrj.response.schema.SchemaResponse; | ||
| import org.apache.storm.solr.client.SolrClientFactory; | ||
| import org.apache.storm.solr.schema.CopyField; | ||
| import org.apache.storm.solr.schema.FieldType; | ||
| import org.apache.storm.solr.schema.Schema; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
|
|
||
| /** | ||
| * <p>this Class build the {@link Schema} by Solr Schema API.</p> | ||
|
Contributor
There was a problem hiding this comment. Is there any reason to prefer the REST schema builder over this one? Otherwise maybe we should remove the other one? |
||
| * | ||
| * @see <a href="https://cwiki.apache.org/confluence/display/solr/Schema+API">Solr Schema API</a> | ||
| * @author alei | ||
| */ | ||
| public class SolrClientSchemaBuilder implements SchemaBuilder { | ||
|
|
||
| private static final Logger LOG = LoggerFactory.getLogger(SolrClientSchemaBuilder.class); | ||
|
|
||
| private Schema schema; | ||
|
|
||
| public SolrClientSchemaBuilder(String collection, SolrClientFactory factory) throws IOException, SolrServerException { | ||
| SchemaRequest request = new SchemaRequest(); | ||
| SchemaResponse response = request.process(factory.getSolrClient(), collection); | ||
| SchemaRepresentation representation = response.getSchemaRepresentation(); | ||
| schema = convert(representation); | ||
| } | ||
|
|
||
| @Override | ||
| public Schema getSchema() { | ||
| return schema; | ||
| } | ||
|
|
||
| private Schema convert(final SchemaRepresentation representation) { | ||
|
Contributor
There was a problem hiding this comment. Nit: It might be good to add a unit test for this, to verify that this conversion does the right thing. |
||
| Schema converted = new Schema(); | ||
| converted.setUniqueKey(representation.getUniqueKey()); | ||
| converted.setName(representation.getName()); | ||
| converted.setVersion(String.valueOf(representation.getVersion())); | ||
|
Contributor
There was a problem hiding this comment. Nit: If this is a float, shouldn't we keep it that way in Schema? |
||
| List<FieldType> fieldTypeList = new ArrayList<>(representation.getFieldTypes().size()); | ||
| for (FieldTypeDefinition type : representation.getFieldTypes()) { | ||
| FieldType fieldType = new FieldType(); | ||
| fieldType.setClazz(type.getAttributes().get("class").toString()); | ||
| if (type.getAttributes().get("multiValued") != null) { | ||
| fieldType.setMultiValued(Boolean.valueOf(type.getAttributes().get("multiValued").toString())); | ||
|
Contributor
There was a problem hiding this comment. Is this attribute already a boolean? If so I don't think there's a reason to toString and then Boolean.valueOf it. |
||
| } | ||
| fieldType.setName(type.getAttributes().get("name").toString()); | ||
| fieldTypeList.add(fieldType); | ||
| } | ||
| converted.setFieldTypes(fieldTypeList); | ||
| List<org.apache.storm.solr.schema.Field> fieldList = | ||
| new ArrayList<>(representation.getFields().size()); | ||
| for (Map<String, Object> field : representation.getFields()) { | ||
| org.apache.storm.solr.schema.Field schemaField = new org.apache.storm.solr.schema.Field(); | ||
| schemaField.setName(field.get("name").toString()); | ||
|
Contributor
There was a problem hiding this comment. Nit: An explicit cast to String might be nicer here, so you get an exception in case this isn't a String for some reason. |
||
| schemaField.setType(field.get("type").toString()); | ||
| fieldList.add(schemaField); | ||
| } | ||
| converted.setFields(fieldList); | ||
| List<org.apache.storm.solr.schema.Field> dynamicFieldList = | ||
| new ArrayList<>(representation.getDynamicFields().size()); | ||
| for (Map<String, Object> field : representation.getDynamicFields()) { | ||
| org.apache.storm.solr.schema.Field schemaField = new org.apache.storm.solr.schema.Field(); | ||
| schemaField.setName(field.get("name").toString()); | ||
| schemaField.setType(field.get("type").toString()); | ||
| dynamicFieldList.add(schemaField); | ||
| } | ||
| converted.setDynamicFields(dynamicFieldList); | ||
| List<CopyField> copyFieldList = new ArrayList<>(representation.getCopyFields().size()); | ||
| for (Map<String, Object> field : representation.getDynamicFields()) { | ||
| CopyField copyField = new CopyField(); | ||
| copyField.setDest(field.get("dest").toString()); | ||
| copyField.setSource(field.get("source").toString()); | ||
| copyFieldList.add(copyField); | ||
| } | ||
| converted.setCopyFields(copyFieldList); | ||
| return converted; | ||
| } | ||
|
|
||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
5.5.0 version is affected by a security problem and is also more than 1.5 years old.
http://lucene.apache.org/solr/news.html
Please consider moving to Solr 6.5.1 or 6.6.0