diff --git a/pallas-console-web/src/pages/index_detail/version_manage/version_info_dialog/version_dynamic_info_dialog.vue b/pallas-console-web/src/pages/index_detail/version_manage/version_info_dialog/version_dynamic_info_dialog.vue index cff4c3ee..159c82d7 100644 --- a/pallas-console-web/src/pages/index_detail/version_manage/version_info_dialog/version_dynamic_info_dialog.vue +++ b/pallas-console-web/src/pages/index_detail/version_manage/version_info_dialog/version_dynamic_info_dialog.vue @@ -129,7 +129,99 @@ - + +
+ ES映射关系配置 + 新增 +
+
+
+ +
+ 1.某些数据库字段，尽管是number类型，但是在做业务查询时仅仅只是做term(s)这类非数学运算非聚合查询，我们非常建议你采用"keyword as number"类型，在这个类型下，ES将会用string格式来建索引以达到更高的检索性能， + 而获取 _source 时我们仍然会以number的格式返回给 Client。 +
+ 2.当你需要做模糊匹配，比如数据库值是 AbC，但是仍然希望abc和ABC都能检索出来，那请选择"keyword[全大写处理]"类型，我们在建索引和查询都做大写处理， + 而获取 _source 时我们仍然会以原值 AbC 的格式返回给 Client。 +
+ 3.所有的DB类型为TINYINT 的字段，我们都假设它是一些枚举值并且不会用于数学运算，因此我们为这些字段自动匹配了"keyword as number"类型，请自行检查。 +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+ @@ -139,17 +231,34 @@ 确 定 + + + diff --git a/pallas-console-web/src/pages/plugin_manage/plugin_upgrade.vue b/pallas-console-web/src/pages/plugin_manage/plugin_upgrade.vue index 56e5783d..3db2c62f 100644 --- a/pallas-console-web/src/pages/plugin_manage/plugin_upgrade.vue +++ b/pallas-console-web/src/pages/plugin_manage/plugin_upgrade.vue @@ -147,11 +147,11 @@ export default { const params = { pluginUpgradeId: row.id, action: operation, - nodeIp: nodeIp, + nodeIp, }; const involveItems = nodeIp == null ? row.clusterId : nodeIp; const msg = this.$createElement('span', null, [this.$createElement('span', null, `确定${text}插件${row.pluginName}吗?`), - this.$createElement('br', null), this.$createElement('span', null, `升级对象: ${involveItems}`)]); + this.$createElement('br', null), this.$createElement('span', null, `升级对象: ${involveItems}`)]); this.$message.confirmMessage(msg, () => { this.loading = true; this.$http.post('/plugin/upgrade/action.json', params).then(() => { diff --git a/pallas-console/src/main/java/com/vip/pallas/console/controller/index/version/IndexVersionController.java b/pallas-console/src/main/java/com/vip/pallas/console/controller/index/version/IndexVersionController.java index 09a75aee..5cb8bf14 100644 --- a/pallas-console/src/main/java/com/vip/pallas/console/controller/index/version/IndexVersionController.java +++ b/pallas-console/src/main/java/com/vip/pallas/console/controller/index/version/IndexVersionController.java @@ -30,6 +30,7 @@ import javax.validation.constraints.Min; import com.vip.pallas.console.vo.IndexVersionDynamicVO; +import com.vip.pallas.mybatis.entity.*; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,11 +55,6 @@ import com.vip.pallas.console.vo.base.BaseIndexVersionOp; import com.vip.pallas.exception.BusinessLevelException; import com.vip.pallas.exception.PallasException; -import com.vip.pallas.mybatis.entity.Cluster; -import com.vip.pallas.mybatis.entity.Index; -import com.vip.pallas.mybatis.entity.IndexOperation; -import com.vip.pallas.mybatis.entity.IndexVersion; -import com.vip.pallas.mybatis.entity.Page; import com.vip.pallas.service.ClusterService; import com.vip.pallas.service.ElasticSearchService; import com.vip.pallas.service.IndexOperationService; @@ -467,8 +463,13 @@ public void update(@RequestBody @Validated IndexVersionDynamicVO params, HttpSer throw new BusinessLevelException(403, "无权限操作"); } - indexVersionService.update(indexVersion); + Object schema = params.getSchema(); + List mappingList = indexVersionService.updateDynamic(indexVersion,(ArrayList)schema); elasticSearchService.dynamicUpdateIndexSettings(index.getIndexName(),index.getId(),indexVersion); + if (mappingList.size()>0){ + elasticSearchService.updateEsMapping(index.getIndexName(),index.getId(),versionId,mappingList); + } + try { AuditLogUtil.log( "add versionId: id - {0}, index - {1}, maxResultWindow - {2}, numOfReplication - {3}," + diff --git a/pallas-console/src/main/java/com/vip/pallas/console/vo/IndexVersionDynamicVO.java b/pallas-console/src/main/java/com/vip/pallas/console/vo/IndexVersionDynamicVO.java index 8d35279d..d865750f 100644 --- a/pallas-console/src/main/java/com/vip/pallas/console/vo/IndexVersionDynamicVO.java +++ b/pallas-console/src/main/java/com/vip/pallas/console/vo/IndexVersionDynamicVO.java @@ -59,6 +59,17 @@ public class IndexVersionDynamicVO implements Serializable { private String translogDurability; + private Object schema; + + @NotNull(message = "schema不能为null") + public Object getSchema() { + return schema; + } + + public void setSchema(Object schema) { + this.schema = schema; + } + public Long getMaxResultWindow() { return maxResultWindow; } diff --git a/pallas-core/src/main/java/com/vip/pallas/bean/EsMappingsUpdateModel.java b/pallas-core/src/main/java/com/vip/pallas/bean/EsMappingsUpdateModel.java new file mode 100644 index 00000000..fdfbbcf2 --- /dev/null +++ b/pallas-core/src/main/java/com/vip/pallas/bean/EsMappingsUpdateModel.java @@ -0,0 +1,22 @@ +package com.vip.pallas.bean; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.gson.annotations.SerializedName; + +import java.util.List; +import java.util.Map; + +import com.vip.pallas.bean.EsMappings.Propertie; + +public class EsMappingsUpdateModel { + + private Map properties; + + public Map getProperties() { + return properties; + } + + public void setProperties(Map properties) { + this.properties = properties; + } +} diff --git a/pallas-core/src/main/java/com/vip/pallas/service/ElasticSearchService.java b/pallas-core/src/main/java/com/vip/pallas/service/ElasticSearchService.java index d5499009..4a44d7f5 100644 --- a/pallas-core/src/main/java/com/vip/pallas/service/ElasticSearchService.java +++ b/pallas-core/src/main/java/com/vip/pallas/service/ElasticSearchService.java @@ -24,6 +24,7 @@ import com.vip.pallas.bean.monitor.ShardInfoModel; import com.vip.pallas.mybatis.entity.IndexVersion; +import com.vip.pallas.mybatis.entity.Mapping; import org.elasticsearch.client.RestClient; import com.vip.pallas.mybatis.entity.Cluster; @@ -31,6 +32,10 @@ public interface ElasticSearchService { public String genMappingJsonByVersionIdAndClusterName(Long versionId, String clusterName); + + public String genMappingJsonByMappingList(List mappings); + + public String updateEsMapping(String indexName, Long indexId, Long versionId,List mappings) throws IOException; public String createIndex(String indexName, Long indexId, Long versionId) throws IOException; diff --git a/pallas-core/src/main/java/com/vip/pallas/service/IndexVersionService.java b/pallas-core/src/main/java/com/vip/pallas/service/IndexVersionService.java index f0f8d301..5fadfd32 100644 --- a/pallas-core/src/main/java/com/vip/pallas/service/IndexVersionService.java +++ b/pallas-core/src/main/java/com/vip/pallas/service/IndexVersionService.java @@ -94,9 +94,6 @@ public void insert(IndexVersion indexVersion, ArrayList schemaArrayNode) { insertMappings(indexVersion, schemaArrayNode); } - public void update(IndexVersion indexVersion) { - indexVersionRepository.updateByPrimaryKeySelective(indexVersion); - } public IndexVersion findById(Long id) { return indexVersionRepository.selectByPrimaryKey(id); @@ -204,6 +201,17 @@ public void disableVersionSync(Long indexId, Long versionId) throws Exception { } + public void update(IndexVersion indexVersion){ + indexVersionRepository.updateByPrimaryKeySelective(indexVersion); + } + + public List updateDynamic(IndexVersion indexVersion,ArrayList schemaArrayNode) { + Date date = new Date(); + indexVersion.setUpdateTime(date); + indexVersionRepository.updateByPrimaryKeySelective(indexVersion); + return insertMappings(indexVersion, schemaArrayNode); + } + public void update(IndexVersion indexVersion, ArrayList schemaArrayNode) { Date date = new Date(); indexVersion.setUpdateTime(date); @@ -214,15 +222,16 @@ public void update(IndexVersion indexVersion, ArrayList schemaArrayNode) { insertMappings(indexVersion, schemaArrayNode); } - private void insertMappings(IndexVersion indexVersion, ArrayList schemaArrayNode) { - + private List insertMappings(IndexVersion indexVersion, ArrayList schemaArrayNode) { + List mappingList = new ArrayList<>(schemaArrayNode.size()); for (Map node : schemaArrayNode) { Mapping mapping = initMapping(node, indexVersion); Boolean dynamicNode = (Boolean) node.get("dynamic"); mapping.setDynamic(dynamicNode != null && dynamicNode); + mappingList.add(mapping); mappingRepository.insert(mapping); // 检查是否有多域 - checkAndAddMultiFieldsMapping(node, mapping.getId(), indexVersion); + mappingList.addAll(checkAndAddMultiFieldsMapping(node, mapping.getId(), indexVersion)); List childNode = (List) node.get("children"); for (Map _childNode : childNode) { Mapping childMapping = initMapping(_childNode, indexVersion); @@ -233,24 +242,30 @@ private void insertMappings(IndexVersion indexVersion, ArrayList schemaArra }else { childMapping.setParentType(MappingParentType.NESTED.val()); } + mappingList.add(childMapping); mappingRepository.insert(childMapping); // 检查nested域是否有多域 - checkAndAddMultiFieldsMapping(_childNode, childMapping.getId(), indexVersion); + mappingList.addAll(checkAndAddMultiFieldsMapping(_childNode, childMapping.getId(), indexVersion)); } } + return mappingList; } - private void checkAndAddMultiFieldsMapping(Map map, Long parentId, IndexVersion indexVersion){ - if (null == map.get("multiField"))return; + private List checkAndAddMultiFieldsMapping(Map map, Long parentId, IndexVersion indexVersion){ + if (null == map.get("multiField"))return Collections.emptyList(); + List multiFieldNodes = (List) map.get("multiField"); + List mappings = new ArrayList<>(multiFieldNodes.size()); // 增加multi-field for (Map _node : multiFieldNodes) { Mapping multiFieldsMapping = initMapping(_node, indexVersion); multiFieldsMapping.setParentId(parentId); multiFieldsMapping.setDynamic(Boolean.FALSE); multiFieldsMapping.setParentType(MappingParentType.MULTI_FIELDS.val()); + mappings.add(multiFieldsMapping); mappingRepository.insert(multiFieldsMapping); } + return mappings; } private Mapping initMapping(Map node, IndexVersion indexVersion) { diff --git a/pallas-core/src/main/java/com/vip/pallas/service/impl/ElasticSearchServiceImpl.java b/pallas-core/src/main/java/com/vip/pallas/service/impl/ElasticSearchServiceImpl.java index ca5c5aef..65d99044 100644 --- a/pallas-core/src/main/java/com/vip/pallas/service/impl/ElasticSearchServiceImpl.java +++ b/pallas-core/src/main/java/com/vip/pallas/service/impl/ElasticSearchServiceImpl.java @@ -138,6 +138,30 @@ public String genMappingJsonByVersionIdAndClusterName(Long versionId, String clu return s; } + + @Override + public String genMappingJsonByMappingList(List mappings) { + if(mappings == null){ + return null; + } + EsMappingsUpdateModel esMappings=new EsMappingsUpdateModel(); + Map propertyMap = new HashMap<>(); + esMappings.setProperties(propertyMap); + Map> mappingMap = new HashMap<>(); + List firstLayerList = new ArrayList<>(); + constructMappings(mappings, firstLayerList, mappingMap); + + addSourceField(firstLayerList); + + for (Mapping mapping : firstLayerList) { + Propertie prop = new Propertie(); + propertyMap.put(mapping.getFieldName(), prop); + convertMappingToProperty(mapping, prop, mappingMap); + } + String s = new Gson().toJson(esMappings); + return s; + } + private EsSourceMapping constructSourceMapping(IndexVersion indexVersion){ // 如果设置了 enabled = false， 则直接忽略后面的配置 if (indexVersion.getSourceDisabled() == Boolean.TRUE){ @@ -387,6 +411,30 @@ public static void constructMappings(List mappingList, List fi } } + @Override + public String updateEsMapping(String indexName, Long indexId, Long versionId,List mappings) throws IOException { + StringBuilder result = new StringBuilder(); + List clusters = clusterRepository.selectPhysicalClustersByIndexId(indexId); + for (Cluster cluster : clusters) { + try { + if (!this.isExistIndex(indexName, cluster.getHttpAddress(), versionId)) { + logger.error("try update mapping,but index not exist,indexName:{},httpAddress:{},versoinId:{}",indexName,cluster.getHttpAddress(),versionId); + continue; + } + NStringEntity entity = new NStringEntity(genMappingJsonByMappingList(mappings), + ContentType.APPLICATION_JSON); + result.append(IOUtils.toString(ElasticRestClient.build(cluster.getHttpAddress()) + .performRequest("PUT", "/" + indexName + "_" + versionId+"/_mapping/item", Collections.emptyMap(), entity) + .getEntity().getContent())).append("\\n"); + } catch (IOException e) { + logger.error(e.getClass() + " " + e.getMessage(), e); + throw e; + } + } + return result.toString(); + + } + @Override public String createIndex(String indexName, Long indexId, Long versionId) throws IOException { StringBuilder result = new StringBuilder();