Skip to content

Commit

Permalink
#107 add feature: dynamic put mapping in index version management [vi…
Browse files Browse the repository at this point in the history 
…p:platform/pallas#925]
  • Loading branch information
@NathanChan
NathanChan committed Jul 30, 2019
1 parent abd59b6 commit e4648f3
Show file tree
Hide file tree
Showing 8 changed files with 388 additions and 18 deletions.
270 changes: 269 additions & 1 deletion ...src/pages/index_detail/version_manage/version_info_dialog/version_dynamic_info_dialog.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,99 @@
</el-row>
</div>
</el-tab-pane>

<el-tab-pane label="Mapping配置" name="second">
<div class="label-title">
<span class="span-title"><i class="fa fa-th-large"></i>ES映射关系配置</span>
<el-button size="mini" type="success" @click="addField(0)"><i class="fa fa-plus"></i>新增</el-button>
</div>
<div>
<div style="margin: 10px">
<el-alert
title="如何选择ES类型"
type="info"
description=" "
show-icon>
<div style="font-size: 12px">
1.某些数据库字段，尽管是number类型，但是在做业务查询时仅仅只是做term(s)这类非数学运算非聚合查询，我们非常建议你采用"keyword as number"类型，在这个类型下，ES将会用string格式来建索引以达到更高的检索性能，
而获取 _source 时我们仍然会以number的格式返回给 Client。
<br/>
2.当你需要做模糊匹配，比如数据库值是 AbC，但是仍然希望abc和ABC都能检索出来，那请选择"keyword[全大写处理]"类型，我们在建索引和查询都做大写处理，
而获取 _source 时我们仍然会以原值 AbC 的格式返回给 Client。
<br/>
3.所有的DB类型为TINYINT 的字段，我们都假设它是一些枚举值并且不会用于数学运算，因此我们为这些字段自动匹配了"keyword as number"类型，请自行检查。
</div>
</el-alert>
</div>
<el-table :data="versionInfo.schema" border style="width: 100%" :max-height="550">
<el-table-column label="操作">
<template slot-scope="scope">
<el-button size="small" type="success" @click="addField(scope.$index)"><i class="el-icon-plus"></i></el-button>
<el-button size="small" type="danger" @click="deleteField(scope.row)"><i class="el-icon-minus"></i></el-button>
</template>
</el-table-column>
<el-table-column label="字段名" min-width="180">
<template slot-scope="scope">
<el-button type="text" v-show="!scope.row.isNew">
<span>{{scope.row.fieldName}}</span>
</el-button>
<el-input style="width:50%" v-model="scope.row.fieldName" placeholder="请输入字段名" v-show="scope.row.isNew"></el-input>
<el-button type="text" v-show="scope.row.isNew">
<span>子字段</span>
</el-button>
<el-button type="warning" size="mini" @click="viewSchemaChildrenWithNoCheck(scope.row)" v-if="scope.row.children.length !== 0" ><i class="fa"></i>nested</el-button>
<el-button type="warning" size="mini" @click="viewSchemaMultiFields(scope.row)" v-if="scope.row.multiField.length !== 0" ><i class="fa"></i>subFields</el-button>
<div>
<el-tag type="success" v-if="scope.row.copyTo.length > 0">copy to: {{scope.row.copyTo}}</el-tag>
</div>
</template>
</el-table-column>
<el-table-column label="ES类型">
<template slot-scope="scope">
<select v-model="scope.row.fieldType" size="small" @change="fieldTypeChange(scope.row)">
<option v-for="item in fieldTypes" :value="item.value" :key="item.value">{{item.label}}</option>
</select>
</template>
</el-table-column>
<el-table-column label="多值/单值">
<template slot-scope="scope">
<select v-model="scope.row.multi" size="small">
<option label="单值" :value="false">单值</option>
<option label="多值" :value="true">多值</option>
</select>
</template>
</el-table-column>
<el-table-column label="是否创建索引" min-width="90">
<template slot-scope="scope">
<el-checkbox v-model="scope.row.search" :disabled="scope.row.fieldType === 'nested'">创建索引</el-checkbox>
</template>
</el-table-column>
<el-table-column :render-header="renderDocValueHeader">
<template slot-scope="scope">
<el-checkbox v-model="scope.row.docValue" :disabled="scope.row.fieldType === 'nested' || scope.row.fieldType === 'text'">启用doc value</el-checkbox>
</template>
</el-table-column>
<el-table-column label="是否启用store">
<template slot-scope="scope">
<el-checkbox v-model="scope.row.store">启用store</el-checkbox>
</template>
</el-table-column>
<el-table-column label="更多操作" width="80">
<template slot-scope="scope">
<el-dropdown trigger="click">
<span class="el-dropdown-link">
操作<i class="el-icon-caret-bottom el-icon--right"></i>
</span>
<el-dropdown-menu class="dropdown-operation" slot="dropdown">
<el-dropdown-item v-if="scope.row.multiField.length === 0"><a @click="viewSchemaChildren(scope.row)"><span><i class="fa fa-play-circle"></i>添加nested/object</span></a></el-dropdown-item>
<el-dropdown-item v-if="scope.row.children.length === 0 && scope.row.fieldType !== 'nested'"><a @click="viewSchemaMultiFields(scope.row)"><span><i class="fa fa-play-circle"></i>添加subFields</span></a></el-dropdown-item>
<el-dropdown-item><a @click="viewSchemaCopyTo(scope.row)"><span><i class="fa fa-play-circle"></i>添加copyTo</span></a></el-dropdown-item>
</el-dropdown-menu>
</el-dropdown>
</template>
</el-table-column>
</el-table>
</div>
</el-tab-pane>
</el-tabs>


Expand All @@ -139,17 +231,34 @@
<el-button type="confirm" @click="submitVersionInfo()">确 定</el-button>
</div>
</el-dialog>
<schema-child-dialog :is-schema-child-visible="isSchemaChildVisible" :schema-child-info="schemaExtInfo" :version-operation="versionOperation" :schema-parent-field-name="schemaParentFieldName" :version-info="versionInfo" @close-schema-dialog="closeSchemaDialog" @add-schema-child="addSchemaChild"></schema-child-dialog>
<schema-multi-field-dialog :is-schema-multi-fields-visible="isSchemaMultiFieldsVisible" :schema-multi-fields-info="schemaExtInfo" :version-operation="versionOperation" :schema-parent-field-name="schemaParentFieldName" @close-schema-dialog="closeSchemaMultiFieldsDialog" @add-schema-multi-field="addSchemaMultiFields"></schema-multi-field-dialog>
<schema-copy-to-dialog :is-copy-to-fields-visible="isCopyToFieldsVisible" :schema-copy-to-info="schemaExtInfo" :copy-to-list="validCopyToFields" :schema-parent-field-name="schemaParentFieldName" @close-schema-dialog="closeSchemaCopyToDialog" @add-schema-copy-to="addSchemaCopyTo" ></schema-copy-to-dialog>
</div>
</template>

<script>
import SchemaChildDialog from './schema_child_dialog';
import SchemaMultiFieldDialog from './schema_multi_field_dialog';
import SchemaCopyToDialog from './schema_copy_to_dialog';
export default {
components: {
'schema-child-dialog': SchemaChildDialog,
'schema-multi-field-dialog': SchemaMultiFieldDialog,
'schema-copy-to-dialog': SchemaCopyToDialog,
},
props: ['versionOperation', 'versionInfo', 'versionInfoTitle', 'clusters', 'isLogical'],
data() {
return {
loading: false,
isVersionInfoVisible: true,
isSchemaChildVisible: false,
isSchemaMultiFieldsVisible: false,
isCopyToFieldsVisible: false,
schemaExtInfo: {},
validCopyToFields: [],
schemaParentFieldName: '',
rules: {
replicationNum: [{ required: true, message: '复制数量不能为空' }, { type: 'number', message: '复制数量必须为数字值' }],
indexSlowThreshold: [{ required: true, message: 'Index Slow Log不能为空' }, { type: 'number', message: 'Index Slow Log必须为数字值' }],
Expand All @@ -161,9 +270,165 @@ export default {
flushThresholdSize: [{ required: true, message: 'flush_threshold_size不能为空' }],
translogDurability: [{ required: true, message: 'translog_durability不能为空' }],
},
fieldTypes: [{
value: 'text',
label: 'text(标准分词)',
}, {
value: 'text_ngram',
label: 'text(逐字分词)',
}, {
value: 'keyword',
label: 'keyword',
}, {
value: 'keyword_normalized',
label: 'keyword(全大写处理)',
}, {
value: 'keyword_as_number',
label: 'keyword as number',
}, {
value: 'date',
label: 'date',
}, {
value: 'boolean',
label: 'boolean',
}, {
value: 'object',
label: 'object',
}, {
value: 'nested',
label: 'nested',
}, {
value: 'long',
label: 'long',
}, {
value: 'integer',
label: 'integer',
}, {
value: 'short',
label: 'short',
}, {
value: 'byte',
label: 'byte',
}, {
value: 'double',
label: 'double',
}, {
value: 'float',
label: 'float',
}],
};
},
methods: {
fieldTypeChange(row) {
if (row.fieldType === 'text') {
this.$set(row, 'docValue', false);
} else if (row.fieldType === 'nested') {
this.$set(row, 'search', false);
this.$set(row, 'docValue', false);
this.$set(row, 'multi', true);
}
},
renderDocValueHeader(h) {
return h(
'span',
[
h('span', { slot: 'reference', style: { 'font-size': '14px', 'margin-right': '5px' } }, '是否启用doc value'),
h('el-popover', { props: { placement: 'top', trigger: 'hover' } },
[
h('div', '是否需要用到 Sort,Aggs,Script 查询'),
h('i', { slot: 'reference', class: { fa: true, 'fa-question-circle': true } }),
],
),
],
);
},
closeSchemaDialog() {
this.isSchemaChildVisible = false;
},
closeSchemaMultiFieldsDialog() {
this.isSchemaMultiFieldsVisible = false;
},
closeSchemaCopyToDialog() {
this.isCopyToFieldsVisible = false;
},
viewSchemaChildrenWithNoCheck(row) {
this.isSchemaChildVisible = true;
this.schemaExtInfo = row;
},
viewSchemaChildren(row) {
if (row.fieldType !== 'nested' && row.fieldType !== 'object') {
this.$message.errorMessage('ES类型必须为nested或者object');
return;
}
this.isSchemaChildVisible = true;
this.schemaExtInfo = row;
},
viewSchemaMultiFields(row) {
this.isSchemaMultiFieldsVisible = true;
this.schemaExtInfo = row;
},
viewSchemaCopyTo(row) {
this.schemaExtInfo = row;
this.validCopyToFields = [];
this.versionInfo.schema.forEach((el) => {
if (el.dbFieldType === 'N/A' || el.dbFieldType === null) {
if (el.fieldType === 'nested') {
this.getNestedFieldName(el, '', this.validCopyToFields);
return;
}
this.validCopyToFields.push(el.fieldName);
}
});
this.copyToListFilter(this.validCopyToFields, row.fieldName);
this.isCopyToFieldsVisible = true;
},
getNestedFieldName(field, parentFieldName, fieldArr) {
if (field.fieldType !== 'nested') {
fieldArr.push(`${parentFieldName}${field.fieldName}`);
return;
}
field.children.forEach((child) => {
if (child.fieldType === 'nested') {
this.getNestedFieldName(child, `${parentFieldName}${field.fieldName}.`, fieldArr);
} else {
fieldArr.push(`${parentFieldName}${field.fieldName}.${child.fieldName}`);
}
});
},
copyToListFilter(fieldArr, fieldName) {
const index = fieldArr.indexOf(fieldName);
if (index >= 0) {
fieldArr.splice(index, 1);
}
},
addSchemaChild() {
this.isSchemaChildVisible = false;
},
addSchemaMultiFields(array) {
console.log(JSON.stringify(array));
this.isSchemaMultiFieldsVisible = false;
},
addSchemaCopyTo(array) {
console.log(JSON.stringify(array));
this.isCopyToFieldsVisible = false;
},
addField(index) {
const newRow = {
fieldName: '',
dbFieldType: 'N/A',
docValue: false,
fieldType: 'keyword',
multi: false,
children: [],
multiField: [],
copyTo: [],
search: false,
isNew: true,
dynamic: false,
store: false,
};
this.versionInfo.schema.splice(index + 1, 0, newRow);
},
getLogicClusterAllocationNodes() {
const result = [];
this.clusterArray.forEach((ele) => {
Expand Down Expand Up @@ -231,6 +496,9 @@ export default {
return result;
},
},
created() {
this.versionInfo.schema = [];
},
};
</script>
Expand Down
4 changes: 2 additions & 2 deletions pallas-console-web/src/pages/plugin_manage/plugin_upgrade.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,11 +147,11 @@ export default {
const params = {
pluginUpgradeId: row.id,
action: operation,
nodeIp: nodeIp,
nodeIp,
};
const involveItems = nodeIp == null ? row.clusterId : nodeIp;
const msg = this.$createElement('span', null, [this.$createElement('span', null, `确定${text}插件${row.pluginName}吗?`),
this.$createElement('br', null), this.$createElement('span', null, `升级对象: ${involveItems}`)]);
this.$createElement('br', null), this.$createElement('span', null, `升级对象: ${involveItems}`)]);
this.$message.confirmMessage(msg, () => {
this.loading = true;
this.$http.post('/plugin/upgrade/action.json', params).then(() => {
Expand Down
13 changes: 7 additions & 6 deletions ...src/main/java/com/vip/pallas/console/controller/index/version/IndexVersionController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import javax.validation.constraints.Min;

import com.vip.pallas.console.vo.IndexVersionDynamicVO;
import com.vip.pallas.mybatis.entity.*;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -54,11 +55,6 @@
import com.vip.pallas.console.vo.base.BaseIndexVersionOp;
import com.vip.pallas.exception.BusinessLevelException;
import com.vip.pallas.exception.PallasException;
import com.vip.pallas.mybatis.entity.Cluster;
import com.vip.pallas.mybatis.entity.Index;
import com.vip.pallas.mybatis.entity.IndexOperation;
import com.vip.pallas.mybatis.entity.IndexVersion;
import com.vip.pallas.mybatis.entity.Page;
import com.vip.pallas.service.ClusterService;
import com.vip.pallas.service.ElasticSearchService;
import com.vip.pallas.service.IndexOperationService;
Expand Down Expand Up @@ -467,8 +463,13 @@ public void update(@RequestBody @Validated IndexVersionDynamicVO params, HttpSer
throw new BusinessLevelException(403, "无权限操作");
}

indexVersionService.update(indexVersion);
Object schema = params.getSchema();
List<Mapping> mappingList = indexVersionService.updateDynamic(indexVersion,(ArrayList)schema);
elasticSearchService.dynamicUpdateIndexSettings(index.getIndexName(),index.getId(),indexVersion);
if (mappingList.size()>0){
elasticSearchService.updateEsMapping(index.getIndexName(),index.getId(),versionId,mappingList);
}

try {
AuditLogUtil.log(
"add versionId: id - {0}, index - {1}, maxResultWindow - {2}, numOfReplication - {3}," +
Expand Down
11 changes: 11 additions & 0 deletions pallas-console/src/main/java/com/vip/pallas/console/vo/IndexVersionDynamicVO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,17 @@ public class IndexVersionDynamicVO implements Serializable {

private String translogDurability;

private Object schema;

@NotNull(message = "schema不能为null")
public Object getSchema() {
return schema;
}

public void setSchema(Object schema) {
this.schema = schema;
}

public Long getMaxResultWindow() {
return maxResultWindow;
}
Expand Down
Loading

0 comments on commit e4648f3

Please sign in to comment.