Skip to content

Add enabled flag for _field_names to replace disabling through index:no #9893

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 26, 2015
Merged

Add enabled flag for _field_names to replace disabling through index:no #9893

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 65 additions & 39 deletions src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
package org.elasticsearch.index.mapper.internal;

import com.google.common.collect.UnmodifiableIterator;

import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.SortedSetDocValuesField;
Expand All @@ -30,14 +29,16 @@
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.mapper.InternalMapper;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MergeContext;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.RootMapper;
import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
Expand All @@ -48,6 +49,7 @@
import java.util.List;
import java.util.Map;

import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
import static org.elasticsearch.index.mapper.MapperBuilders.fieldNames;
import static org.elasticsearch.index.mapper.core.TypeParsers.parseField;

Expand All @@ -65,79 +67,83 @@ public class FieldNamesFieldMapper extends AbstractFieldMapper<String> implement

public static class Defaults extends AbstractFieldMapper.Defaults {
public static final String NAME = FieldNamesFieldMapper.NAME;
public static final String INDEX_NAME = FieldNamesFieldMapper.NAME;


public static final EnabledAttributeMapper ENABLED_STATE = EnabledAttributeMapper.UNSET_ENABLED;
public static final FieldType FIELD_TYPE = new FieldType(AbstractFieldMapper.Defaults.FIELD_TYPE);
// TODO: this field should be removed?
public static final FieldType FIELD_TYPE_PRE_1_3_0;

static {
FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
FIELD_TYPE.setTokenized(false);
FIELD_TYPE.setStored(false);
FIELD_TYPE.setOmitNorms(true);
FIELD_TYPE.freeze();
FIELD_TYPE_PRE_1_3_0 = new FieldType(FIELD_TYPE);
FIELD_TYPE_PRE_1_3_0.setIndexOptions(IndexOptions.NONE);
FIELD_TYPE_PRE_1_3_0.freeze();
}
}

public static class Builder extends AbstractFieldMapper.Builder<Builder, FieldNamesFieldMapper> {

private boolean indexIsExplicit;
private EnabledAttributeMapper enabledState = Defaults.ENABLED_STATE;

public Builder() {
super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE));
indexName = Defaults.INDEX_NAME;
indexName = Defaults.NAME;
}

@Override
public Builder index(boolean index) {
indexIsExplicit = true;
enabled(index);
return super.index(index);
}

public Builder enabled(boolean enabled) {
this.enabledState = enabled ? EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED;
return this;
}

@Override
public FieldNamesFieldMapper build(BuilderContext context) {
if ((context.indexCreatedVersion() == null || context.indexCreatedVersion().before(Version.V_1_3_0)) && !indexIsExplicit) {
fieldType.setIndexOptions(IndexOptions.NONE);
}
return new FieldNamesFieldMapper(name, indexName, boost, fieldType, fieldDataSettings, context.indexSettings());
return new FieldNamesFieldMapper(name, indexName, boost, fieldType, enabledState, fieldDataSettings, context.indexSettings());
}
}

public static class TypeParser implements Mapper.TypeParser {
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
if (parserContext.indexVersionCreated().onOrAfter(Version.V_1_3_0)) {
FieldNamesFieldMapper.Builder builder = fieldNames();
parseField(builder, builder.name, node, parserContext);
return builder;
} else {
throw new ElasticsearchIllegalArgumentException("type="+CONTENT_TYPE+" is not supported on indices created before version 1.3.0 is your cluster running multiple datanode versions?");
if (parserContext.indexVersionCreated().before(Version.V_1_3_0)) {
throw new ElasticsearchIllegalArgumentException("type="+CONTENT_TYPE+" is not supported on indices created before version 1.3.0. Is your cluster running multiple datanode versions?");
}

FieldNamesFieldMapper.Builder builder = fieldNames();
parseField(builder, builder.name, node, parserContext);

for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
Map.Entry<String, Object> entry = iterator.next();
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("enabled")) {
builder.enabled(nodeBooleanValue(fieldNode));
iterator.remove();
}
}
return builder;
}
}

private final FieldType defaultFieldType;

private static FieldType defaultFieldType(Settings indexSettings) {
return indexSettings != null && Version.indexCreated(indexSettings).onOrAfter(Version.V_1_3_0) ? Defaults.FIELD_TYPE : Defaults.FIELD_TYPE_PRE_1_3_0;
}
private EnabledAttributeMapper enabledState;

public FieldNamesFieldMapper(Settings indexSettings) {
this(Defaults.NAME, Defaults.INDEX_NAME, indexSettings);
}

protected FieldNamesFieldMapper(String name, String indexName, Settings indexSettings) {
this(name, indexName, Defaults.BOOST, new FieldType(defaultFieldType(indexSettings)), null, indexSettings);
this(Defaults.NAME, Defaults.NAME, Defaults.BOOST, new FieldType(Defaults.FIELD_TYPE), Defaults.ENABLED_STATE, null, indexSettings);
}

public FieldNamesFieldMapper(String name, String indexName, float boost, FieldType fieldType, @Nullable Settings fieldDataSettings, Settings indexSettings) {
public FieldNamesFieldMapper(String name, String indexName, float boost, FieldType fieldType, EnabledAttributeMapper enabledState, @Nullable Settings fieldDataSettings, Settings indexSettings) {
super(new Names(name, indexName, indexName, name), boost, fieldType, null, Lucene.KEYWORD_ANALYZER,
Lucene.KEYWORD_ANALYZER, null, null, fieldDataSettings, indexSettings);
this.defaultFieldType = defaultFieldType(indexSettings);
this.defaultFieldType = Defaults.FIELD_TYPE;
this.enabledState = enabledState;
}

public boolean enabled() {
return enabledState.enabled;
}

@Override
Expand Down Expand Up @@ -216,7 +222,7 @@ public String next() {

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
if (fieldType.indexOptions() == IndexOptions.NONE && !fieldType.stored() && !hasDocValues()) {
if (enabledState.enabled == false) {
return;
}
for (ParseContext.Document document : context.docs()) {
Expand Down Expand Up @@ -244,12 +250,32 @@ protected String contentType() {

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
XContentBuilder json = XContentFactory.jsonBuilder();
super.toXContent(json, params);
if (json.string().equals("\"" + NAME + "\"{\"type\":\"" + CONTENT_TYPE + "\"}")) {
boolean includeDefaults = params.paramAsBoolean("include_defaults", false);

if (includeDefaults == false && fieldType().equals(Defaults.FIELD_TYPE) && enabledState == Defaults.ENABLED_STATE) {
return builder;
}
return super.toXContent(builder, params);

builder.startObject(NAME);
if (includeDefaults || enabledState != Defaults.ENABLED_STATE) {
builder.field("enabled", enabledState.enabled);
}
if (includeDefaults || fieldType().equals(Defaults.FIELD_TYPE) == false) {
super.doXContentBody(builder, includeDefaults, params);
}

builder.endObject();
return builder;
}

@Override
public void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappingException {
FieldNamesFieldMapper fieldNamesMapperMergeWith = (FieldNamesFieldMapper)mergeWith;
if (!mergeContext.mergeFlags().simulate()) {
if (fieldNamesMapperMergeWith.enabledState != enabledState && !fieldNamesMapperMergeWith.enabledState.unset()) {
this.enabledState = fieldNamesMapperMergeWith.enabledState;
}
}
}

@Override
Expand Down
119 changes: 102 additions & 17 deletions src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,25 +19,33 @@

package org.elasticsearch.index.mapper.internal;

import com.google.common.collect.ImmutableSet;
import org.apache.lucene.index.IndexableField;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.DocumentMapperParser;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.test.ElasticsearchSingleNodeTest;

import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

public class FieldNamesFieldMapperTests extends ElasticsearchSingleNodeTest {

private static Set<String> extract(String path) {
return ImmutableSet.<String>builder().addAll(FieldNamesFieldMapper.extractFieldNames(path)).build();
private static SortedSet<String> extract(String path) {
SortedSet<String> set = new TreeSet<>();
for (String fieldName : FieldNamesFieldMapper.extractFieldNames(path)) {
set.add(fieldName);
}
return set;
}

private static <T> SortedSet<T> set(T... values) {
return new TreeSet<>(Arrays.asList(values));
}

private static <T> Set<T> set(T... values) {
return new HashSet<T>(Arrays.asList(values));
void assertFieldNames(SortedSet<String> expected, ParsedDocument doc) {
String[] got = doc.rootDoc().getValues("_field_names");
assertEquals(expected, set(got));
}

public void testExtractFieldNames() {
Expand All @@ -50,8 +58,9 @@ public void testExtractFieldNames() {
assertEquals(set("", ".", ".."), extract(".."));
}

public void test() throws Exception {
DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string());
public void testInjectIntoDocDuringParsing() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);

ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
Expand All @@ -61,13 +70,89 @@ public void test() throws Exception {
.endObject()
.endObject()
.bytes());

assertFieldNames(set("a", "b", "b.c", "_uid", "_type", "_version", "_source", "_all"), doc);
}

public void testExplicitEnabled() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_field_names").field("enabled", true).endObject()
.endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
FieldNamesFieldMapper fieldNamesMapper = docMapper.rootMapper(FieldNamesFieldMapper.class);
assertTrue(fieldNamesMapper.enabled());

final Set<String> fieldNames = new HashSet<>();
for (IndexableField field : doc.rootDoc().getFields()) {
if (FieldNamesFieldMapper.CONTENT_TYPE.equals(field.name())) {
fieldNames.add(field.stringValue());
}
}
assertEquals(new HashSet<>(Arrays.asList("a", "b", "b.c", "_uid", "_type", "_version", "_source", "_all")), fieldNames);
ParsedDocument doc = docMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field", "value")
.endObject()
.bytes());

assertFieldNames(set("field", "_uid", "_type", "_version", "_source", "_all"), doc);
}

public void testDisabled() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_field_names").field("enabled", false).endObject()
.endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
FieldNamesFieldMapper fieldNamesMapper = docMapper.rootMapper(FieldNamesFieldMapper.class);
assertFalse(fieldNamesMapper.enabled());

ParsedDocument doc = docMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field", "value")
.endObject()
.bytes());

assertNull(doc.rootDoc().get("_field_names"));
}

public void testDisablingBackcompat() throws Exception {
// before 1.5, disabling happened by setting index:no
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_field_names").field("index", "no").endObject()
.endObject().endObject().string();

DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
FieldNamesFieldMapper fieldNamesMapper = docMapper.rootMapper(FieldNamesFieldMapper.class);
assertFalse(fieldNamesMapper.enabled());

ParsedDocument doc = docMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field", "value")
.endObject()
.bytes());

assertNull(doc.rootDoc().get("_field_names"));
}

public void testFieldTypeSettings() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_field_names").field("store", "yes").endObject()
.endObject().endObject().string();

DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
FieldNamesFieldMapper fieldNamesMapper = docMapper.rootMapper(FieldNamesFieldMapper.class);
assertTrue(fieldNamesMapper.fieldType().stored());
}

public void testMergingMappings() throws Exception {
String enabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_field_names").field("enabled", true).endObject()
.endObject().endObject().string();
String disabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_field_names").field("enabled", false).endObject()
.endObject().endObject().string();
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();

DocumentMapper mapperEnabled = parser.parse(enabledMapping);
DocumentMapper mapperDisabled = parser.parse(disabledMapping);
mapperEnabled.merge(mapperDisabled, DocumentMapper.MergeFlags.mergeFlags().simulate(false));
assertFalse(mapperEnabled.rootMapper(FieldNamesFieldMapper.class).enabled());

mapperEnabled = parser.parse(enabledMapping);
mapperDisabled.merge(mapperEnabled, DocumentMapper.MergeFlags.mergeFlags().simulate(false));
assertTrue(mapperEnabled.rootMapper(FieldNamesFieldMapper.class).enabled());
}
}