From 2f8382672c038fbacd5353d5e77d562e680c40fa Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Fri, 31 May 2024 23:05:25 +0530 Subject: [PATCH 1/4] Star tree index config changes Signed-off-by: Bharathwaj G --- CHANGELOG.md | 1 + distribution/src/config/opensearch.yml | 4 + .../metadata/MetadataCreateIndexService.java | 2 + .../common/settings/ClusterSettings.java | 12 +- .../common/settings/FeatureFlagSettings.java | 3 +- .../common/settings/IndexScopedSettings.java | 4 +- .../opensearch/common/util/FeatureFlags.java | 10 + .../org/opensearch/index/IndexModule.java | 7 +- .../org/opensearch/index/IndexService.java | 19 +- .../org/opensearch/index/IndexSettings.java | 8 + .../index/compositeindex/CompositeField.java | 49 ++ .../compositeindex/CompositeFieldSpec.java | 22 + .../compositeindex/CompositeIndexConfig.java | 505 ++++++++++++++++++ .../CompositeIndexSettings.java | 179 +++++++ .../index/compositeindex/DateDimension.java | 55 ++ .../index/compositeindex/Dimension.java | 33 ++ .../index/compositeindex/Metric.java | 42 ++ .../index/compositeindex/MetricType.java | 42 ++ .../compositeindex/StarTreeFieldSpec.java | 73 +++ .../index/compositeindex/package-info.java | 13 + .../DefaultCompositeIndexSettings.java | 28 + .../opensearch/indices/IndicesService.java | 9 +- .../main/java/org/opensearch/node/Node.java | 6 +- .../MetadataRolloverServiceTests.java | 1 + .../opensearch/index/IndexModuleTests.java | 4 +- .../CompositeIndexConfigSettingsTests.java | 233 ++++++++ .../indices/cluster/ClusterStateChanges.java | 1 + .../snapshots/SnapshotResiliencyTests.java | 4 +- 28 files changed, 1357 insertions(+), 12 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/Dimension.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/Metric.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/MetricType.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/package-info.java create mode 100644 server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 539f5a6628dac..ca83a4b296e6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Query Insights] Add exporter support for top n queries ([#12982](https://github.com/opensearch-project/OpenSearch/pull/12982)) - [Query Insights] Add X-Opaque-Id to search request metadata for top n queries ([#13374](https://github.com/opensearch-project/OpenSearch/pull/13374)) - Add support for query level resource usage tracking ([#13172](https://github.com/opensearch-project/OpenSearch/pull/13172)) +- [Star Tree Index] Star tree index config changes ([#13917](https://github.com/opensearch-project/OpenSearch/pull/13917)) ### Dependencies - Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559)) diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml index 10bab9b3fce92..b44248cf74335 100644 --- a/distribution/src/config/opensearch.yml +++ b/distribution/src/config/opensearch.yml @@ -125,3 +125,7 @@ ${path.logs} # Gates the functionality of enabling Opensearch to use pluggable caches with respective store names via setting. # #opensearch.experimental.feature.pluggable.caching.enabled: false +# +# Gates the functionality of star tree index, which improves the performance of search aggregations. +# +#opensearch.experimental.feature.composite_index.enabled: true diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 16edec112f123..59e78605c33cb 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -1324,6 +1324,8 @@ private static void updateIndexMappingsAndBuildSortOrder( // at this point. The validation will take place later in the process // (when all shards are copied in a single place). indexService.getIndexSortSupplier().get(); + // validate composite index fields + indexService.getCompositeIndexConfigSupplier().get(); } if (request.dataStreamName() != null) { MetadataCreateDataStreamService.validateTimestampFieldMapping(mapperService); diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 09f32884e0ae1..457bda1410500 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -114,6 +114,7 @@ import org.opensearch.index.ShardIndexingPressureMemoryManager; import org.opensearch.index.ShardIndexingPressureSettings; import org.opensearch.index.ShardIndexingPressureStore; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.remote.RemoteStorePressureSettings; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; import org.opensearch.index.store.remote.filecache.FileCacheSettings; @@ -745,7 +746,16 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_MAX_TRANSLOG_READERS, RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA, - SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING + SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING, + RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA, + + // Composite index settings + CompositeIndexSettings.COMPOSITE_INDEX_ENABLED_SETTING, + CompositeIndexSettings.COMPOSITE_INDEX_MAX_FIELDS_SETTING, + CompositeIndexSettings.COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING, + CompositeIndexSettings.STAR_TREE_DEFAULT_MAX_LEAF_DOCS, + CompositeIndexSettings.DEFAULT_METRICS_LIST, + CompositeIndexSettings.DEFAULT_DATE_INTERVALS ) ) ); diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 238df1bd90113..1bb1c2462ffda 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -36,7 +36,8 @@ protected FeatureFlagSettings( FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING, FeatureFlags.TIERED_REMOTE_INDEX_SETTING, FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, - FeatureFlags.PLUGGABLE_CACHE_SETTING, FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL_SETTING + FeatureFlags.PLUGGABLE_CACHE_SETTING, + FeatureFlags.COMPOSITE_INDEX_SETTING ); } diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 6fe8dec9c21b1..38e93b22806ea 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -249,8 +249,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { } } }, Property.IndexScope), // this allows similarity settings to be passed - Setting.groupSetting("index.analysis.", Property.IndexScope) // this allows analysis settings to be passed - + Setting.groupSetting("index.analysis.", Property.IndexScope), // this allows analysis settings to be passed + Setting.groupSetting("index.composite_index.", Property.IndexScope) // this allows composite index settings to be passed ) ) ); diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 82f43921d2d28..da69bb4eda3da 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -72,6 +72,12 @@ public class FeatureFlags { */ public static final String REMOTE_PUBLICATION_EXPERIMENTAL = "opensearch.experimental.feature.remote_store.publication.enabled"; + /** + * Gates the functionality of composite index i.e. star tree index, which improves the performance of search + * aggregations. + */ + public static final String COMPOSITE_INDEX = "opensearch.experimental.feature.composite_index.enabled"; + public static final Setting REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING = Setting.boolSetting( REMOTE_STORE_MIGRATION_EXPERIMENTAL, false, @@ -94,6 +100,8 @@ public class FeatureFlags { public static final Setting PLUGGABLE_CACHE_SETTING = Setting.boolSetting(PLUGGABLE_CACHE, false, Property.NodeScope); + public static final Setting COMPOSITE_INDEX_SETTING = Setting.boolSetting(COMPOSITE_INDEX, false, Property.NodeScope); + public static final Setting REMOTE_PUBLICATION_EXPERIMENTAL_SETTING = Setting.boolSetting( REMOTE_PUBLICATION_EXPERIMENTAL, false, @@ -108,6 +116,8 @@ public class FeatureFlags { DATETIME_FORMATTER_CACHING_SETTING, TIERED_REMOTE_INDEX_SETTING, PLUGGABLE_CACHE_SETTING, + COMPOSITE_INDEX_SETTING, + PLUGGABLE_CACHE_SETTING, REMOTE_PUBLICATION_EXPERIMENTAL_SETTING ); /** diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index 3c4cb4fd596c1..aaec6bfec2123 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -66,6 +66,7 @@ import org.opensearch.index.cache.query.DisabledQueryCache; import org.opensearch.index.cache.query.IndexQueryCache; import org.opensearch.index.cache.query.QueryCache; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; @@ -606,7 +607,8 @@ public IndexService newIndexService( BiFunction translogFactorySupplier, Supplier clusterDefaultRefreshIntervalSupplier, RecoverySettings recoverySettings, - RemoteStoreSettings remoteStoreSettings + RemoteStoreSettings remoteStoreSettings, + CompositeIndexSettings compositeIndexSettings ) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = indexReaderWrapper @@ -665,7 +667,8 @@ public IndexService newIndexService( translogFactorySupplier, clusterDefaultRefreshIntervalSupplier, recoverySettings, - remoteStoreSettings + remoteStoreSettings, + compositeIndexSettings ); success = true; return indexService; diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index e501d7eff3f81..6f994e8f67c5b 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -72,6 +72,8 @@ import org.opensearch.index.cache.IndexCache; import org.opensearch.index.cache.bitset.BitsetFilterCache; import org.opensearch.index.cache.query.QueryCache; +import org.opensearch.index.compositeindex.CompositeIndexConfig; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; @@ -183,6 +185,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final CircuitBreakerService circuitBreakerService; private final IndexNameExpressionResolver expressionResolver; private final Supplier indexSortSupplier; + private final Supplier compositeIndexConfigSupplier; private final ValuesSourceRegistry valuesSourceRegistry; private final BiFunction translogFactorySupplier; private final Supplier clusterDefaultRefreshIntervalSupplier; @@ -223,7 +226,8 @@ public IndexService( BiFunction translogFactorySupplier, Supplier clusterDefaultRefreshIntervalSupplier, RecoverySettings recoverySettings, - RemoteStoreSettings remoteStoreSettings + RemoteStoreSettings remoteStoreSettings, + CompositeIndexSettings compositeIndexSettings ) { super(indexSettings); this.allowExpensiveQueries = allowExpensiveQueries; @@ -261,6 +265,14 @@ public IndexService( } else { this.indexSortSupplier = () -> null; } + + if (indexSettings.getCompositeIndexConfig().hasCompositeFields()) { + this.compositeIndexConfigSupplier = () -> indexSettings.getCompositeIndexConfig() + .validateAndGetCompositeIndexConfig(mapperService::fieldType, compositeIndexSettings); + } else { + this.compositeIndexConfigSupplier = () -> null; + } + indexFieldData.setListener(new FieldDataCacheListener(this)); this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this)); this.warmer = new IndexWarmer(threadPool, indexFieldData, bitsetFilterCache.createListener(threadPool)); @@ -273,6 +285,7 @@ public IndexService( this.bitsetFilterCache = null; this.warmer = null; this.indexCache = null; + this.compositeIndexConfigSupplier = () -> null; } this.shardStoreDeleter = shardStoreDeleter; @@ -385,6 +398,10 @@ public Supplier getIndexSortSupplier() { return indexSortSupplier; } + public Supplier getCompositeIndexConfigSupplier() { + return compositeIndexConfigSupplier; + } + public synchronized void close(final String reason, boolean delete) throws IOException { if (closed.compareAndSet(false, true)) { deleted.compareAndSet(false, delete); diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 613e93698d683..d344e9f580e96 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -48,6 +48,7 @@ import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.Index; +import org.opensearch.index.compositeindex.CompositeIndexConfig; import org.opensearch.index.remote.RemoteStorePathStrategy; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.translog.Translog; @@ -760,6 +761,8 @@ public static IndexMergePolicy fromString(String text) { private final LogByteSizeMergePolicyProvider logByteSizeMergePolicyProvider; private final IndexSortConfig indexSortConfig; private final IndexScopedSettings scopedSettings; + + private final CompositeIndexConfig compositeIndexConfig; private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis(); private final boolean softDeleteEnabled; private volatile long softDeleteRetentionOperations; @@ -985,6 +988,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this.tieredMergePolicyProvider = new TieredMergePolicyProvider(logger, this); this.logByteSizeMergePolicyProvider = new LogByteSizeMergePolicyProvider(logger, this); this.indexSortConfig = new IndexSortConfig(this); + this.compositeIndexConfig = new CompositeIndexConfig(this); searchIdleAfter = scopedSettings.get(INDEX_SEARCH_IDLE_AFTER); defaultPipeline = scopedSettings.get(DEFAULT_PIPELINE); setTranslogRetentionAge(scopedSettings.get(INDEX_TRANSLOG_RETENTION_AGE_SETTING)); @@ -1740,6 +1744,10 @@ public IndexSortConfig getIndexSortConfig() { return indexSortConfig; } + public CompositeIndexConfig getCompositeIndexConfig() { + return compositeIndexConfig; + } + public IndexScopedSettings getScopedSettings() { return scopedSettings; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java new file mode 100644 index 0000000000000..a0b95b0409da0 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.List; + +/** + * Composite field which contains dimensions, metrics and index mode specific specs + * + * @opensearch.experimental + */ +@ExperimentalApi +public class CompositeField { + private final String name; + private final List dimensionsOrder; + private final List metrics; + private final CompositeFieldSpec compositeFieldSpec; + + public CompositeField(String name, List dimensions, List metrics, CompositeFieldSpec compositeFieldSpec) { + this.name = name; + this.dimensionsOrder = dimensions; + this.metrics = metrics; + this.compositeFieldSpec = compositeFieldSpec; + } + + public String getName() { + return name; + } + + public List getDimensionsOrder() { + return dimensionsOrder; + } + + public List getMetrics() { + return metrics; + } + + public CompositeFieldSpec getSpec() { + return compositeFieldSpec; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java new file mode 100644 index 0000000000000..51a2b454abcd9 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * CompositeFieldSpec interface. + * + * @opensearch.experimental + */ + +@ExperimentalApi +public interface CompositeFieldSpec { + void setDefaults(CompositeIndexSettings compositeIndexSettings); +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java new file mode 100644 index 0000000000000..308c4b62d3996 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java @@ -0,0 +1,505 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.Rounding; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +import static org.opensearch.index.compositeindex.CompositeIndexSettings.COMPOSITE_INDEX_ENABLED_SETTING; + +/** + * Configuration of composite index containing list of composite fields. + * Each composite field contains dimensions, metrics along with composite index (eg: star tree) specific settings. + * Each composite field will generate a composite index in indexing flow. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class CompositeIndexConfig { + private static final Set> ALLOWED_DIMENSION_MAPPED_FIELD_TYPES = Set.of( + NumberFieldMapper.NumberFieldType.class, + DateFieldMapper.DateFieldType.class + ); + + private static final Set> ALLOWED_METRIC_MAPPED_FIELD_TYPES = Set.of( + NumberFieldMapper.NumberFieldType.class + ); + + private static final String COMPOSITE_INDEX_CONFIG = "index.composite_index.config"; + private static final String DIMENSIONS_ORDER = "dimensions_order"; + private static final String DIMENSIONS_CONFIG = "dimensions_config"; + private static final String METRICS = "metrics"; + private static final String METRICS_CONFIG = "metrics_config"; + private static final String FIELD = "field"; + private static final String TYPE = "type"; + private static final String INDEX_MODE = "index_mode"; + private static final String STAR_TREE_BUILD_MODE = "build_mode"; + private static final String MAX_LEAF_DOCS = "max_leaf_docs"; + private static final String SKIP_STAR_NODE_CREATION_FOR_DIMS = "skip_star_node_creation_for_dimensions"; + private static final String SPEC = "_spec"; + private final List compositeFields = new ArrayList<>(); + + public CompositeIndexConfig(IndexSettings indexSettings) { + + final Map compositeIndexSettings = indexSettings.getSettings().getGroups(COMPOSITE_INDEX_CONFIG); + Set fields = compositeIndexSettings.keySet(); + for (String field : fields) { + compositeFields.add(buildCompositeField(field, compositeIndexSettings.get(field))); + } + } + + /** + * This returns composite field after performing basic validations and doesn't do field type validations etc + * + */ + private CompositeField buildCompositeField(String field, Settings compositeFieldSettings) { + List dimensions = new ArrayList<>(); + List metrics = new ArrayList<>(); + List dimensionsOrder = compositeFieldSettings.getAsList(DIMENSIONS_ORDER); + if (dimensionsOrder == null || dimensionsOrder.isEmpty()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "dimensions_order is required for composite index field [%s]", field) + ); + } + if (dimensionsOrder.size() < 2) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Atleast two dimensions are required to build composite index field [%s]", field) + ); + } + + Map dimConfig = compositeFieldSettings.getGroups(DIMENSIONS_CONFIG); + + for (String dimension : dimConfig.keySet()) { + if (!dimensionsOrder.contains(dimension)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "dimension [%s] is not present in dimensions_order for composite index field [%s]", + dimension, + field + ) + ); + } + } + + List metricFields = compositeFieldSettings.getAsList(METRICS); + if (metricFields == null || metricFields.isEmpty()) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "metrics is required for composite index field [%s]", field)); + } + Map metricsConfig = compositeFieldSettings.getGroups(METRICS_CONFIG); + + for (String metricField : metricsConfig.keySet()) { + if (!metricFields.contains(metricField)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "metric field [%s] is not present in 'metrics' for composite index field [%s]", + metricField, + field + ) + ); + } + } + + Set uniqueDimensions = new HashSet<>(); + for (String dimension : dimensionsOrder) { + if (!uniqueDimensions.add(dimension)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "duplicate dimension [%s] found in dimensions_order for composite index field [%s]", + dimension, + field + ) + ); + } + dimensions.add(DimensionFactory.create(dimension, dimConfig.get(dimension))); + } + uniqueDimensions = null; + Set uniqueMetricFields = new HashSet<>(); + for (String metricField : metricFields) { + if (!uniqueMetricFields.add(metricField)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "duplicate metric field [%s] found in 'metrics' for composite index field [%s]", + metricField, + field + ) + ); + } + Settings metricSettings = metricsConfig.get(metricField); + if (metricSettings == null) { + // fill cluster level defaults in create flow as part of CompositeIndexSupplier + metrics.add(new Metric(metricField, new ArrayList<>())); + } else { + String name = metricSettings.get(FIELD, metricField); + List metricsList = metricSettings.getAsList(METRICS); + if (metricsList.isEmpty()) { + // fill cluster level defaults in create flow as part of CompositeIndexSupplier + metrics.add(new Metric(name, new ArrayList<>())); + } else { + List metricTypes = new ArrayList<>(); + Set uniqueMetricTypes = new HashSet<>(); + for (String metric : metricsList) { + if (!uniqueMetricTypes.add(metric)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "duplicate metric type [%s] found in metrics for composite index field [%s]", + metric, + field + ) + ); + } + metricTypes.add(MetricType.fromTypeName(metric)); + } + uniqueMetricTypes = null; + metrics.add(new Metric(name, metricTypes)); + } + } + } + uniqueMetricFields = null; + + IndexMode indexMode = IndexMode.fromTypeName(compositeFieldSettings.get(INDEX_MODE, IndexMode.STARTREE.typeName)); + Settings fieldSpec = compositeFieldSettings.getAsSettings(indexMode.typeName + SPEC); + CompositeFieldSpec compositeFieldSpec = CompositeFieldSpecFactory.create(indexMode, fieldSpec, dimensionsOrder); + return new CompositeField(field, dimensions, metrics, compositeFieldSpec); + } + + public static Rounding.DateTimeUnit getTimeUnit(String expression) { + if (!DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) { + throw new IllegalArgumentException("unknown calendar interval specified in composite index config"); + } + return DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression); + } + + /** + * Dimension factory based on field type + */ + private static class DimensionFactory { + static Dimension create(String dimension, Settings settings) { + if (settings == null) { + return new Dimension(dimension); + } + String field = settings.get(FIELD, dimension); + String type = settings.get(TYPE, DimensionType.DEFAULT.getTypeName()); + switch (DimensionType.fromTypeName(type)) { + case DEFAULT: + return new Dimension(field); + case DATE: + return new DateDimension(field, settings); + default: + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid dimension type [%s] in composite index config", type) + ); + } + } + + static Dimension createEmptyMappedDimension(Dimension dimension, MappedFieldType type) { + if (type instanceof DateFieldMapper.DateFieldType) { + return new DateDimension(dimension.getField(), new ArrayList<>()); + } + return dimension; + } + } + + /** + * The type of dimension source fields + * Default fields are of Numeric type + */ + private enum DimensionType { + DEFAULT("default"), + DATE("date"); + + private final String typeName; + + DimensionType(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static DimensionType fromTypeName(String typeName) { + for (DimensionType dimensionType : DimensionType.values()) { + if (dimensionType.getTypeName().equalsIgnoreCase(typeName)) { + return dimensionType; + } + } + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid dimension type in composite index config: [%s] ", typeName) + ); + } + } + + /** + * Composite field spec factory based on index mode + */ + private static class CompositeFieldSpecFactory { + static CompositeFieldSpec create(IndexMode indexMode, Settings settings, List dimensions) { + if (settings == null) { + return new StarTreeFieldSpec(10000, new ArrayList<>(), StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP); + } + switch (indexMode) { + case STARTREE: + return buildStarTreeFieldSpec(settings, dimensions); + default: + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid index mode [%s] in composite index config", indexMode) + ); + } + } + } + + private static StarTreeFieldSpec buildStarTreeFieldSpec(Settings settings, List dimensions) { + StarTreeFieldSpec.StarTreeBuildMode buildMode = StarTreeFieldSpec.StarTreeBuildMode.fromTypeName( + settings.get(STAR_TREE_BUILD_MODE, StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP.getTypeName()) + ); + // Fill default value as part of create flow as part of supplier + int maxLeafDocs = settings.getAsInt(MAX_LEAF_DOCS, Integer.MAX_VALUE); + if (maxLeafDocs < 1) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid max_leaf_docs [%s] in composite index config", maxLeafDocs) + ); + } + List skipStarNodeCreationInDims = settings.getAsList(SKIP_STAR_NODE_CREATION_FOR_DIMS, new ArrayList<>()); + for (String dim : skipStarNodeCreationInDims) { + if (!dimensions.contains(dim)) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Invalid dimension [%s] in skip_star_node_creation_for_dims", dim) + ); + } + } + return new StarTreeFieldSpec(maxLeafDocs, skipStarNodeCreationInDims, buildMode); + } + + /** + * Enum for index mode of the underlying composite index + * The default and only index supported right now is star tree index + */ + private enum IndexMode { + STARTREE("startree"); + + private final String typeName; + + IndexMode(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static IndexMode fromTypeName(String typeName) { + for (IndexMode indexType : IndexMode.values()) { + if (indexType.getTypeName().equalsIgnoreCase(typeName)) { + return indexType; + } + } + throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid index mode in composite index config: [%s] ", typeName)); + } + } + + /** + * Returns the composite fields built based on compositeIndexConfig index settings + */ + public List getCompositeFields() { + return compositeFields; + } + + /** + * Returns whether there are any composite fields as part of the compositeIndexConfig + */ + public boolean hasCompositeFields() { + return !compositeFields.isEmpty(); + } + + /** + * Validates the composite fields based on IndexSettingDefaults and the mappedFieldType + * Updates CompositeIndexConfig with newer, completely updated composite fields + * + */ + public CompositeIndexConfig validateAndGetCompositeIndexConfig( + Function fieldTypeLookup, + CompositeIndexSettings compositeIndexSettings + ) { + if (hasCompositeFields() == false) { + return null; + } + if (!compositeIndexSettings.isEnabled()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "composite index cannot be created, enable it using [%s] setting", + COMPOSITE_INDEX_ENABLED_SETTING.getKey() + ) + ); + } + if (compositeFields.size() > compositeIndexSettings.getMaxFields()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "composite index can have atmost [%s] fields", compositeIndexSettings.getMaxFields()) + ); + } + List validatedAndMappedCompositeFields = new ArrayList<>(); + for (CompositeField compositeField : compositeFields) { + if (compositeField.getDimensionsOrder().size() > compositeIndexSettings.getMaxDimensions()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "composite index can have atmost [%s] dimensions", compositeIndexSettings.getMaxDimensions()) + ); + } + List dimensions = new ArrayList<>(); + for (Dimension dimension : compositeField.getDimensionsOrder()) { + validateCompositeDimensionField(dimension.getField(), fieldTypeLookup, compositeField.getName()); + dimension = mapDimension(dimension, fieldTypeLookup.apply(dimension.getField())); + dimension.setDefaults(compositeIndexSettings); + dimensions.add(dimension); + } + List metrics = new ArrayList<>(); + for (Metric metric : compositeField.getMetrics()) { + validateCompositeMetricField(metric.getField(), fieldTypeLookup, compositeField.getName()); + metric.setDefaults(compositeIndexSettings); + metrics.add(metric); + } + compositeField.getSpec().setDefaults(compositeIndexSettings); + validatedAndMappedCompositeFields.add( + new CompositeField(compositeField.getName(), dimensions, metrics, compositeField.getSpec()) + ); + } + this.compositeFields.clear(); + this.compositeFields.addAll(validatedAndMappedCompositeFields); + return this; + } + + /** + * Maps the dimension to right dimension type based on MappedFieldType + */ + private Dimension mapDimension(Dimension dimension, MappedFieldType fieldType) { + if (!isDimensionMappedToFieldType(dimension, fieldType)) { + return DimensionFactory.createEmptyMappedDimension(dimension, fieldType); + } + return dimension; + } + + /** + * Checks whether dimension field type is same as the source field type + */ + private boolean isDimensionMappedToFieldType(Dimension dimension, MappedFieldType fieldType) { + if (fieldType instanceof DateFieldMapper.DateFieldType) { + return dimension instanceof DateDimension; + } + return true; + } + + /** + * Validations : + * The dimension field should be one of the source fields of the index + * The dimension fields must be aggregation compatible (doc values + field data supported) + * The dimension fields should be of numberField type / dateField type + * + */ + private void validateCompositeDimensionField( + String field, + Function fieldTypeLookup, + String compositeFieldName + ) { + final MappedFieldType ft = fieldTypeLookup.apply(field); + if (ft == null) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "unknown dimension field [%s] as part of composite field [%s]", field, compositeFieldName) + ); + } + if (!isAllowedDimensionFieldType(ft)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "composite index is not supported for the dimension field [%s] with field type [%s] as part of " + + "composite field [%s]", + field, + ft.typeName(), + compositeFieldName + ) + ); + } + // doc values not present / field data not supported + if (!ft.isAggregatable()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Aggregations not supported for the dimension field [%s] with field type [%s] as part of " + "composite field [%s]", + field, + ft.typeName(), + compositeFieldName + ) + ); + } + } + + /** + * Validations : + * The metric field should be one of the source fields of the index + * The metric fields must be aggregation compatible (doc values + field data supported) + * The metric fields should be of numberField type + * + */ + private void validateCompositeMetricField(String field, Function fieldTypeLookup, String compositeFieldName) { + final MappedFieldType ft = fieldTypeLookup.apply(field); + if (ft == null) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "unknown metric field [%s] as part of composite field [%s]", field, compositeFieldName) + ); + } + if (!isAllowedMetricFieldType(ft)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "composite index is not supported for the metric field [%s] with field type [%s] as part of " + "composite field [%s]", + field, + ft.typeName(), + compositeFieldName + ) + ); + } + if (!ft.isAggregatable()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Aggregations not supported for the composite index metric field [%s] with field type [%s] as part of " + + "composite field [%s]", + field, + ft.typeName(), + compositeFieldName + ) + ); + } + } + + private static boolean isAllowedDimensionFieldType(MappedFieldType fieldType) { + return ALLOWED_DIMENSION_MAPPED_FIELD_TYPES.stream().anyMatch(allowedType -> allowedType.isInstance(fieldType)); + } + + private static boolean isAllowedMetricFieldType(MappedFieldType fieldType) { + return ALLOWED_METRIC_MAPPED_FIELD_TYPES.stream().anyMatch(allowedType -> allowedType.isInstance(fieldType)); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java new file mode 100644 index 0000000000000..59bdfda5469ea --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java @@ -0,0 +1,179 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.Rounding; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.util.FeatureFlags; + +import java.util.Arrays; +import java.util.List; + +/** + * Cluster level settings which configures defaults for composite index + */ +@ExperimentalApi +public class CompositeIndexSettings { + /** + * This cluster level setting determines whether composite index is enabled or not + */ + public static final Setting COMPOSITE_INDEX_ENABLED_SETTING = Setting.boolSetting( + "indices.composite_index.enabled", + false, + value -> { + if (FeatureFlags.isEnabled(FeatureFlags.COMPOSITE_INDEX_SETTING) == false && value == true) { + throw new IllegalArgumentException( + "star tree index is under an experimental feature and can be activated only by enabling " + + FeatureFlags.COMPOSITE_INDEX_SETTING.getKey() + + " feature flag in the JVM options" + ); + } + }, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * This setting determines the max number of composite fields that can be part of composite index config. For each + * composite field, we will generate associated composite index. (eg : star tree index per field ) + */ + public static final Setting COMPOSITE_INDEX_MAX_FIELDS_SETTING = Setting.intSetting( + "indices.composite_index.max_fields", + 1, + 1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * This setting determines the max number of dimensions that can be part of composite index field. Number of + * dimensions and associated cardinality has direct effect of composite index size and query performance. + */ + public static final Setting COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING = Setting.intSetting( + "indices.composite_index.field.max_dimensions", + 10, + 2, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * This setting configures the default "maxLeafDocs" setting of star tree. This affects both query performance and + * star tree index size. Lesser the leaves, better the query latency but higher storage size and vice versa + *

+ * We can remove this later or change it to an enum based constant setting. + * + * @opensearch.experimental + */ + public static final Setting STAR_TREE_DEFAULT_MAX_LEAF_DOCS = Setting.intSetting( + "indices.composite_index.startree.default.max_leaf_docs", + 10000, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * Default intervals for date dimension as part of composite fields + */ + public static final Setting> DEFAULT_DATE_INTERVALS = Setting.listSetting( + "indices.composite_index.field.default.date_intervals", + Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), Rounding.DateTimeUnit.HOUR_OF_DAY.shortName()), + CompositeIndexConfig::getTimeUnit, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + public static final Setting> DEFAULT_METRICS_LIST = Setting.listSetting( + "indices.composite_index.field.default.metrics", + Arrays.asList( + MetricType.AVG.toString(), + MetricType.COUNT.toString(), + MetricType.SUM.toString(), + MetricType.MAX.toString(), + MetricType.MIN.toString() + ), + MetricType::fromTypeName, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + private volatile int maxLeafDocs; + + private volatile List defaultDateIntervals; + + private volatile List defaultMetrics; + private volatile int maxDimensions; + private volatile int maxFields; + private volatile boolean enabled; + + public CompositeIndexSettings(ClusterSettings clusterSettings) { + this.setMaxLeafDocs(clusterSettings.get(STAR_TREE_DEFAULT_MAX_LEAF_DOCS)); + this.setDefaultDateIntervals(clusterSettings.get(DEFAULT_DATE_INTERVALS)); + this.setDefaultMetrics(clusterSettings.get(DEFAULT_METRICS_LIST)); + this.setMaxDimensions(clusterSettings.get(COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING)); + this.setMaxFields(clusterSettings.get(COMPOSITE_INDEX_MAX_FIELDS_SETTING)); + this.setEnabled(clusterSettings.get(COMPOSITE_INDEX_ENABLED_SETTING)); + + clusterSettings.addSettingsUpdateConsumer(STAR_TREE_DEFAULT_MAX_LEAF_DOCS, this::setMaxLeafDocs); + clusterSettings.addSettingsUpdateConsumer(DEFAULT_DATE_INTERVALS, this::setDefaultDateIntervals); + clusterSettings.addSettingsUpdateConsumer(DEFAULT_METRICS_LIST, this::setDefaultMetrics); + clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING, this::setMaxDimensions); + clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_MAX_FIELDS_SETTING, this::setMaxFields); + clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_ENABLED_SETTING, this::setEnabled); + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public void setMaxLeafDocs(int maxLeafDocs) { + this.maxLeafDocs = maxLeafDocs; + } + + public void setDefaultDateIntervals(List defaultDateIntervals) { + this.defaultDateIntervals = defaultDateIntervals; + } + + public void setDefaultMetrics(List defaultMetrics) { + this.defaultMetrics = defaultMetrics; + } + + public void setMaxDimensions(int maxDimensions) { + this.maxDimensions = maxDimensions; + } + + public void setMaxFields(int maxFields) { + this.maxFields = maxFields; + } + + public int getMaxDimensions() { + return maxDimensions; + } + + public int getMaxFields() { + return maxFields; + } + + public int getMaxLeafDocs() { + return maxLeafDocs; + } + + public boolean isEnabled() { + return enabled; + } + + public List getDefaultDateIntervals() { + return defaultDateIntervals; + } + + public List getDefaultMetrics() { + return defaultMetrics; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java new file mode 100644 index 0000000000000..40145f9f80ef3 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.Rounding; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.Settings; + +import java.util.ArrayList; +import java.util.List; + +/** + * Date dimension class + * + * @opensearch.experimental + */ +@ExperimentalApi +public class DateDimension extends Dimension { + private final List calendarIntervals; + + public DateDimension(String name, Settings settings) { + super(name); + List intervalStrings = settings.getAsList("calendar_interval"); + if (intervalStrings == null || intervalStrings.isEmpty()) { + this.calendarIntervals = new ArrayList<>(); + } else { + this.calendarIntervals = new ArrayList<>(); + for (String interval : intervalStrings) { + this.calendarIntervals.add(CompositeIndexConfig.getTimeUnit(interval)); + } + } + } + + public DateDimension(String name, List calendarIntervals) { + super(name); + this.calendarIntervals = calendarIntervals; + } + + @Override + public void setDefaults(CompositeIndexSettings compositeIndexSettings) { + if (calendarIntervals.isEmpty()) { + this.calendarIntervals.addAll(compositeIndexSettings.getDefaultDateIntervals()); + } + } + + public List getIntervals() { + return calendarIntervals; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java b/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java new file mode 100644 index 0000000000000..c06df856d5b8a --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Composite index dimension base class + * + * @opensearch.experimental + */ +@ExperimentalApi +public class Dimension { + private final String field; + + public Dimension(String field) { + this.field = field; + } + + public String getField() { + return field; + } + + public void setDefaults(CompositeIndexSettings compositeIndexSettings) { + // no implementation + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/Metric.java b/server/src/main/java/org/opensearch/index/compositeindex/Metric.java new file mode 100644 index 0000000000000..dddc3c795078c --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/Metric.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.List; + +/** + * Holds details of metrics field as part of composite field + */ +@ExperimentalApi +public class Metric { + private final String field; + private final List metrics; + + public Metric(String field, List metrics) { + this.field = field; + this.metrics = metrics; + } + + public String getField() { + return field; + } + + public List getMetrics() { + return metrics; + } + + public void setDefaults(CompositeIndexSettings compositeIndexSettings) { + if (metrics.isEmpty()) { + metrics.addAll(compositeIndexSettings.getDefaultMetrics()); + } + } + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/MetricType.java b/server/src/main/java/org/opensearch/index/compositeindex/MetricType.java new file mode 100644 index 0000000000000..9261279fa5957 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/MetricType.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Supported metric types for composite index + */ +@ExperimentalApi +public enum MetricType { + COUNT("count"), + AVG("avg"), + SUM("sum"), + MIN("min"), + MAX("max"); + + private final String typeName; + + MetricType(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static MetricType fromTypeName(String typeName) { + for (MetricType metric : MetricType.values()) { + if (metric.getTypeName().equalsIgnoreCase(typeName)) { + return metric; + } + } + throw new IllegalArgumentException("Invalid metric type: " + typeName); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java b/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java new file mode 100644 index 0000000000000..c38f8ca8813c7 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.List; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Star tree index specific settings for a composite field. + */ +@ExperimentalApi +public class StarTreeFieldSpec implements CompositeFieldSpec { + + private final AtomicInteger maxLeafDocs = new AtomicInteger(); + private final List skipStarNodeCreationInDims; + private final StarTreeBuildMode buildMode; + + public StarTreeFieldSpec(int maxLeafDocs, List skipStarNodeCreationInDims, StarTreeBuildMode buildMode) { + this.maxLeafDocs.set(maxLeafDocs); + this.skipStarNodeCreationInDims = skipStarNodeCreationInDims; + this.buildMode = buildMode; + } + + /** + * Star tree build mode using which sorting and aggregations are performed during index creation. + * + * @opensearch.experimental + */ + @ExperimentalApi + public enum StarTreeBuildMode { + ON_HEAP("onheap"), + OFF_HEAP("offheap"); + + private final String typeName; + + StarTreeBuildMode(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static StarTreeBuildMode fromTypeName(String typeName) { + for (StarTreeBuildMode starTreeBuildMode : StarTreeBuildMode.values()) { + if (starTreeBuildMode.getTypeName().equalsIgnoreCase(typeName)) { + return starTreeBuildMode; + } + } + throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid star tree build mode: [%s] ", typeName)); + } + } + + public int maxLeafDocs() { + return maxLeafDocs.get(); + } + + @Override + public void setDefaults(CompositeIndexSettings compositeIndexSettings) { + if (maxLeafDocs.get() == Integer.MAX_VALUE) { + maxLeafDocs.set(compositeIndexSettings.getMaxLeafDocs()); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/package-info.java new file mode 100644 index 0000000000000..59f18efec26b1 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Core classes for handling composite indices. + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex; diff --git a/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java b/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java new file mode 100644 index 0000000000000..be49269f0264c --- /dev/null +++ b/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.indices; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.compositeindex.CompositeIndexSettings; + +/** + * Utility to provide a {@link CompositeIndexSettings} instance containing all defaults + * + * @opensearch.experimental + */ +@ExperimentalApi +public final class DefaultCompositeIndexSettings { + private DefaultCompositeIndexSettings() {} + + public static final CompositeIndexSettings INSTANCE = new CompositeIndexSettings( + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); +} diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 251be8a990055..b1fb2f77e2981 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -106,6 +106,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalysisRegistry; import org.opensearch.index.cache.request.ShardRequestCache; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.CommitStats; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.engine.EngineConfigFactory; @@ -354,6 +355,7 @@ public class IndicesService extends AbstractLifecycleComponent private final BiFunction translogFactorySupplier; private volatile TimeValue clusterDefaultRefreshInterval; private final SearchRequestStats searchRequestStats; + private final CompositeIndexSettings compositeIndexSettings; @Override protected void doStart() { @@ -388,7 +390,8 @@ public IndicesService( @Nullable RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory, RecoverySettings recoverySettings, CacheService cacheService, - RemoteStoreSettings remoteStoreSettings + RemoteStoreSettings remoteStoreSettings, + CompositeIndexSettings compositeIndexSettings ) { this.settings = settings; this.threadPool = threadPool; @@ -495,6 +498,7 @@ protected void closeInternal() { .addSettingsUpdateConsumer(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING, this::onRefreshIntervalUpdate); this.recoverySettings = recoverySettings; this.remoteStoreSettings = remoteStoreSettings; + this.compositeIndexSettings = compositeIndexSettings; } /** @@ -903,7 +907,8 @@ private synchronized IndexService createIndexService( translogFactorySupplier, this::getClusterDefaultRefreshInterval, this.recoverySettings, - this.remoteStoreSettings + this.remoteStoreSettings, + this.compositeIndexSettings ); } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index f7a901335f34a..869f54416696d 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -147,6 +147,7 @@ import org.opensearch.index.IndexingPressureService; import org.opensearch.index.SegmentReplicationStatsTracker; import org.opensearch.index.analysis.AnalysisRegistry; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.recovery.RemoteStoreRestoreService; import org.opensearch.index.remote.RemoteIndexPathUploader; @@ -838,6 +839,8 @@ protected Node( final SearchRequestStats searchRequestStats = new SearchRequestStats(clusterService.getClusterSettings()); final SearchRequestSlowLog searchRequestSlowLog = new SearchRequestSlowLog(clusterService); + final CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings(clusterService.getClusterSettings()); + remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(clusterService, settings); CacheModule cacheModule = new CacheModule(pluginsService.filterPlugins(CachePlugin.class), settings); CacheService cacheService = cacheModule.getCacheService(); @@ -868,7 +871,8 @@ protected Node( remoteStoreStatsTrackerFactory, recoverySettings, cacheService, - remoteStoreSettings + remoteStoreSettings, + compositeIndexSettings ); final IngestService ingestService = new IngestService( diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 3d6a54055d3d5..d5269252681a7 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -1118,6 +1118,7 @@ private IndicesService mockIndicesServices(DocumentMapper documentMapper) throws when(indexService.getIndexEventListener()).thenReturn(new IndexEventListener() { }); when(indexService.getIndexSortSupplier()).thenReturn(() -> null); + when(indexService.getCompositeIndexConfigSupplier()).thenReturn(() -> null); // noinspection unchecked return ((CheckedFunction) invocationOnMock.getArguments()[1]).apply(indexService); }); diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index 4ce4936c047d9..8f45a872e752c 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -99,6 +99,7 @@ import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.RemoteBlobStoreInternalTranslogFactory; import org.opensearch.index.translog.TranslogFactory; +import org.opensearch.indices.DefaultCompositeIndexSettings; import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.indices.IndicesModule; import org.opensearch.indices.IndicesQueryCache; @@ -264,7 +265,8 @@ private IndexService newIndexService(IndexModule module) throws IOException { translogFactorySupplier, () -> IndexSettings.DEFAULT_REFRESH_INTERVAL, DefaultRecoverySettings.INSTANCE, - DefaultRemoteStoreSettings.INSTANCE + DefaultRemoteStoreSettings.INSTANCE, + DefaultCompositeIndexSettings.INSTANCE ); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java new file mode 100644 index 0000000000000..98a017bf1acbd --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java @@ -0,0 +1,233 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.Rounding; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import static org.opensearch.index.IndexSettingsTests.newIndexMeta; + +/** + * Composite index config settings unit tests + */ +public class CompositeIndexConfigSettingsTests extends OpenSearchTestCase { + private static IndexSettings indexSettings(Settings settings) { + return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); + } + + public void testDefaultSettings() { + Settings settings = Settings.EMPTY; + IndexSettings indexSettings = indexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertFalse(compositeIndexConfig.hasCompositeFields()); + } + + public void testMinimumMetrics() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .build(); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertEquals("metrics is required for composite index field [my_field]", exception.getMessage()); + } + + public void testMinimumDimensions() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "invalid") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertEquals("Atleast two dimensions are required to build composite index field [my_field]", exception.getMessage()); + } + + public void testInvalidDimensionType() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "invalid") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertEquals("Invalid dimension type in composite index config: [invalid] ", exception.getMessage()); + } + + public void testInvalidIndexMode() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .put("index.composite_index.config.my_field.index_mode", "invalid") + .build(); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertEquals("Invalid index mode in composite index config: [invalid] ", exception.getMessage()); + } + + public void testValidCompositeIndexConfig() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + IndexSettings indexSettings = indexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + } + + public void testCompositeIndexMultipleFields() { + Settings settings = Settings.builder() + .put("indices.composite_index.max_fields", 2) + .putList("index.composite_index.config.field1.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.field1.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.field1.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.field1.metrics", Arrays.asList("metric1")) + .putList("index.composite_index.config.field2.dimensions_order", Arrays.asList("dim3", "dim4")) + .put("index.composite_index.config.field2.dimensions_config.dim3.field", "dim3_field") + .put("index.composite_index.config.field2.dimensions_config.dim4.field", "dim4_field") + .putList("index.composite_index.config.field2.metrics", Arrays.asList("metric2")) + .build(); + IndexSettings indexSettings = indexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(2, compositeIndexConfig.getCompositeFields().size()); + } + + public void testCompositeIndexDateIntervalsSetting() { + Settings settings = Settings.builder() + .putList("indices.composite_index.field.default.date_intervals", Arrays.asList("day", "week")) + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "date") + .putList("index.composite_index.config.my_field.dimensions_config.dim1.calendar_interval", Arrays.asList("day", "week")) + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + IndexSettings indexSettings = indexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); + List expectedIntervals = Arrays.asList( + Rounding.DateTimeUnit.DAY_OF_MONTH, + Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR + ); + assertEquals(expectedIntervals, ((DateDimension) compositeField.getDimensionsOrder().get(0)).getIntervals()); + } + + public void testCompositeIndexMetricsSetting() { + Settings settings = Settings.builder() + .putList("indices.composite_index.field.default.metrics", Arrays.asList("count", "max")) + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .putList("index.composite_index.config.my_field.metrics_config.metric1.metrics", Arrays.asList("count", "max")) + .build(); + IndexSettings indexSettings = indexSettings(settings); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); + List expectedMetrics = Arrays.asList(MetricType.COUNT, MetricType.MAX); + assertEquals(expectedMetrics, compositeField.getMetrics().get(0).getMetrics()); + } + + public void testValidateWithoutCompositeSettingEnabled() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "default") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.LONG)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("metric2", new NumberFieldMapper.NumberFieldType("metric2", NumberFieldMapper.NumberType.LONG)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(createIndexSettings(settings)); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings(clusterSettings); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, compositeIndexSettings) + ); + assertEquals( + "composite index cannot be created, enable it using [indices.composite_index.enabled] setting", + exception.getMessage() + ); + + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + } + + public void testEnabledWithFFOff() { + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(testWithEnabledSettings(settings)); + Settings settings1 = Settings.builder().put(settings).put("indices.composite_index.enabled", true).build(); + ClusterSettings clusterSettings = new ClusterSettings(settings1, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> new CompositeIndexSettings(clusterSettings) + ); + assertEquals( + "star tree index is under an experimental feature and can be activated only by enabling opensearch.experimental.feature.composite_index.enabled feature flag in the JVM options", + exception.getMessage() + ); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); + assertTrue(compositeField.getDimensionsOrder().get(0) instanceof Dimension); + assertTrue(compositeField.getDimensionsOrder().get(1) instanceof DateDimension); + } + + private IndexSettings createIndexSettings(Settings settings) { + return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); + } + + public IndexSettings testWithEnabledSettings(Settings settings) { + Settings settings1 = Settings.builder().put(settings).put("indices.composite_index.enabled", true).build(); + IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", settings1), Settings.EMPTY); + return indexSettings; + } + +} diff --git a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java index 8f1d58cf201e9..a59e061330b91 100644 --- a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java @@ -218,6 +218,7 @@ public ClusterStateChanges(NamedXContentRegistry xContentRegistry, ThreadPool th when(indexService.getIndexEventListener()).thenReturn(new IndexEventListener() { }); when(indexService.getIndexSortSupplier()).thenReturn(() -> null); + when(indexService.getCompositeIndexConfigSupplier()).thenReturn(() -> null); // noinspection unchecked return ((CheckedFunction) invocationOnMock.getArguments()[1]).apply(indexService); }); diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 390dcf08e6ad0..358bacf657e93 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -192,6 +192,7 @@ import org.opensearch.index.shard.PrimaryReplicaSyncer; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.remote.filecache.FileCacheStats; +import org.opensearch.indices.DefaultCompositeIndexSettings; import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.indices.IndicesModule; import org.opensearch.indices.IndicesService; @@ -2078,7 +2079,8 @@ public void onFailure(final Exception e) { new RemoteStoreStatsTrackerFactory(clusterService, settings), DefaultRecoverySettings.INSTANCE, new CacheModule(new ArrayList<>(), settings).getCacheService(), - DefaultRemoteStoreSettings.INSTANCE + DefaultRemoteStoreSettings.INSTANCE, + DefaultCompositeIndexSettings.INSTANCE ); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); snapshotShardsService = new SnapshotShardsService( From e50a7b202bc0d91366cddbc826eae8de9c9555e7 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Thu, 6 Jun 2024 14:48:05 +0530 Subject: [PATCH 2/4] Adding integ settings and correcting the flow to contain only enabled setting at cluster level Signed-off-by: Bharathwaj G --- .../common/settings/ClusterSettings.java | 10 +- .../common/settings/FeatureFlagSettings.java | 2 +- .../common/settings/IndexScopedSettings.java | 12 +- .../org/opensearch/index/IndexModule.java | 5 +- .../org/opensearch/index/IndexService.java | 6 +- .../compositeindex/CompositeFieldSpec.java | 4 +- .../compositeindex/CompositeIndexConfig.java | 302 +++++++---- .../CompositeIndexSettings.java | 179 ------- .../index/compositeindex/DateDimension.java | 16 +- .../index/compositeindex/Dimension.java | 4 - .../index/compositeindex/Metric.java | 4 +- .../compositeindex/StarTreeFieldSpec.java | 7 - .../DefaultCompositeIndexSettings.java | 28 -- .../opensearch/indices/IndicesService.java | 40 +- .../main/java/org/opensearch/node/Node.java | 7 +- .../opensearch/index/IndexModuleTests.java | 3 +- .../CompositeIndexConfigSettingsTests.java | 472 ++++++++++++++++-- .../CompositeIndexIntegTests.java | 239 +++++++++ .../snapshots/SnapshotResiliencyTests.java | 4 +- 19 files changed, 949 insertions(+), 395 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java delete mode 100644 server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexIntegTests.java diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 457bda1410500..03ee2dd469d44 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -114,7 +114,6 @@ import org.opensearch.index.ShardIndexingPressureMemoryManager; import org.opensearch.index.ShardIndexingPressureSettings; import org.opensearch.index.ShardIndexingPressureStore; -import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.remote.RemoteStorePressureSettings; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; import org.opensearch.index.store.remote.filecache.FileCacheSettings; @@ -749,13 +748,8 @@ public void apply(Settings value, Settings current, Settings previous) { SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA, - // Composite index settings - CompositeIndexSettings.COMPOSITE_INDEX_ENABLED_SETTING, - CompositeIndexSettings.COMPOSITE_INDEX_MAX_FIELDS_SETTING, - CompositeIndexSettings.COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING, - CompositeIndexSettings.STAR_TREE_DEFAULT_MAX_LEAF_DOCS, - CompositeIndexSettings.DEFAULT_METRICS_LIST, - CompositeIndexSettings.DEFAULT_DATE_INTERVALS + // Composite index setting + IndicesService.COMPOSITE_INDEX_ENABLED_SETTING ) ) ); diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 1bb1c2462ffda..940dcc0f714b3 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -36,7 +36,7 @@ protected FeatureFlagSettings( FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING, FeatureFlags.TIERED_REMOTE_INDEX_SETTING, FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, - FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL_SETTING + FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL_SETTING, FeatureFlags.PLUGGABLE_CACHE_SETTING, FeatureFlags.COMPOSITE_INDEX_SETTING ); diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 38e93b22806ea..814e50b99ae24 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -51,6 +51,7 @@ import org.opensearch.index.SearchSlowLog; import org.opensearch.index.TieredMergePolicyProvider; import org.opensearch.index.cache.bitset.BitsetFilterCache; +import org.opensearch.index.compositeindex.CompositeIndexConfig; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.fielddata.IndexFieldDataService; import org.opensearch.index.mapper.FieldMapper; @@ -238,6 +239,14 @@ public final class IndexScopedSettings extends AbstractScopedSettings { // Settings for concurrent segment search IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING, IndexSettings.ALLOW_DERIVED_FIELDS, + + // Settings for composite index defaults + CompositeIndexConfig.STAR_TREE_DEFAULT_MAX_LEAF_DOCS, + CompositeIndexConfig.COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING, + CompositeIndexConfig.COMPOSITE_INDEX_MAX_FIELDS_SETTING, + CompositeIndexConfig.DEFAULT_METRICS_LIST, + CompositeIndexConfig.DEFAULT_DATE_INTERVALS, + // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { Map groups = s.getAsGroups(); @@ -250,7 +259,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { } }, Property.IndexScope), // this allows similarity settings to be passed Setting.groupSetting("index.analysis.", Property.IndexScope), // this allows analysis settings to be passed - Setting.groupSetting("index.composite_index.", Property.IndexScope) // this allows composite index settings to be passed + Setting.groupSetting("index.composite_index.config.", Property.IndexScope) // this allows composite index settings to be + // passed ) ) ); diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index aaec6bfec2123..18122745a4218 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -66,7 +66,6 @@ import org.opensearch.index.cache.query.DisabledQueryCache; import org.opensearch.index.cache.query.IndexQueryCache; import org.opensearch.index.cache.query.QueryCache; -import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; @@ -608,7 +607,7 @@ public IndexService newIndexService( Supplier clusterDefaultRefreshIntervalSupplier, RecoverySettings recoverySettings, RemoteStoreSettings remoteStoreSettings, - CompositeIndexSettings compositeIndexSettings + BooleanSupplier isCompositeIndexCreationEnabled ) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = indexReaderWrapper @@ -668,7 +667,7 @@ public IndexService newIndexService( clusterDefaultRefreshIntervalSupplier, recoverySettings, remoteStoreSettings, - compositeIndexSettings + isCompositeIndexCreationEnabled ); success = true; return indexService; diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 6f994e8f67c5b..96081388baae5 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -73,7 +73,6 @@ import org.opensearch.index.cache.bitset.BitsetFilterCache; import org.opensearch.index.cache.query.QueryCache; import org.opensearch.index.compositeindex.CompositeIndexConfig; -import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; @@ -227,7 +226,7 @@ public IndexService( Supplier clusterDefaultRefreshIntervalSupplier, RecoverySettings recoverySettings, RemoteStoreSettings remoteStoreSettings, - CompositeIndexSettings compositeIndexSettings + BooleanSupplier isCompositeIndexCreationEnabled ) { super(indexSettings); this.allowExpensiveQueries = allowExpensiveQueries; @@ -267,8 +266,9 @@ public IndexService( } if (indexSettings.getCompositeIndexConfig().hasCompositeFields()) { + // The validation is done right after the merge of the mapping later in the process ( similar to sort ) this.compositeIndexConfigSupplier = () -> indexSettings.getCompositeIndexConfig() - .validateAndGetCompositeIndexConfig(mapperService::fieldType, compositeIndexSettings); + .validateAndGetCompositeIndexConfig(mapperService::fieldType, isCompositeIndexCreationEnabled); } else { this.compositeIndexConfigSupplier = () -> null; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java index 51a2b454abcd9..829084ebf8b1a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java @@ -17,6 +17,4 @@ */ @ExperimentalApi -public interface CompositeFieldSpec { - void setDefaults(CompositeIndexSettings compositeIndexSettings); -} +public interface CompositeFieldSpec {} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java index 308c4b62d3996..b18bf40ed8d93 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java @@ -10,23 +10,26 @@ import org.opensearch.common.Rounding; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.indices.IndicesService; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.function.BooleanSupplier; import java.util.function.Function; -import static org.opensearch.index.compositeindex.CompositeIndexSettings.COMPOSITE_INDEX_ENABLED_SETTING; - /** * Configuration of composite index containing list of composite fields. * Each composite field contains dimensions, metrics along with composite index (eg: star tree) specific settings. @@ -58,11 +61,104 @@ public class CompositeIndexConfig { private static final String SKIP_STAR_NODE_CREATION_FOR_DIMS = "skip_star_node_creation_for_dimensions"; private static final String SPEC = "_spec"; private final List compositeFields = new ArrayList<>(); + private final IndexSettings indexSettings; - public CompositeIndexConfig(IndexSettings indexSettings) { + /** + * This setting determines the max number of composite fields that can be part of composite index config. For each + * composite field, we will generate associated composite index. (eg : star tree index per field ) + */ + public static final Setting COMPOSITE_INDEX_MAX_FIELDS_SETTING = Setting.intSetting( + "index.composite_index.max_fields", + 1, + 1, + 1, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * This setting determines the max number of dimensions that can be part of composite index field. Number of + * dimensions and associated cardinality has direct effect of composite index size and query performance. + */ + public static final Setting COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING = Setting.intSetting( + "index.composite_index.field.max_dimensions", + 10, + 2, + 10, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * This setting configures the default "maxLeafDocs" setting of star tree. This affects both query performance and + * star tree index size. Lesser the leaves, better the query latency but higher storage size and vice versa + *

+ * We can remove this later or change it to an enum based constant setting. + * + * @opensearch.experimental + */ + public static final Setting STAR_TREE_DEFAULT_MAX_LEAF_DOCS = Setting.intSetting( + "index.composite_index.startree.default.max_leaf_docs", + 10000, + 1, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * Default intervals for date dimension as part of composite fields + */ + public static final Setting> DEFAULT_DATE_INTERVALS = Setting.listSetting( + "index.composite_index.field.default.date_intervals", + Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), Rounding.DateTimeUnit.HOUR_OF_DAY.shortName()), + CompositeIndexConfig::getTimeUnit, + Setting.Property.IndexScope, + Setting.Property.Final + ); + public static final Setting> DEFAULT_METRICS_LIST = Setting.listSetting( + "index.composite_index.field.default.metrics", + Arrays.asList( + MetricType.AVG.toString(), + MetricType.COUNT.toString(), + MetricType.SUM.toString(), + MetricType.MAX.toString(), + MetricType.MIN.toString() + ), + MetricType::fromTypeName, + Setting.Property.IndexScope, + Setting.Property.Final + ); + private volatile int maxLeafDocs; + + private volatile List defaultDateIntervals; + private volatile List defaultMetrics; + private volatile int maxDimensions; + private volatile int maxFields; + + public CompositeIndexConfig(IndexSettings indexSettings) { + this.setMaxLeafDocs(indexSettings.getValue(STAR_TREE_DEFAULT_MAX_LEAF_DOCS)); + this.setDefaultDateIntervals(indexSettings.getValue(DEFAULT_DATE_INTERVALS)); + this.setDefaultMetrics(indexSettings.getValue(DEFAULT_METRICS_LIST)); + this.setMaxDimensions(indexSettings.getValue(COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING)); + this.setMaxFields(indexSettings.getValue(COMPOSITE_INDEX_MAX_FIELDS_SETTING)); final Map compositeIndexSettings = indexSettings.getSettings().getGroups(COMPOSITE_INDEX_CONFIG); + this.indexSettings = indexSettings; Set fields = compositeIndexSettings.keySet(); + if (!fields.isEmpty()) { + if (!FeatureFlags.isEnabled(FeatureFlags.COMPOSITE_INDEX_SETTING)) { + throw new IllegalArgumentException( + "star tree index is under an experimental feature and can be activated only by enabling " + + FeatureFlags.COMPOSITE_INDEX_SETTING.getKey() + + " feature flag in the JVM options" + ); + } + if (fields.size() > getMaxFields()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "composite index can have atmost [%s] fields", getMaxFields()) + ); + } + } for (String field : fields) { compositeFields.add(buildCompositeField(field, compositeIndexSettings.get(field))); } @@ -87,6 +183,12 @@ private CompositeField buildCompositeField(String field, Settings compositeField ); } + if (dimensionsOrder.size() > getMaxDimensions()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "composite index can have atmost [%s] dimensions", getMaxDimensions()) + ); + } + Map dimConfig = compositeFieldSettings.getGroups(DIMENSIONS_CONFIG); for (String dimension : dimConfig.keySet()) { @@ -133,9 +235,8 @@ private CompositeField buildCompositeField(String field, Settings compositeField ) ); } - dimensions.add(DimensionFactory.create(dimension, dimConfig.get(dimension))); + dimensions.add(DimensionFactory.create(dimension, dimConfig.get(dimension), this)); } - uniqueDimensions = null; Set uniqueMetricFields = new HashSet<>(); for (String metricField : metricFields) { if (!uniqueMetricFields.add(metricField)) { @@ -150,14 +251,12 @@ private CompositeField buildCompositeField(String field, Settings compositeField } Settings metricSettings = metricsConfig.get(metricField); if (metricSettings == null) { - // fill cluster level defaults in create flow as part of CompositeIndexSupplier - metrics.add(new Metric(metricField, new ArrayList<>())); + metrics.add(new Metric(metricField, getDefaultMetrics())); } else { String name = metricSettings.get(FIELD, metricField); List metricsList = metricSettings.getAsList(METRICS); if (metricsList.isEmpty()) { - // fill cluster level defaults in create flow as part of CompositeIndexSupplier - metrics.add(new Metric(name, new ArrayList<>())); + metrics.add(new Metric(name, getDefaultMetrics())); } else { List metricTypes = new ArrayList<>(); Set uniqueMetricTypes = new HashSet<>(); @@ -174,31 +273,22 @@ private CompositeField buildCompositeField(String field, Settings compositeField } metricTypes.add(MetricType.fromTypeName(metric)); } - uniqueMetricTypes = null; metrics.add(new Metric(name, metricTypes)); } } } - uniqueMetricFields = null; IndexMode indexMode = IndexMode.fromTypeName(compositeFieldSettings.get(INDEX_MODE, IndexMode.STARTREE.typeName)); Settings fieldSpec = compositeFieldSettings.getAsSettings(indexMode.typeName + SPEC); - CompositeFieldSpec compositeFieldSpec = CompositeFieldSpecFactory.create(indexMode, fieldSpec, dimensionsOrder); + CompositeFieldSpec compositeFieldSpec = CompositeFieldSpecFactory.create(indexMode, fieldSpec, dimensionsOrder, dimensions, this); return new CompositeField(field, dimensions, metrics, compositeFieldSpec); } - public static Rounding.DateTimeUnit getTimeUnit(String expression) { - if (!DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) { - throw new IllegalArgumentException("unknown calendar interval specified in composite index config"); - } - return DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression); - } - /** * Dimension factory based on field type */ private static class DimensionFactory { - static Dimension create(String dimension, Settings settings) { + static Dimension create(String dimension, Settings settings, CompositeIndexConfig compositeIndexConfig) { if (settings == null) { return new Dimension(dimension); } @@ -208,20 +298,13 @@ static Dimension create(String dimension, Settings settings) { case DEFAULT: return new Dimension(field); case DATE: - return new DateDimension(field, settings); + return new DateDimension(field, settings, compositeIndexConfig); default: throw new IllegalArgumentException( String.format(Locale.ROOT, "Invalid dimension type [%s] in composite index config", type) ); } } - - static Dimension createEmptyMappedDimension(Dimension dimension, MappedFieldType type) { - if (type instanceof DateFieldMapper.DateFieldType) { - return new DateDimension(dimension.getField(), new ArrayList<>()); - } - return dimension; - } } /** @@ -258,13 +341,23 @@ public static DimensionType fromTypeName(String typeName) { * Composite field spec factory based on index mode */ private static class CompositeFieldSpecFactory { - static CompositeFieldSpec create(IndexMode indexMode, Settings settings, List dimensions) { + static CompositeFieldSpec create( + IndexMode indexMode, + Settings settings, + List dimensionsOrder, + List dimensions, + CompositeIndexConfig compositeIndexConfig + ) { if (settings == null) { - return new StarTreeFieldSpec(10000, new ArrayList<>(), StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP); + return new StarTreeFieldSpec( + compositeIndexConfig.getMaxLeafDocs(), + new ArrayList<>(), + StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP + ); } switch (indexMode) { case STARTREE: - return buildStarTreeFieldSpec(settings, dimensions); + return buildStarTreeFieldSpec(settings, dimensionsOrder, dimensions, compositeIndexConfig); default: throw new IllegalArgumentException( String.format(Locale.ROOT, "Invalid index mode [%s] in composite index config", indexMode) @@ -273,26 +366,35 @@ static CompositeFieldSpec create(IndexMode indexMode, Settings settings, List dimensions) { + private static StarTreeFieldSpec buildStarTreeFieldSpec( + Settings settings, + List dimensionsString, + List dimensions, + CompositeIndexConfig compositeIndexConfig + ) { StarTreeFieldSpec.StarTreeBuildMode buildMode = StarTreeFieldSpec.StarTreeBuildMode.fromTypeName( settings.get(STAR_TREE_BUILD_MODE, StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP.getTypeName()) ); - // Fill default value as part of create flow as part of supplier - int maxLeafDocs = settings.getAsInt(MAX_LEAF_DOCS, Integer.MAX_VALUE); + int maxLeafDocs = settings.getAsInt(MAX_LEAF_DOCS, compositeIndexConfig.getMaxLeafDocs()); if (maxLeafDocs < 1) { throw new IllegalArgumentException( String.format(Locale.ROOT, "Invalid max_leaf_docs [%s] in composite index config", maxLeafDocs) ); } List skipStarNodeCreationInDims = settings.getAsList(SKIP_STAR_NODE_CREATION_FOR_DIMS, new ArrayList<>()); + Set skipListWithMappedFieldNames = new HashSet<>(); for (String dim : skipStarNodeCreationInDims) { - if (!dimensions.contains(dim)) { + if (!dimensionsString.contains(dim)) { throw new IllegalArgumentException( String.format(Locale.ROOT, "Invalid dimension [%s] in skip_star_node_creation_for_dims", dim) ); } + boolean duplicate = !(skipListWithMappedFieldNames.add(dimensions.get(dimensionsString.indexOf(dim)).getField())); + if (duplicate) { + throw new IllegalArgumentException(String.format("duplicate dimension [%s] found in skipStarNodeCreationInDims", dim)); + } } - return new StarTreeFieldSpec(maxLeafDocs, skipStarNodeCreationInDims, buildMode); + return new StarTreeFieldSpec(maxLeafDocs, new ArrayList<>(skipListWithMappedFieldNames), buildMode); } /** @@ -337,71 +439,37 @@ public boolean hasCompositeFields() { } /** - * Validates the composite fields based on IndexSettingDefaults and the mappedFieldType - * Updates CompositeIndexConfig with newer, completely updated composite fields + * Validates the composite fields based on defaults and based on the mappedFieldType + * Updates CompositeIndexConfig with newer, completely updated composite fields. * */ public CompositeIndexConfig validateAndGetCompositeIndexConfig( Function fieldTypeLookup, - CompositeIndexSettings compositeIndexSettings + BooleanSupplier isCompositeIndexCreationEnabled ) { if (hasCompositeFields() == false) { return null; } - if (!compositeIndexSettings.isEnabled()) { + if (!isCompositeIndexCreationEnabled.getAsBoolean()) { throw new IllegalArgumentException( String.format( Locale.ROOT, "composite index cannot be created, enable it using [%s] setting", - COMPOSITE_INDEX_ENABLED_SETTING.getKey() + IndicesService.COMPOSITE_INDEX_ENABLED_SETTING.getKey() ) ); } - if (compositeFields.size() > compositeIndexSettings.getMaxFields()) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "composite index can have atmost [%s] fields", compositeIndexSettings.getMaxFields()) - ); - } - List validatedAndMappedCompositeFields = new ArrayList<>(); for (CompositeField compositeField : compositeFields) { - if (compositeField.getDimensionsOrder().size() > compositeIndexSettings.getMaxDimensions()) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "composite index can have atmost [%s] dimensions", compositeIndexSettings.getMaxDimensions()) - ); - } - List dimensions = new ArrayList<>(); for (Dimension dimension : compositeField.getDimensionsOrder()) { - validateCompositeDimensionField(dimension.getField(), fieldTypeLookup, compositeField.getName()); - dimension = mapDimension(dimension, fieldTypeLookup.apply(dimension.getField())); - dimension.setDefaults(compositeIndexSettings); - dimensions.add(dimension); + validateDimensionField(dimension, fieldTypeLookup, compositeField.getName()); } - List metrics = new ArrayList<>(); for (Metric metric : compositeField.getMetrics()) { - validateCompositeMetricField(metric.getField(), fieldTypeLookup, compositeField.getName()); - metric.setDefaults(compositeIndexSettings); - metrics.add(metric); + validateMetricField(metric.getField(), fieldTypeLookup, compositeField.getName()); } - compositeField.getSpec().setDefaults(compositeIndexSettings); - validatedAndMappedCompositeFields.add( - new CompositeField(compositeField.getName(), dimensions, metrics, compositeField.getSpec()) - ); } - this.compositeFields.clear(); - this.compositeFields.addAll(validatedAndMappedCompositeFields); return this; } - /** - * Maps the dimension to right dimension type based on MappedFieldType - */ - private Dimension mapDimension(Dimension dimension, MappedFieldType fieldType) { - if (!isDimensionMappedToFieldType(dimension, fieldType)) { - return DimensionFactory.createEmptyMappedDimension(dimension, fieldType); - } - return dimension; - } - /** * Checks whether dimension field type is same as the source field type */ @@ -419,15 +487,27 @@ private boolean isDimensionMappedToFieldType(Dimension dimension, MappedFieldTyp * The dimension fields should be of numberField type / dateField type * */ - private void validateCompositeDimensionField( - String field, - Function fieldTypeLookup, - String compositeFieldName - ) { - final MappedFieldType ft = fieldTypeLookup.apply(field); + private void validateDimensionField(Dimension dimension, Function fieldTypeLookup, String compositeFieldName) { + final MappedFieldType ft = fieldTypeLookup.apply(dimension.getField()); if (ft == null) { throw new IllegalArgumentException( - String.format(Locale.ROOT, "unknown dimension field [%s] as part of composite field [%s]", field, compositeFieldName) + String.format( + Locale.ROOT, + "unknown dimension field [%s] as part of composite field [%s]", + dimension.getField(), + compositeFieldName + ) + ); + } + if (!isDimensionMappedToFieldType(dimension, ft)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "dimension field [%s] with field type [%s] is not same as the source " + "field type for composite field [%s]", + dimension.getField(), + ft.typeName(), + compositeFieldName + ) ); } if (!isAllowedDimensionFieldType(ft)) { @@ -436,7 +516,7 @@ private void validateCompositeDimensionField( Locale.ROOT, "composite index is not supported for the dimension field [%s] with field type [%s] as part of " + "composite field [%s]", - field, + dimension.getField(), ft.typeName(), compositeFieldName ) @@ -448,7 +528,7 @@ private void validateCompositeDimensionField( String.format( Locale.ROOT, "Aggregations not supported for the dimension field [%s] with field type [%s] as part of " + "composite field [%s]", - field, + dimension.getField(), ft.typeName(), compositeFieldName ) @@ -463,7 +543,7 @@ private void validateCompositeDimensionField( * The metric fields should be of numberField type * */ - private void validateCompositeMetricField(String field, Function fieldTypeLookup, String compositeFieldName) { + private void validateMetricField(String field, Function fieldTypeLookup, String compositeFieldName) { final MappedFieldType ft = fieldTypeLookup.apply(field); if (ft == null) { throw new IllegalArgumentException( @@ -502,4 +582,52 @@ private static boolean isAllowedDimensionFieldType(MappedFieldType fieldType) { private static boolean isAllowedMetricFieldType(MappedFieldType fieldType) { return ALLOWED_METRIC_MAPPED_FIELD_TYPES.stream().anyMatch(allowedType -> allowedType.isInstance(fieldType)); } + + public static Rounding.DateTimeUnit getTimeUnit(String expression) { + if (!DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) { + throw new IllegalArgumentException("unknown calendar interval specified in composite index config"); + } + return DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression); + } + + public void setMaxLeafDocs(int maxLeafDocs) { + this.maxLeafDocs = maxLeafDocs; + } + + public void setDefaultDateIntervals(List defaultDateIntervals) { + this.defaultDateIntervals = defaultDateIntervals; + } + + public void setDefaultMetrics(List defaultMetrics) { + this.defaultMetrics = defaultMetrics; + } + + public void setMaxDimensions(int maxDimensions) { + this.maxDimensions = maxDimensions; + } + + public void setMaxFields(int maxFields) { + this.maxFields = maxFields; + } + + public int getMaxDimensions() { + return maxDimensions; + } + + public int getMaxFields() { + return maxFields; + } + + public int getMaxLeafDocs() { + return maxLeafDocs; + } + + public List getDefaultDateIntervals() { + return defaultDateIntervals; + } + + public List getDefaultMetrics() { + return defaultMetrics; + } + } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java deleted file mode 100644 index 59bdfda5469ea..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex; - -import org.opensearch.common.Rounding; -import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Setting; -import org.opensearch.common.util.FeatureFlags; - -import java.util.Arrays; -import java.util.List; - -/** - * Cluster level settings which configures defaults for composite index - */ -@ExperimentalApi -public class CompositeIndexSettings { - /** - * This cluster level setting determines whether composite index is enabled or not - */ - public static final Setting COMPOSITE_INDEX_ENABLED_SETTING = Setting.boolSetting( - "indices.composite_index.enabled", - false, - value -> { - if (FeatureFlags.isEnabled(FeatureFlags.COMPOSITE_INDEX_SETTING) == false && value == true) { - throw new IllegalArgumentException( - "star tree index is under an experimental feature and can be activated only by enabling " - + FeatureFlags.COMPOSITE_INDEX_SETTING.getKey() - + " feature flag in the JVM options" - ); - } - }, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * This setting determines the max number of composite fields that can be part of composite index config. For each - * composite field, we will generate associated composite index. (eg : star tree index per field ) - */ - public static final Setting COMPOSITE_INDEX_MAX_FIELDS_SETTING = Setting.intSetting( - "indices.composite_index.max_fields", - 1, - 1, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * This setting determines the max number of dimensions that can be part of composite index field. Number of - * dimensions and associated cardinality has direct effect of composite index size and query performance. - */ - public static final Setting COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING = Setting.intSetting( - "indices.composite_index.field.max_dimensions", - 10, - 2, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * This setting configures the default "maxLeafDocs" setting of star tree. This affects both query performance and - * star tree index size. Lesser the leaves, better the query latency but higher storage size and vice versa - *

- * We can remove this later or change it to an enum based constant setting. - * - * @opensearch.experimental - */ - public static final Setting STAR_TREE_DEFAULT_MAX_LEAF_DOCS = Setting.intSetting( - "indices.composite_index.startree.default.max_leaf_docs", - 10000, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * Default intervals for date dimension as part of composite fields - */ - public static final Setting> DEFAULT_DATE_INTERVALS = Setting.listSetting( - "indices.composite_index.field.default.date_intervals", - Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), Rounding.DateTimeUnit.HOUR_OF_DAY.shortName()), - CompositeIndexConfig::getTimeUnit, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - public static final Setting> DEFAULT_METRICS_LIST = Setting.listSetting( - "indices.composite_index.field.default.metrics", - Arrays.asList( - MetricType.AVG.toString(), - MetricType.COUNT.toString(), - MetricType.SUM.toString(), - MetricType.MAX.toString(), - MetricType.MIN.toString() - ), - MetricType::fromTypeName, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - private volatile int maxLeafDocs; - - private volatile List defaultDateIntervals; - - private volatile List defaultMetrics; - private volatile int maxDimensions; - private volatile int maxFields; - private volatile boolean enabled; - - public CompositeIndexSettings(ClusterSettings clusterSettings) { - this.setMaxLeafDocs(clusterSettings.get(STAR_TREE_DEFAULT_MAX_LEAF_DOCS)); - this.setDefaultDateIntervals(clusterSettings.get(DEFAULT_DATE_INTERVALS)); - this.setDefaultMetrics(clusterSettings.get(DEFAULT_METRICS_LIST)); - this.setMaxDimensions(clusterSettings.get(COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING)); - this.setMaxFields(clusterSettings.get(COMPOSITE_INDEX_MAX_FIELDS_SETTING)); - this.setEnabled(clusterSettings.get(COMPOSITE_INDEX_ENABLED_SETTING)); - - clusterSettings.addSettingsUpdateConsumer(STAR_TREE_DEFAULT_MAX_LEAF_DOCS, this::setMaxLeafDocs); - clusterSettings.addSettingsUpdateConsumer(DEFAULT_DATE_INTERVALS, this::setDefaultDateIntervals); - clusterSettings.addSettingsUpdateConsumer(DEFAULT_METRICS_LIST, this::setDefaultMetrics); - clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING, this::setMaxDimensions); - clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_MAX_FIELDS_SETTING, this::setMaxFields); - clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_ENABLED_SETTING, this::setEnabled); - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public void setMaxLeafDocs(int maxLeafDocs) { - this.maxLeafDocs = maxLeafDocs; - } - - public void setDefaultDateIntervals(List defaultDateIntervals) { - this.defaultDateIntervals = defaultDateIntervals; - } - - public void setDefaultMetrics(List defaultMetrics) { - this.defaultMetrics = defaultMetrics; - } - - public void setMaxDimensions(int maxDimensions) { - this.maxDimensions = maxDimensions; - } - - public void setMaxFields(int maxFields) { - this.maxFields = maxFields; - } - - public int getMaxDimensions() { - return maxDimensions; - } - - public int getMaxFields() { - return maxFields; - } - - public int getMaxLeafDocs() { - return maxLeafDocs; - } - - public boolean isEnabled() { - return enabled; - } - - public List getDefaultDateIntervals() { - return defaultDateIntervals; - } - - public List getDefaultMetrics() { - return defaultMetrics; - } -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java index 40145f9f80ef3..3e88759ec56b9 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java @@ -24,11 +24,11 @@ public class DateDimension extends Dimension { private final List calendarIntervals; - public DateDimension(String name, Settings settings) { + public DateDimension(String name, Settings settings, CompositeIndexConfig compositeIndexConfig) { super(name); List intervalStrings = settings.getAsList("calendar_interval"); if (intervalStrings == null || intervalStrings.isEmpty()) { - this.calendarIntervals = new ArrayList<>(); + this.calendarIntervals = compositeIndexConfig.getDefaultDateIntervals(); } else { this.calendarIntervals = new ArrayList<>(); for (String interval : intervalStrings) { @@ -37,18 +37,6 @@ public DateDimension(String name, Settings settings) { } } - public DateDimension(String name, List calendarIntervals) { - super(name); - this.calendarIntervals = calendarIntervals; - } - - @Override - public void setDefaults(CompositeIndexSettings compositeIndexSettings) { - if (calendarIntervals.isEmpty()) { - this.calendarIntervals.addAll(compositeIndexSettings.getDefaultDateIntervals()); - } - } - public List getIntervals() { return calendarIntervals; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java b/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java index c06df856d5b8a..a18ffcd1df0db 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java @@ -26,8 +26,4 @@ public Dimension(String field) { public String getField() { return field; } - - public void setDefaults(CompositeIndexSettings compositeIndexSettings) { - // no implementation - } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/Metric.java b/server/src/main/java/org/opensearch/index/compositeindex/Metric.java index dddc3c795078c..9467cf1176f7a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/Metric.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/Metric.java @@ -33,9 +33,9 @@ public List getMetrics() { return metrics; } - public void setDefaults(CompositeIndexSettings compositeIndexSettings) { + public void setDefaults(CompositeIndexConfig compositeIndexConfig) { if (metrics.isEmpty()) { - metrics.addAll(compositeIndexSettings.getDefaultMetrics()); + metrics.addAll(compositeIndexConfig.getDefaultMetrics()); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java b/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java index c38f8ca8813c7..c3ac54feeb5e4 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java @@ -63,11 +63,4 @@ public static StarTreeBuildMode fromTypeName(String typeName) { public int maxLeafDocs() { return maxLeafDocs.get(); } - - @Override - public void setDefaults(CompositeIndexSettings compositeIndexSettings) { - if (maxLeafDocs.get() == Integer.MAX_VALUE) { - maxLeafDocs.set(compositeIndexSettings.getMaxLeafDocs()); - } - } } diff --git a/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java b/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java deleted file mode 100644 index be49269f0264c..0000000000000 --- a/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.indices; - -import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Settings; -import org.opensearch.index.compositeindex.CompositeIndexSettings; - -/** - * Utility to provide a {@link CompositeIndexSettings} instance containing all defaults - * - * @opensearch.experimental - */ -@ExperimentalApi -public final class DefaultCompositeIndexSettings { - private DefaultCompositeIndexSettings() {} - - public static final CompositeIndexSettings INSTANCE = new CompositeIndexSettings( - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) - ); -} diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index b1fb2f77e2981..4731ad07285c0 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -74,6 +74,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.AbstractRefCounted; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.OpenSearchExecutors; @@ -106,7 +107,6 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalysisRegistry; import org.opensearch.index.cache.request.ShardRequestCache; -import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.CommitStats; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.engine.EngineConfigFactory; @@ -307,6 +307,25 @@ public class IndicesService extends AbstractLifecycleComponent Property.Final ); + /** + * This cluster level setting determines whether composite index is enabled or not + */ + public static final Setting COMPOSITE_INDEX_ENABLED_SETTING = Setting.boolSetting( + "indices.composite_index.enabled", + false, + value -> { + if (FeatureFlags.isEnabled(FeatureFlags.COMPOSITE_INDEX_SETTING) == false && value == true) { + throw new IllegalArgumentException( + "star tree index is under an experimental feature and can be activated only by enabling " + + FeatureFlags.COMPOSITE_INDEX_SETTING.getKey() + + " feature flag in the JVM options" + ); + } + }, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + /** * The node's settings. */ @@ -355,7 +374,7 @@ public class IndicesService extends AbstractLifecycleComponent private final BiFunction translogFactorySupplier; private volatile TimeValue clusterDefaultRefreshInterval; private final SearchRequestStats searchRequestStats; - private final CompositeIndexSettings compositeIndexSettings; + private volatile boolean compositeIndexCreationEnabled; @Override protected void doStart() { @@ -390,9 +409,9 @@ public IndicesService( @Nullable RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory, RecoverySettings recoverySettings, CacheService cacheService, - RemoteStoreSettings remoteStoreSettings, - CompositeIndexSettings compositeIndexSettings + RemoteStoreSettings remoteStoreSettings ) { + // Build compositeIndexSettings (cluster level defaults) into node settings to initialize compositeIndexConfig this.settings = settings; this.threadPool = threadPool; this.pluginsService = pluginsService; @@ -443,6 +462,8 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon this.directoryFactories = directoryFactories; this.recoveryStateFactories = recoveryStateFactories; + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(COMPOSITE_INDEX_ENABLED_SETTING, this::setCompositeIndexCreationEnabled); // doClose() is called when shutting down a node, yet there might still be ongoing requests // that we need to wait for before closing some resources such as the caches. In order to // avoid closing these resources while ongoing requests are still being processed, we use a @@ -498,7 +519,6 @@ protected void closeInternal() { .addSettingsUpdateConsumer(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING, this::onRefreshIntervalUpdate); this.recoverySettings = recoverySettings; this.remoteStoreSettings = remoteStoreSettings; - this.compositeIndexSettings = compositeIndexSettings; } /** @@ -908,7 +928,7 @@ private synchronized IndexService createIndexService( this::getClusterDefaultRefreshInterval, this.recoverySettings, this.remoteStoreSettings, - this.compositeIndexSettings + this::isCompositeIndexCreationEnabled ); } @@ -1901,6 +1921,14 @@ private void setIdFieldDataEnabled(boolean value) { this.idFieldDataEnabled = value; } + private void setCompositeIndexCreationEnabled(boolean value) { + this.compositeIndexCreationEnabled = value; + } + + public boolean isCompositeIndexCreationEnabled() { + return compositeIndexCreationEnabled; + } + private void updateDanglingIndicesInfo(Index index) { assert DiscoveryNode.isDataNode(settings) : "dangling indices information should only be persisted on data nodes"; assert nodeWriteDanglingIndicesInfo : "writing dangling indices info is not enabled"; diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 869f54416696d..3afe6c3b2441e 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -147,7 +147,6 @@ import org.opensearch.index.IndexingPressureService; import org.opensearch.index.SegmentReplicationStatsTracker; import org.opensearch.index.analysis.AnalysisRegistry; -import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.recovery.RemoteStoreRestoreService; import org.opensearch.index.remote.RemoteIndexPathUploader; @@ -839,11 +838,10 @@ protected Node( final SearchRequestStats searchRequestStats = new SearchRequestStats(clusterService.getClusterSettings()); final SearchRequestSlowLog searchRequestSlowLog = new SearchRequestSlowLog(clusterService); - final CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings(clusterService.getClusterSettings()); - remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(clusterService, settings); CacheModule cacheModule = new CacheModule(pluginsService.filterPlugins(CachePlugin.class), settings); CacheService cacheService = cacheModule.getCacheService(); + final IndicesService indicesService = new IndicesService( settings, pluginsService, @@ -871,8 +869,7 @@ protected Node( remoteStoreStatsTrackerFactory, recoverySettings, cacheService, - remoteStoreSettings, - compositeIndexSettings + remoteStoreSettings ); final IngestService ingestService = new IngestService( diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index 8f45a872e752c..829e65569bc75 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -99,7 +99,6 @@ import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.RemoteBlobStoreInternalTranslogFactory; import org.opensearch.index.translog.TranslogFactory; -import org.opensearch.indices.DefaultCompositeIndexSettings; import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.indices.IndicesModule; import org.opensearch.indices.IndicesQueryCache; @@ -266,7 +265,7 @@ private IndexService newIndexService(IndexModule module) throws IOException { () -> IndexSettings.DEFAULT_REFRESH_INTERVAL, DefaultRecoverySettings.INSTANCE, DefaultRemoteStoreSettings.INSTANCE, - DefaultCompositeIndexSettings.INSTANCE + () -> false ); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java index 98a017bf1acbd..e10cce26f70c2 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java @@ -9,43 +9,51 @@ package org.opensearch.index.compositeindex; import org.opensearch.common.Rounding; -import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.index.mapper.IpFieldMapper; +import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.test.OpenSearchTestCase; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.BooleanSupplier; import java.util.function.Function; +import static org.opensearch.common.util.FeatureFlags.COMPOSITE_INDEX; import static org.opensearch.index.IndexSettingsTests.newIndexMeta; /** * Composite index config settings unit tests */ public class CompositeIndexConfigSettingsTests extends OpenSearchTestCase { - private static IndexSettings indexSettings(Settings settings) { - return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); - } public void testDefaultSettings() { Settings settings = Settings.EMPTY; - IndexSettings indexSettings = indexSettings(settings); + IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); assertFalse(compositeIndexConfig.hasCompositeFields()); } public void testMinimumMetrics() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + Settings settings = Settings.builder() .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) .build(); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + ; + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); assertEquals("metrics is required for composite index field [my_field]", exception.getMessage()); + + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testMinimumDimensions() { @@ -55,8 +63,11 @@ public void testMinimumDimensions() { .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "invalid") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) .build(); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); assertEquals("Atleast two dimensions are required to build composite index field [my_field]", exception.getMessage()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testInvalidDimensionType() { @@ -66,8 +77,11 @@ public void testInvalidDimensionType() { .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "invalid") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) .build(); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); assertEquals("Invalid dimension type in composite index config: [invalid] ", exception.getMessage()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testInvalidIndexMode() { @@ -78,8 +92,31 @@ public void testInvalidIndexMode() { .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) .put("index.composite_index.config.my_field.index_mode", "invalid") .build(); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); assertEquals("Invalid index mode in composite index config: [invalid] ", exception.getMessage()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testDefaultNumberofCompositeFieldsValidation() { + Settings settings = Settings.builder() + .put("index.composite_index.max_fields", 1) + .putList("index.composite_index.config.field1.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.field1.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.field1.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.field1.metrics", Arrays.asList("metric1")) + .putList("index.composite_index.config.field2.dimensions_order", Arrays.asList("dim3", "dim4")) + .put("index.composite_index.config.field2.dimensions_config.dim3.field", "dim3_field") + .put("index.composite_index.config.field2.dimensions_config.dim4.field", "dim4_field") + .putList("index.composite_index.config.field2.metrics", Arrays.asList("metric2")) + .build(); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); + assertEquals("composite index can have atmost [1] fields", exception.getMessage()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testValidCompositeIndexConfig() { @@ -87,17 +124,25 @@ public void testValidCompositeIndexConfig() { .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .put("index.composite_index.config.my_field.index_mode", "startree") .build(); - IndexSettings indexSettings = indexSettings(settings); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IndexSettings indexSettings = getEnabledIndexSettings(settings); CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); assertTrue(compositeIndexConfig.hasCompositeFields()); assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + assertEquals("dim1_field", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0).getField()); + assertEquals("dim2_field", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1).getField()); + assertTrue(compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1) instanceof DateDimension); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testCompositeIndexMultipleFields() { Settings settings = Settings.builder() - .put("indices.composite_index.max_fields", 2) + .put("index.composite_index.max_fields", 2) .putList("index.composite_index.config.field1.dimensions_order", Arrays.asList("dim1", "dim2")) .put("index.composite_index.config.field1.dimensions_config.dim1.field", "dim1_field") .put("index.composite_index.config.field1.dimensions_config.dim2.field", "dim2_field") @@ -107,15 +152,33 @@ public void testCompositeIndexMultipleFields() { .put("index.composite_index.config.field2.dimensions_config.dim4.field", "dim4_field") .putList("index.composite_index.config.field2.metrics", Arrays.asList("metric2")) .build(); - IndexSettings indexSettings = indexSettings(settings); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + Exception ex = expectThrows(IllegalArgumentException.class, () -> getEnabledAndMultiFieldIndexSettings(settings)); + assertEquals("Failed to parse value [2] for setting [index.composite_index.max_fields] must be <= 1", ex.getMessage()); + /** + * // uncomment once we add support for multiple fields + IndexSettings indexSettings = getEnabledAndMultiFieldIndexSettings(settings); CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); assertTrue(compositeIndexConfig.hasCompositeFields()); assertEquals(2, compositeIndexConfig.getCompositeFields().size()); + assertEquals(2, compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().size()); + assertEquals(2, compositeIndexConfig.getCompositeFields().get(1).getDimensionsOrder().size()); + assertEquals(1, compositeIndexConfig.getCompositeFields().get(0).getMetrics().size()); + assertEquals(1, compositeIndexConfig.getCompositeFields().get(1).getMetrics().size()); + assertEquals("dim1_field", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0).getField()); + assertEquals("dim2_field", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1).getField()); + assertEquals("dim3_field", compositeIndexConfig.getCompositeFields().get(1).getDimensionsOrder().get(0).getField()); + assertEquals("dim4_field", compositeIndexConfig.getCompositeFields().get(1).getDimensionsOrder().get(1).getField()); + assertEquals("metric1", compositeIndexConfig.getCompositeFields().get(0).getMetrics().get(0).getField()); + assertEquals("metric2", compositeIndexConfig.getCompositeFields().get(1).getMetrics().get(0).getField()); + **/ + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testCompositeIndexDateIntervalsSetting() { Settings settings = Settings.builder() - .putList("indices.composite_index.field.default.date_intervals", Arrays.asList("day", "week")) + .putList("index.composite_index.field.default.date_intervals", Arrays.asList("day", "week")) .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "date") @@ -123,7 +186,9 @@ public void testCompositeIndexDateIntervalsSetting() { .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) .build(); - IndexSettings indexSettings = indexSettings(settings); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IndexSettings indexSettings = getEnabledIndexSettings(settings); CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); assertTrue(compositeIndexConfig.hasCompositeFields()); assertEquals(1, compositeIndexConfig.getCompositeFields().size()); @@ -133,27 +198,31 @@ public void testCompositeIndexDateIntervalsSetting() { Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR ); assertEquals(expectedIntervals, ((DateDimension) compositeField.getDimensionsOrder().get(0)).getIntervals()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testCompositeIndexMetricsSetting() { Settings settings = Settings.builder() - .putList("indices.composite_index.field.default.metrics", Arrays.asList("count", "max")) + .putList("index.composite_index.field.default.metrics", Arrays.asList("count", "max")) .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) .putList("index.composite_index.config.my_field.metrics_config.metric1.metrics", Arrays.asList("count", "max")) .build(); - IndexSettings indexSettings = indexSettings(settings); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IndexSettings indexSettings = getEnabledIndexSettings(settings); CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); assertTrue(compositeIndexConfig.hasCompositeFields()); assertEquals(1, compositeIndexConfig.getCompositeFields().size()); CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); List expectedMetrics = Arrays.asList(MetricType.COUNT, MetricType.MAX); assertEquals(expectedMetrics, compositeField.getMetrics().get(0).getMetrics()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } - public void testValidateWithoutCompositeSettingEnabled() { + public void testCompositeIndexEnabledSetting() { Settings settings = Settings.builder() .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") @@ -162,29 +231,25 @@ public void testValidateWithoutCompositeSettingEnabled() { .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) .build(); - + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); Map fieldTypes = new HashMap<>(); fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.LONG)); fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); fieldTypes.put("metric2", new NumberFieldMapper.NumberFieldType("metric2", NumberFieldMapper.NumberType.LONG)); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(createIndexSettings(settings)); Function fieldTypeLookup = fieldTypes::get; - - CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(createIndexSettings(settings)); - ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings(clusterSettings); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, compositeIndexSettings) + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getDisabledSupplier()) ); assertEquals( "composite index cannot be created, enable it using [indices.composite_index.enabled] setting", exception.getMessage() ); - - assertTrue(compositeIndexConfig.hasCompositeFields()); - assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testEnabledWithFFOff() { @@ -200,34 +265,363 @@ public void testEnabledWithFFOff() { fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> new CompositeIndexConfig(getEnabledIndexSettings(settings)) + ); + assertEquals( + "star tree index is under an experimental feature and can be activated only by enabling " + + "opensearch.experimental.feature.composite_index.enabled feature flag in the JVM options", + exception.getMessage() + ); + } + + public void testUnknownDimField() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + + Map fieldTypes = new HashMap<>(); + + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals("unknown dimension field [dim1_field] as part of composite field [my_field]", exception.getMessage()); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testUnknownMetricField() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals("unknown metric field [metric2] as part of composite field [my_field]", exception.getMessage()); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testInvalidDimensionMappedType() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new IpFieldMapper.IpFieldType("dim1_field")); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + Function fieldTypeLookup = fieldTypes::get; - CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(testWithEnabledSettings(settings)); - Settings settings1 = Settings.builder().put(settings).put("indices.composite_index.enabled", true).build(); - ClusterSettings clusterSettings = new ClusterSettings(settings1, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> new CompositeIndexSettings(clusterSettings) + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) ); assertEquals( - "star tree index is under an experimental feature and can be activated only by enabling opensearch.experimental.feature.composite_index.enabled feature flag in the JVM options", + "composite index is not supported for the dimension field [dim1_field] with field type [ip] as part of composite field [my_field]", exception.getMessage() ); - assertTrue(compositeIndexConfig.hasCompositeFields()); - assertEquals(1, compositeIndexConfig.getCompositeFields().size()); - CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); - assertTrue(compositeField.getDimensionsOrder().get(0) instanceof Dimension); - assertTrue(compositeField.getDimensionsOrder().get(1) instanceof DateDimension); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testDimsWithNoDocValues() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put( + "dim1_field", + new NumberFieldMapper.NumberFieldType( + "dim1_field", + NumberFieldMapper.NumberType.LONG, + false, + false, + false, + true, + null, + Collections.emptyMap() + ) + ); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "Aggregations not supported for the dimension field [dim1_field] with field type [long] as part of composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testMetricsWithNoDocValues() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put( + "metric1", + new NumberFieldMapper.NumberFieldType( + "metric1", + NumberFieldMapper.NumberType.LONG, + false, + false, + false, + true, + null, + Collections.emptyMap() + ) + ); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "Aggregations not supported for the composite index metric field [metric1] with field type [long] as part of composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testInvalidDimensionMappedKeywordType() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new KeywordFieldMapper.KeywordFieldType("dim1_field")); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "composite index is not supported for the dimension field [dim1_field] with " + + "field type [keyword] as part of composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testInvalidMetricMappedKeywordType() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new DateFieldMapper.DateFieldType("dim1_field")); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new DateFieldMapper.DateFieldType("metric1")); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "dimension field [dim1_field] with field type [date] is not same as the source field type for composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testDefaults() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("metric2", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + CompositeIndexConfig config = compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()); + + assertTrue(config.hasCompositeFields()); + assertEquals(1, config.getCompositeFields().size()); + CompositeField compositeField = config.getCompositeFields().get(0); + List expectedMetrics = Arrays.asList(MetricType.AVG, MetricType.COUNT, MetricType.SUM, MetricType.MAX, MetricType.MIN); + assertEquals(expectedMetrics, compositeField.getMetrics().get(0).getMetrics()); + StarTreeFieldSpec spec = (StarTreeFieldSpec) compositeField.getSpec(); + assertEquals(10000, spec.maxLeafDocs()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testDimTypeValidation() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("metric2", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + Exception ex = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "dimension field [dim2_field] with field type [date] is not same as the source field type for composite field [my_field]", + ex.getMessage() + ); } private IndexSettings createIndexSettings(Settings settings) { return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); } - public IndexSettings testWithEnabledSettings(Settings settings) { - Settings settings1 = Settings.builder().put(settings).put("indices.composite_index.enabled", true).build(); - IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", settings1), Settings.EMPTY); + private Settings getEnabledSettings() { + return Settings.builder().put("index.composite_index.enabled", true).build(); + } + + public IndexSettings getEnabledIndexSettings(Settings settings) { + Settings enabledSettings = Settings.builder().put(settings).put("index.composite_index.enabled", true).build(); + IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", enabledSettings), getEnabledSettings()); return indexSettings; } + public IndexSettings getEnabledAndMultiFieldIndexSettings(Settings settings) { + Settings multiFieldEnabledSettings = Settings.builder() + .put(settings) + .put("index.composite_index.enabled", true) + .put(CompositeIndexConfig.COMPOSITE_INDEX_MAX_FIELDS_SETTING.getKey(), 2) + .build(); + IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", multiFieldEnabledSettings), multiFieldEnabledSettings); + return indexSettings; + } + + private BooleanSupplier getEnabledSupplier() { + return new BooleanSupplier() { + @Override + public boolean getAsBoolean() { + return true; + } + }; + } + + private BooleanSupplier getDisabledSupplier() { + return new BooleanSupplier() { + @Override + public boolean getAsBoolean() { + return false; + } + }; + } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexIntegTests.java b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexIntegTests.java new file mode 100644 index 0000000000000..26c7e7882e40b --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexIntegTests.java @@ -0,0 +1,239 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.opensearch.common.Rounding; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.index.Index; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.IndexService; +import org.opensearch.indices.IndicesService; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +public class CompositeIndexIntegTests extends OpenSearchIntegTestCase { + + private static final XContentBuilder TEST_MAPPING = createTestMapping(); + + private static XContentBuilder createTestMapping() { + try { + return jsonBuilder().startObject() + .startObject("properties") + .startObject("timestamp") + .field("type", "date") + .endObject() + .startObject("numeric") + .field("type", "integer") + .field("doc_values", false) + .endObject() + .startObject("numeric_dv") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .startObject("keyword_dv") + .field("type", "keyword") + .field("doc_values", true) + .endObject() + .startObject("keyword") + .field("type", "keyword") + .field("doc_values", false) + .endObject() + .endObject() + .endObject(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.COMPOSITE_INDEX, "true").build(); + } + + @Before + public final void setupNodeSettings() { + Settings request = Settings.builder().put(IndicesService.COMPOSITE_INDEX_ENABLED_SETTING.getKey(), true).build(); + assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(request).get()); + } + + @After + public final void cleanupNodeSettings() { + assertAcked( + client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().putNull("*")) + .setTransientSettings(Settings.builder().putNull("*")) + ); + } + + public void testInvalidCompositeIndex() { + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + ).setMapping(TEST_MAPPING).get() + ); + assertEquals( + "dimension field [timestamp] with field type [date] is not same as the source field type for composite field [my_field]", + ex.getMessage() + ); + + ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get() + ); + assertEquals( + "Aggregations not supported for the dimension field [numeric] with field type [integer] as part of composite field [my_field]", + ex.getMessage() + ); + + ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get() + ); + assertEquals( + "Aggregations not supported for the composite index metric field [numeric] with field type [integer] as part of composite field [my_field]", + ex.getMessage() + ); + + ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("invalid", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + ).setMapping(TEST_MAPPING).get() + ); + assertEquals("unknown dimension field [invalid] as part of composite field [my_field]", ex.getMessage()); + + ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("invalid")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get() + ); + assertEquals("unknown metric field [invalid] as part of composite field [my_field]", ex.getMessage()); + + FeatureFlagSetter.set(FeatureFlags.COMPOSITE_INDEX); + prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "2") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get(); + GetSettingsRequest getSettingsRequest = new GetSettingsRequest().indices("test"); + GetSettingsResponse indexSettings = client().admin().indices().getSettings(getSettingsRequest).actionGet(); + indexSettings.getIndexToSettings().get("test"); + final Index index = resolveIndex("test"); + Iterable dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class); + for (IndicesService service : dataNodeInstances) { + if (service.hasIndex(index)) { + IndexService indexService = service.indexService(index); + CompositeIndexConfig compositeIndexConfig = indexService.getIndexSettings().getCompositeIndexConfig(); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + assertEquals("timestamp", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0).getField()); + assertEquals("numeric_dv", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1).getField()); + assertTrue(compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0) instanceof DateDimension); + } + } + } + + public void testValidCompositeIndex() { + prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "2") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get(); + GetSettingsRequest getSettingsRequest = new GetSettingsRequest().indices("test"); + GetSettingsResponse indexSettings = client().admin().indices().getSettings(getSettingsRequest).actionGet(); + indexSettings.getIndexToSettings().get("test"); + final Index index = resolveIndex("test"); + Iterable dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class); + for (IndicesService service : dataNodeInstances) { + if (service.hasIndex(index)) { + IndexService indexService = service.indexService(index); + CompositeIndexConfig compositeIndexConfig = indexService.getIndexSettings().getCompositeIndexConfig(); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + assertEquals("timestamp", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0).getField()); + assertEquals("numeric_dv", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1).getField()); + assertTrue(compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0) instanceof DateDimension); + List expectedMetrics = Arrays.asList( + MetricType.AVG, + MetricType.COUNT, + MetricType.SUM, + MetricType.MAX, + MetricType.MIN + ); + assertEquals(expectedMetrics, compositeIndexConfig.getCompositeFields().get(0).getMetrics().get(0).getMetrics()); + List expectedIntervals = Arrays.asList( + Rounding.DateTimeUnit.MINUTES_OF_HOUR, + Rounding.DateTimeUnit.HOUR_OF_DAY + ); + assertEquals( + expectedIntervals, + ((DateDimension) compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0)).getIntervals() + ); + } + } + + } + +} diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 358bacf657e93..390dcf08e6ad0 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -192,7 +192,6 @@ import org.opensearch.index.shard.PrimaryReplicaSyncer; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.remote.filecache.FileCacheStats; -import org.opensearch.indices.DefaultCompositeIndexSettings; import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.indices.IndicesModule; import org.opensearch.indices.IndicesService; @@ -2079,8 +2078,7 @@ public void onFailure(final Exception e) { new RemoteStoreStatsTrackerFactory(clusterService, settings), DefaultRecoverySettings.INSTANCE, new CacheModule(new ArrayList<>(), settings).getCacheService(), - DefaultRemoteStoreSettings.INSTANCE, - DefaultCompositeIndexSettings.INSTANCE + DefaultRemoteStoreSettings.INSTANCE ); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); snapshotShardsService = new SnapshotShardsService( From feab2288fc1cf0f574043fd07c177ceb018bbe47 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Sun, 9 Jun 2024 20:56:28 +0530 Subject: [PATCH 3/4] Base Single Tree Builder for Star Tree Signed-off-by: Sarthak Aggarwal --- CHANGELOG.md | 1 + .../BaseCompositeFieldStarTreeBuilder.java | 747 ++++++++++++++++++ .../lucene/index/StarTreeDocValuesWriter.java | 37 + .../common/inject/util/Modules.java | 2 +- .../compositeindex/CompositeIndexConfig.java | 8 +- .../index/compositeindex/MetricType.java | 3 +- .../compositeindex/StarTreeFieldSpec.java | 19 +- .../startree/aggregators/DataType.java | 67 ++ .../aggregators/MetricTypeFieldPair.java | 115 +++ .../aggregators/SumValueAggregator.java | 63 ++ .../startree/aggregators/ValueAggregator.java | 62 ++ .../aggregators/ValueAggregatorFactory.java | 50 ++ .../startree/aggregators/package-info.java | 13 + .../builder/CompositeFieldWriter.java | 31 + .../builder/DocValuesIteratorFactory.java | 38 + .../startree/builder/StarTreeDocValues.java | 36 + .../StarTreeDocValuesIteratorFactory.java | 54 ++ .../startree/builder/package-info.java | 13 + .../startree/node/StarTree.java | 31 + .../startree/node/StarTreeNode.java | 52 ++ .../startree/node/package-info.java | 13 + .../startree/utils/StarTreeBuilderUtils.java | 131 +++ .../startree/utils/package-info.java | 13 + .../aggregators/MetricTypeFieldPairTests.java | 69 ++ .../aggregators/SumValueAggregatorTests.java | 63 ++ .../ValueAggregatorFactoryTests.java | 42 + ...StarTreeDocValuesIteratorFactoryTests.java | 113 +++ 27 files changed, 1878 insertions(+), 8 deletions(-) create mode 100644 server/src/main/java/org/apache/lucene/index/BaseCompositeFieldStarTreeBuilder.java create mode 100644 server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/DataType.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/MetricTypeFieldPair.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/SumValueAggregator.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregator.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregatorFactory.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/builder/CompositeFieldWriter.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/builder/DocValuesIteratorFactory.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/builder/StarTreeDocValues.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/builder/StarTreeDocValuesIteratorFactory.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/builder/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/node/StarTree.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/node/StarTreeNode.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/node/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/utils/StarTreeBuilderUtils.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/utils/package-info.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/MetricTypeFieldPairTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/SumValueAggregatorTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregatorFactoryTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/startree/builder/StarTreeDocValuesIteratorFactoryTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index ca83a4b296e6e..f64d1a45881ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Query Insights] Add X-Opaque-Id to search request metadata for top n queries ([#13374](https://github.com/opensearch-project/OpenSearch/pull/13374)) - Add support for query level resource usage tracking ([#13172](https://github.com/opensearch-project/OpenSearch/pull/13172)) - [Star Tree Index] Star tree index config changes ([#13917](https://github.com/opensearch-project/OpenSearch/pull/13917)) +- [Star Tree Index] Star tree Base Single Tree Builder ([#14104](https://github.com/opensearch-project/OpenSearch/pull/14104)) ### Dependencies - Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559)) diff --git a/server/src/main/java/org/apache/lucene/index/BaseCompositeFieldStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseCompositeFieldStarTreeBuilder.java new file mode 100644 index 0000000000000..4c78976c0d4a7 --- /dev/null +++ b/server/src/main/java/org/apache/lucene/index/BaseCompositeFieldStarTreeBuilder.java @@ -0,0 +1,747 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.apache.lucene.index; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.ByteBlockPool; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.Counter; +import org.opensearch.common.util.io.IOUtils; +import org.opensearch.index.compositeindex.CompositeField; +import org.opensearch.index.compositeindex.DateDimension; +import org.opensearch.index.compositeindex.Dimension; +import org.opensearch.index.compositeindex.Metric; +import org.opensearch.index.compositeindex.MetricType; +import org.opensearch.index.compositeindex.StarTreeFieldSpec; +import org.opensearch.index.compositeindex.startree.aggregators.MetricTypeFieldPair; +import org.opensearch.index.compositeindex.startree.aggregators.ValueAggregator; +import org.opensearch.index.compositeindex.startree.aggregators.ValueAggregatorFactory; +import org.opensearch.index.compositeindex.startree.builder.CompositeFieldWriter; +import org.opensearch.index.compositeindex.startree.builder.StarTreeDocValues; +import org.opensearch.index.compositeindex.startree.builder.StarTreeDocValuesIteratorFactory; +import org.opensearch.index.compositeindex.startree.node.StarTreeNode; +import org.opensearch.index.compositeindex.startree.utils.StarTreeBuilderUtils; +import org.opensearch.index.mapper.NumberFieldMapper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Base class for star tree builder + */ +public abstract class BaseCompositeFieldStarTreeBuilder implements CompositeFieldWriter { + + // TODO: STAR_TREE_CODEC will be moved to CodecService once the Star Tree Codec is defined + public static final String STAR_TREE_CODEC = "startreecodec"; + + private static final Logger logger = LogManager.getLogger(BaseCompositeFieldStarTreeBuilder.class); + + public static final int STAR_IN_DOC_VALUES_INDEX = 0; + + public final String[] dimensionsSplitOrder; + public final Set skipStarNodeCreationForDimensions; + public final String[] metrics; + + public final int numMetrics; + public final int numDimensions; + public int numDocs; + public int totalDocs; + public int numNodes; + public final int maxLeafDocuments; + + public final StarTreeBuilderUtils.TreeNode rootNode = getNewNode(); + + public IndexOutput indexOutput; + public DocIdSetIterator[] dimensionReaders; + public DocIdSetIterator[] metricReaders; + + public ValueAggregator[] valueAggregators; + public DocValuesConsumer docValuesConsumer; + public DocValuesProducer docValuesProducer; + + private final StarTreeDocValuesIteratorFactory starTreeDocValuesIteratorFactory; + private final CompositeField compositeField; + private final StarTreeFieldSpec starTreeFieldSpec; + private final List metricTypeFieldPairs; + private final SegmentWriteState segmentWriteState; + + /** + * Constructor for base star tree builder + * @param compositeField holds the configuration for the star tree + * @param docValuesProducer helps return the doc values iterator for each type based on field name + * @param docValuesConsumer to consume the new aggregated metrics during flush + * @param state stores the segment state + */ + protected BaseCompositeFieldStarTreeBuilder( + CompositeField compositeField, + DocValuesProducer docValuesProducer, + DocValuesConsumer docValuesConsumer, + SegmentWriteState state + ) throws IOException { + + logger.info("Building in base star tree builder"); + + String docFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, "stttree"); + logger.info("Star tree file name : {}", docFileName); + + indexOutput = state.directory.createOutput(docFileName, state.context); + CodecUtil.writeIndexHeader(indexOutput, STAR_TREE_CODEC, 0, state.segmentInfo.getId(), state.segmentSuffix); + + starTreeDocValuesIteratorFactory = new StarTreeDocValuesIteratorFactory(); + this.compositeField = compositeField; + this.starTreeFieldSpec = (StarTreeFieldSpec) (compositeField.getSpec()); + this.segmentWriteState = state; + this.docValuesConsumer = docValuesConsumer; + this.docValuesProducer = docValuesProducer; + + List dimensionsSplitOrder = compositeField.getDimensionsOrder(); + numDimensions = dimensionsSplitOrder.size(); + this.dimensionsSplitOrder = new String[numDimensions]; + // let's see how can populate this + skipStarNodeCreationForDimensions = new HashSet<>(); + totalDocs = state.segmentInfo.maxDoc(); + dimensionReaders = new DocIdSetIterator[numDimensions]; + Set skipStarNodeCreationForDimensions = this.starTreeFieldSpec.getSkipStarNodeCreationInDims(); + + for (int i = 0; i < numDimensions; i++) { + String dimension = dimensionsSplitOrder.get(i).getField(); + this.dimensionsSplitOrder[i] = dimension; + if (skipStarNodeCreationForDimensions.contains(dimensionsSplitOrder.get(i).getField())) { + this.skipStarNodeCreationForDimensions.add(i); + } + FieldInfo dimensionFieldInfos = state.fieldInfos.fieldInfo(dimension); + DocValuesType dimensionDocValuesType = state.fieldInfos.fieldInfo(dimension).getDocValuesType(); + dimensionReaders[i] = starTreeDocValuesIteratorFactory.createIterator( + dimensionDocValuesType, + dimensionFieldInfos, + docValuesProducer + ); + } + + this.metricTypeFieldPairs = generateAggregationFunctionColumnPairs(); + numMetrics = metricTypeFieldPairs.size(); + metrics = new String[numMetrics]; + valueAggregators = new ValueAggregator[numMetrics]; + metricReaders = new DocIdSetIterator[numMetrics]; + + int index = 0; + for (MetricTypeFieldPair metricTypeFieldPair : metricTypeFieldPairs) { + metrics[index] = metricTypeFieldPair.toFieldName(); + valueAggregators[index] = ValueAggregatorFactory.getValueAggregator(metricTypeFieldPair.getFunctionType()); + // Ignore the column for COUNT aggregation function + if (valueAggregators[index].getAggregationType() != MetricType.COUNT) { + String metricName = metricTypeFieldPair.getField(); + FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metricName); + DocValuesType metricDocValuesType = state.fieldInfos.fieldInfo(metricName).getDocValuesType(); + metricReaders[index] = starTreeDocValuesIteratorFactory.createIterator( + metricDocValuesType, + metricFieldInfos, + docValuesProducer + ); + } + index++; + } + maxLeafDocuments = starTreeFieldSpec.maxLeafDocs(); + } + + /** + * Generates the AggregationFunctionColumnPairs for all the metrics on a field + */ + private List generateAggregationFunctionColumnPairs() { + List metricTypeFieldPairs = new ArrayList<>(); + for (Metric metric : this.compositeField.getMetrics()) { + for (MetricType metricType : metric.getMetrics()) { + MetricTypeFieldPair metricTypeFieldPair = new MetricTypeFieldPair(metricType, metric.getField()); + metricTypeFieldPairs.add(metricTypeFieldPair); + } + } + return metricTypeFieldPairs; + } + + /** + * Appends a starTreeDocument to the star-tree. + * + * @param starTreeDocument StarTreeDocument to be appended + */ + public abstract void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException; + + /** + * Returns the starTreeDocument of the given document Id in the star-tree. + * + * @param docId Document Id + * @return Star-tree Document + */ + public abstract StarTreeDocument getStarTreeDocument(int docId) throws IOException; + + /** + * Returns the dimension value of the given document and dimension Id in the star-tree. + * + * @param docId Document Id + * @param dimensionId Dimension Id + * @return Dimension value + */ + public abstract long getDimensionValue(int docId, int dimensionId) throws IOException; + + /** + * Sorts and aggregates the starTreeDocument in the segment, and returns a starTreeDocument iterator for all the + * aggregated starTreeDocument. + * + *

This method reads starTreeDocument from segment and generates the initial starTreeDocument for the star-tree. + * + * @param numDocs Number of documents in the segment + * @return Iterator for the aggregated starTreeDocument + */ + public abstract Iterator sortAndAggregateSegmentStarTreeDocument(int numDocs) throws IOException; + + /** + * Generates aggregated starTreeDocument for star-node. + * + *

This method will do the following steps: + * + *

    + *
  • Creates a temporary buffer for the given range of documents + *
  • Replaces the value for the given dimension Id to {@code STAR} + *
  • Sorts the starTreeDocument inside the temporary buffer + *
  • Aggregates the starTreeDocument with same dimensions + *
  • Returns an iterator for the aggregated starTreeDocument + *
+ * + * @param startDocId Start document Id in the star-tree + * @param endDocId End document Id (exclusive) in the star-tree + * @param dimensionId Dimension Id of the star-node + * @return Iterator for the aggregated starTreeDocument + */ + public abstract Iterator generateStarTreeForStarNode(int startDocId, int endDocId, int dimensionId) + throws IOException; + + /** + * Returns the next segment starTreeDocument for the dimensions + */ + long[] getNextSegmentStarTreeDocumentDimensions() throws IOException { + long[] dimensions = new long[numDimensions]; + for (int i = 0; i < numDimensions; i++) { + try { + dimensionReaders[i].nextDoc(); + } catch (IOException e) { + logger.error("unable to iterate to next doc", e); + } + + if (compositeField.getDimensionsOrder().get(i) instanceof DateDimension) { + dimensions[i] = handleDateDimension( + dimensionsSplitOrder[i], + starTreeDocValuesIteratorFactory.getNextValue(dimensionReaders[i]) + ); + } else { + dimensions[i] = starTreeDocValuesIteratorFactory.getNextValue(dimensionReaders[i]); + } + } + return dimensions; + } + + /** + * Returns the next segment starTreeDocument + */ + protected StarTreeDocument getNextSegmentStarTreeDocument() throws IOException { + long[] dimensions = getNextSegmentStarTreeDocumentDimensions(); + + Object[] metrics = new Object[numMetrics]; + for (int i = 0; i < numMetrics; i++) { + // Ignore the column for COUNT aggregation function + if (metricReaders[i] != null) { + try { + metricReaders[i].nextDoc(); + } catch (IOException e) { + // TODO : handle null values in columns + logger.error("unable to iterate to next doc", e); + } + metrics[i] = starTreeDocValuesIteratorFactory.getNextValue(metricReaders[i]); + } + } + return new StarTreeDocument(dimensions, metrics); + } + + /** + * Merges a segment starTreeDocument (raw) into the aggregated starTreeDocument. + * + *

Will create a new aggregated starTreeDocument if the current one is {@code null}. + * + * @param aggregatedStarTreeDocument Aggregated starTreeDocument + * @param segmentStarTreeDocument Segment starTreeDocument + * @return Merged starTreeDocument + */ + protected StarTreeDocument mergeSegmentStarTreeDocument( + StarTreeDocument aggregatedStarTreeDocument, + StarTreeDocument segmentStarTreeDocument + ) { + // TODO: HANDLE KEYWORDS LATER! + if (aggregatedStarTreeDocument == null) { + long[] dimensions = Arrays.copyOf(segmentStarTreeDocument._dimensions, numDimensions); + Object[] metrics = new Object[numMetrics]; + for (int i = 0; i < numMetrics; i++) { + metrics[i] = valueAggregators[i].getInitialAggregatedValue(segmentStarTreeDocument._metrics[i]); + } + return new StarTreeDocument(dimensions, metrics); + } else { + for (int i = 0; i < numMetrics; i++) { + aggregatedStarTreeDocument._metrics[i] = valueAggregators[i].applyRawValue( + aggregatedStarTreeDocument._metrics[i], + segmentStarTreeDocument._metrics[i] + ); + } + return aggregatedStarTreeDocument; + } + } + + /** + * Merges a star-tree starTreeDocument (aggregated) into the aggregated starTreeDocument. + * + *

Will create a new aggregated starTreeDocument if the current one is {@code null}. + * + * @param aggregatedStarTreeDocument Aggregated starTreeDocument + * @param starTreeStarTreeDocument Star-tree Document + * @return Merged starTreeDocument + */ + protected StarTreeDocument mergeStarTreeDocument( + StarTreeDocument aggregatedStarTreeDocument, + StarTreeDocument starTreeStarTreeDocument + ) { + if (aggregatedStarTreeDocument == null) { + long[] dimensions = Arrays.copyOf(starTreeStarTreeDocument._dimensions, numDimensions); + Object[] metrics = new Object[numMetrics]; + for (int i = 0; i < numMetrics; i++) { + metrics[i] = valueAggregators[i].cloneAggregatedValue((Long) starTreeStarTreeDocument._metrics[i]); + } + return new StarTreeDocument(dimensions, metrics); + } else { + for (int i = 0; i < numMetrics; i++) { + aggregatedStarTreeDocument._metrics[i] = valueAggregators[i].applyAggregatedValue( + (Long) starTreeStarTreeDocument._metrics[i], + (Long) aggregatedStarTreeDocument._metrics[i] + ); + } + return aggregatedStarTreeDocument; + } + } + + public abstract void build(List starTreeDocValues) throws IOException; + + public void build() throws IOException { + long startTime = System.currentTimeMillis(); + logger.info("Tree of Aggregations build is a go with config {}", compositeField); + + Iterator starTreeDocumentIterator = sortAndAggregateSegmentStarTreeDocument(totalDocs); + logger.info("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); + build(starTreeDocumentIterator); + logger.info("Finished Building TOA in ms : {}", (System.currentTimeMillis() - startTime)); + } + + /** + * Builds the star tree using Star-Tree Document + */ + public void build(Iterator starTreeDocumentIterator) throws IOException { + int numSegmentStarTreeDocument = totalDocs; + + while (starTreeDocumentIterator.hasNext()) { + appendToStarTree(starTreeDocumentIterator.next()); + } + int numStarTreeDocument = numDocs; + logger.info("Generated star tree docs : [{}] from segment docs : [{}]", numStarTreeDocument, numSegmentStarTreeDocument); + + if (numDocs == 0) { + StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); + return; + } + + constructStarTree(rootNode, 0, numDocs); + int numStarTreeDocumentUnderStarNode = numDocs - numStarTreeDocument; + logger.info( + "Finished constructing star-tree, got [ {} ] tree nodes and [ {} ] starTreeDocument under star-node", + numNodes, + numStarTreeDocumentUnderStarNode + ); + + createAggregatedDocs(rootNode); + int numAggregatedStarTreeDocument = numDocs - numStarTreeDocument - numStarTreeDocumentUnderStarNode; + logger.info("Finished creating aggregated documents : {}", numAggregatedStarTreeDocument); + + // Create doc values indices in disk + createSortedDocValuesIndices(docValuesConsumer); + + // Serialize and save in disk + StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); + + // TODO: Write star tree metadata for off heap implementation + + } + + /** + * Appends a starTreeDocument to star tree + */ + private void appendToStarTree(StarTreeDocument starTreeDocument) throws IOException { + appendStarTreeDocument(starTreeDocument); + numDocs++; + } + + /** + * Returns a new node + */ + private StarTreeBuilderUtils.TreeNode getNewNode() { + numNodes++; + return new StarTreeBuilderUtils.TreeNode(); + } + + /** + * Implements the algorithm to construct a star tree + */ + private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocId, int endDocId) throws IOException { + + int childDimensionId = node.dimensionId + 1; + if (childDimensionId == numDimensions) { + return; + } + + // Construct all non-star children nodes + node.childDimensionId = childDimensionId; + Map children = constructNonStarNodes(startDocId, endDocId, childDimensionId); + node.children = children; + + // Construct star-node if required + if (!skipStarNodeCreationForDimensions.contains(childDimensionId) && children.size() > 1) { + children.put(StarTreeNode.ALL, constructStarNode(startDocId, endDocId, childDimensionId)); + } + + // Further split on child nodes if required + for (StarTreeBuilderUtils.TreeNode child : children.values()) { + if (child.endDocId - child.startDocId > maxLeafDocuments) { + constructStarTree(child, child.startDocId, child.endDocId); + } + } + } + + /** + * Constructs non star tree nodes + */ + private Map constructNonStarNodes(int startDocId, int endDocId, int dimensionId) + throws IOException { + Map nodes = new HashMap<>(); + int nodeStartDocId = startDocId; + long nodeDimensionValue = getDimensionValue(startDocId, dimensionId); + for (int i = startDocId + 1; i < endDocId; i++) { + long dimensionValue = getDimensionValue(i, dimensionId); + if (dimensionValue != nodeDimensionValue) { + StarTreeBuilderUtils.TreeNode child = getNewNode(); + child.dimensionId = dimensionId; + child.dimensionValue = nodeDimensionValue; + child.startDocId = nodeStartDocId; + child.endDocId = i; + nodes.put(nodeDimensionValue, child); + + nodeStartDocId = i; + nodeDimensionValue = dimensionValue; + } + } + StarTreeBuilderUtils.TreeNode lastNode = getNewNode(); + lastNode.dimensionId = dimensionId; + lastNode.dimensionValue = nodeDimensionValue; + lastNode.startDocId = nodeStartDocId; + lastNode.endDocId = endDocId; + nodes.put(nodeDimensionValue, lastNode); + return nodes; + } + + /** + * Constructs star tree nodes + */ + private StarTreeBuilderUtils.TreeNode constructStarNode(int startDocId, int endDocId, int dimensionId) throws IOException { + StarTreeBuilderUtils.TreeNode starNode = getNewNode(); + starNode.dimensionId = dimensionId; + starNode.dimensionValue = StarTreeNode.ALL; + starNode.startDocId = numDocs; + Iterator starTreeDocumentIterator = generateStarTreeForStarNode(startDocId, endDocId, dimensionId); + while (starTreeDocumentIterator.hasNext()) { + appendToStarTree(starTreeDocumentIterator.next()); + } + starNode.endDocId = numDocs; + return starNode; + } + + /** + * Returns aggregated starTreeDocument + */ + private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node) throws IOException { + StarTreeDocument aggregatedStarTreeDocument = null; + if (node.children == null) { + // For leaf node + + if (node.startDocId == node.endDocId - 1) { + // If it has only one document, use it as the aggregated document + aggregatedStarTreeDocument = getStarTreeDocument(node.startDocId); + node.aggregatedDocId = node.startDocId; + } else { + // If it has multiple documents, aggregate all of them + for (int i = node.startDocId; i < node.endDocId; i++) { + aggregatedStarTreeDocument = mergeStarTreeDocument(aggregatedStarTreeDocument, getStarTreeDocument(i)); + } + assert aggregatedStarTreeDocument != null; + for (int i = node.dimensionId + 1; i < numDimensions; i++) { + aggregatedStarTreeDocument._dimensions[i] = STAR_IN_DOC_VALUES_INDEX; + } + node.aggregatedDocId = numDocs; + appendToStarTree(aggregatedStarTreeDocument); + } + } else { + // For non-leaf node + if (node.children.containsKey(StarTreeNode.ALL)) { + // If it has star child, use the star child aggregated document directly + for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { + if (child.dimensionValue == StarTreeNode.ALL) { + aggregatedStarTreeDocument = createAggregatedDocs(child); + node.aggregatedDocId = child.aggregatedDocId; + } else { + createAggregatedDocs(child); + } + } + } else { + // If no star child exists, aggregate all aggregated documents from non-star children + for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { + aggregatedStarTreeDocument = mergeStarTreeDocument(aggregatedStarTreeDocument, createAggregatedDocs(child)); + } + assert aggregatedStarTreeDocument != null; + for (int i = node.dimensionId + 1; i < numDimensions; i++) { + aggregatedStarTreeDocument._dimensions[i] = STAR_IN_DOC_VALUES_INDEX; + } + node.aggregatedDocId = numDocs; + appendToStarTree(aggregatedStarTreeDocument); + } + } + return aggregatedStarTreeDocument; + } + + /** + * Create sorted doc values indices + */ + private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer) throws IOException { + List dimensionWriters = new ArrayList<>(); + List metricWriters = new ArrayList<>(); + FieldInfo[] dimensionFieldInfoList = new FieldInfo[dimensionReaders.length]; + FieldInfo[] metricFieldInfoList = new FieldInfo[metricReaders.length]; + + // star tree index field number + int fieldNum = 0; + for (int i = 0; i < dimensionReaders.length; i++) { + FieldInfo originalDimensionFieldInfo = segmentWriteState.fieldInfos.fieldInfo(dimensionsSplitOrder[i]); + final FieldInfo fi = new FieldInfo( + dimensionsSplitOrder[i] + "_dim", + fieldNum, + false, + originalDimensionFieldInfo.omitsNorms(), + originalDimensionFieldInfo.hasPayloads(), + originalDimensionFieldInfo.getIndexOptions(), + originalDimensionFieldInfo.getDocValuesType(), + -1, + originalDimensionFieldInfo.attributes(), + originalDimensionFieldInfo.getPointDimensionCount(), + originalDimensionFieldInfo.getPointIndexDimensionCount(), + originalDimensionFieldInfo.getPointNumBytes(), + originalDimensionFieldInfo.getVectorDimension(), + originalDimensionFieldInfo.getVectorEncoding(), + originalDimensionFieldInfo.getVectorSimilarityFunction(), + false, + originalDimensionFieldInfo.isParentField() + ); + dimensionFieldInfoList[i] = fi; + StarTreeDocValuesWriter starTreeDimensionDocValuesWriter = new StarTreeDocValuesWriter( + originalDimensionFieldInfo.getDocValuesType(), + getDocValuesWriter(originalDimensionFieldInfo.getDocValuesType(), fi, Counter.newCounter()) + ); + dimensionWriters.add(starTreeDimensionDocValuesWriter); + fieldNum++; + } + for (int i = 0; i < metricReaders.length; i++) { + FieldInfo originalMetricFieldInfo = segmentWriteState.fieldInfos.fieldInfo(metrics[i]); + FieldInfo fi = new FieldInfo( + metrics[i] + "_metric", + fieldNum, + false, + originalMetricFieldInfo.omitsNorms(), + originalMetricFieldInfo.hasPayloads(), + originalMetricFieldInfo.getIndexOptions(), + originalMetricFieldInfo.getDocValuesType(), + -1, + originalMetricFieldInfo.attributes(), + originalMetricFieldInfo.getPointDimensionCount(), + originalMetricFieldInfo.getPointIndexDimensionCount(), + originalMetricFieldInfo.getPointNumBytes(), + originalMetricFieldInfo.getVectorDimension(), + originalMetricFieldInfo.getVectorEncoding(), + originalMetricFieldInfo.getVectorSimilarityFunction(), + false, + originalMetricFieldInfo.isParentField() + ); + metricFieldInfoList[i] = fi; + StarTreeDocValuesWriter starTreeMetricDocValuesWriter = new StarTreeDocValuesWriter( + originalMetricFieldInfo.getDocValuesType(), + getDocValuesWriter(originalMetricFieldInfo.getDocValuesType(), fi, Counter.newCounter()) + ); + metricWriters.add(starTreeMetricDocValuesWriter); + fieldNum++; + } + + Map> ordinalToSortedSetDocValueMap = new HashMap<>(); + for (int docId = 0; docId < numDocs; docId++) { + StarTreeDocument starTreeDocument = getStarTreeDocument(docId); + for (int i = 0; i < starTreeDocument._dimensions.length; i++) { + long val = starTreeDocument._dimensions[i]; + StarTreeDocValuesWriter starTreeDocValuesWriter = dimensionWriters.get(i); + switch (starTreeDocValuesWriter.getDocValuesType()) { + case SORTED_SET: + if (val == -1) continue; + ((SortedSetDocValuesWriter) starTreeDocValuesWriter.getDocValuesWriter()).addValue( + docId, + getSortedSetDocValueBytes(val, i, ordinalToSortedSetDocValueMap) + ); + break; + case SORTED_NUMERIC: + ((SortedNumericDocValuesWriter) starTreeDocValuesWriter.getDocValuesWriter()).addValue(docId, val); + break; + default: + throw new IllegalStateException("Unsupported doc values type"); + } + } + for (int i = 0; i < starTreeDocument._metrics.length; i++) { + try { + Number parse = NumberFieldMapper.NumberType.LONG.parse(starTreeDocument._metrics[i], true); + StarTreeDocValuesWriter starTreeDocValuesWriter = metricWriters.get(i); + ((SortedNumericDocValuesWriter) starTreeDocValuesWriter.getDocValuesWriter()).addValue(docId, parse.longValue()); + } catch (IllegalArgumentException e) { + logger.info("could not parse the value, exiting creation of star tree"); + } + } + } + + getStarTreeDocValueProducers(docValuesConsumer, dimensionWriters, dimensionFieldInfoList, dimensionReaders); + getStarTreeDocValueProducers(docValuesConsumer, metricWriters, metricFieldInfoList, metricReaders); + } + + private BytesRef getSortedSetDocValueBytes(long val, int counter, Map> ordinalToSortedSetDocValueMap) + throws IOException { + String dimensionName = dimensionsSplitOrder[counter]; + if (!ordinalToSortedSetDocValueMap.containsKey(dimensionName)) { + ordinalToSortedSetDocValueMap.put(dimensionName, new HashMap<>()); + } + BytesRef bytes; + if (ordinalToSortedSetDocValueMap.get(dimensionName).containsKey(val)) { + bytes = ordinalToSortedSetDocValueMap.get(dimensionName).get(val); + } else { + bytes = ((SortedSetDocValues) dimensionReaders[counter]).lookupOrd(val); + ordinalToSortedSetDocValueMap.get(dimensionName).put(val, BytesRef.deepCopyOf(bytes)); + } + return bytes; + } + + /** + * Consumes the newly created aggregated metric fields + */ + private void getStarTreeDocValueProducers( + DocValuesConsumer docValuesConsumer, + List docValuesWriters, + FieldInfo[] fieldInfoList, + DocIdSetIterator[] readers + ) throws IOException { + for (int i = 0; i < readers.length; i++) { + final int counter = i; + DocValuesProducer docValuesProducer = new EmptyDocValuesProducer() { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) { + return ((SortedNumericDocValuesWriter) docValuesWriters.get(counter).getDocValuesWriter()).getDocValues(); + } + + @Override + public SortedSetDocValues getSortedSet(FieldInfo field) { + return ((SortedSetDocValuesWriter) docValuesWriters.get(counter).getDocValuesWriter()).getDocValues(); + } + }; + docValuesConsumer.addSortedNumericField(fieldInfoList[i], docValuesProducer); + } + } + + /** + * Returns the respective doc values writer based on doc values type + */ + private DocValuesWriter getDocValuesWriter(DocValuesType docValuesType, FieldInfo fi, Counter counter) { + final ByteBlockPool.Allocator byteBlockAllocator = new ByteBlockPool.DirectTrackingAllocator(counter); + final ByteBlockPool docValuesBytePool = new ByteBlockPool(byteBlockAllocator); + switch (docValuesType) { + case SORTED_SET: + return new SortedSetDocValuesWriter(fi, counter, docValuesBytePool); + case SORTED_NUMERIC: + return new SortedNumericDocValuesWriter(fi, counter); + default: + throw new IllegalArgumentException("Unsupported DocValuesType: " + docValuesType); + } + } + + /** + * Handles the dimension of date time field type + */ + private long handleDateDimension(final String fieldName, final long val) { + // TODO: handle timestamp granularity + return val; + } + + public void close() throws IOException { + boolean success = false; + try { + if (indexOutput != null) { + indexOutput.writeInt(-1); + CodecUtil.writeFooter(indexOutput); // write checksum + } + success = true; + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + if (success) { + IOUtils.close(indexOutput); + } else { + IOUtils.closeWhileHandlingException(indexOutput); + } + indexOutput = null; + } + } + + /** + * Star tree document + */ + public static class StarTreeDocument { + public final long[] _dimensions; + public final Object[] _metrics; + + public StarTreeDocument(long[] dimensions, Object[] metrics) { + _dimensions = dimensions; + _metrics = metrics; + } + + @Override + public String toString() { + return Arrays.toString(_dimensions) + " | " + Arrays.toString(_metrics); + } + } +} diff --git a/server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java b/server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java new file mode 100644 index 0000000000000..e8d96226da2ee --- /dev/null +++ b/server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.apache.lucene.index; + +/** + * A wrapper for {@link DocValuesWriter} that contains the {@link DocValuesType} of the doc + */ +public class StarTreeDocValuesWriter { + + private final DocValuesType docValuesType; + private final DocValuesWriter docValuesWriter; + + public StarTreeDocValuesWriter(DocValuesType docValuesType, DocValuesWriter docValuesWriter) { + this.docValuesType = docValuesType; + this.docValuesWriter = docValuesWriter; + } + + /** + * Get the doc values type + */ + public DocValuesType getDocValuesType() { + return docValuesType; + } + + /** + * Get the doc values writer + */ + public DocValuesWriter getDocValuesWriter() { + return docValuesWriter; + } +} diff --git a/server/src/main/java/org/opensearch/common/inject/util/Modules.java b/server/src/main/java/org/opensearch/common/inject/util/Modules.java index ae37cb3d29a68..b5a5a83ac3af9 100644 --- a/server/src/main/java/org/opensearch/common/inject/util/Modules.java +++ b/server/src/main/java/org/opensearch/common/inject/util/Modules.java @@ -202,7 +202,7 @@ public Void visit(Binding binding) { if (!overriddenKeys.remove(binding.getKey())) { super.visit(binding); - // Record when a scope instance is used in a binding + // record when a scope instance is used in a binding Scope scope = getScopeInstanceOrNull(binding); if (scope != null) { scopeInstancesInUse.put(scope, binding.getSource()); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java index b18bf40ed8d93..5ed96cf2f6294 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java @@ -351,7 +351,7 @@ static CompositeFieldSpec create( if (settings == null) { return new StarTreeFieldSpec( compositeIndexConfig.getMaxLeafDocs(), - new ArrayList<>(), + new HashSet<>(), StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP ); } @@ -391,10 +391,12 @@ private static StarTreeFieldSpec buildStarTreeFieldSpec( } boolean duplicate = !(skipListWithMappedFieldNames.add(dimensions.get(dimensionsString.indexOf(dim)).getField())); if (duplicate) { - throw new IllegalArgumentException(String.format("duplicate dimension [%s] found in skipStarNodeCreationInDims", dim)); + throw new IllegalArgumentException( + String.format(Locale.ROOT, "duplicate dimension [%s] found in skipStarNodeCreationInDims", dim) + ); } } - return new StarTreeFieldSpec(maxLeafDocs, new ArrayList<>(skipListWithMappedFieldNames), buildMode); + return new StarTreeFieldSpec(maxLeafDocs, new HashSet<>(skipListWithMappedFieldNames), buildMode); } /** diff --git a/server/src/main/java/org/opensearch/index/compositeindex/MetricType.java b/server/src/main/java/org/opensearch/index/compositeindex/MetricType.java index 9261279fa5957..eb2492e80e294 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/MetricType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/MetricType.java @@ -19,7 +19,8 @@ public enum MetricType { AVG("avg"), SUM("sum"), MIN("min"), - MAX("max"); + MAX("max"), + UNSUPPORTED("unsupported"); private final String typeName; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java b/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java index c3ac54feeb5e4..21dcd55510d4d 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java @@ -10,8 +10,8 @@ import org.opensearch.common.annotation.ExperimentalApi; -import java.util.List; import java.util.Locale; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; /** @@ -21,10 +21,23 @@ public class StarTreeFieldSpec implements CompositeFieldSpec { private final AtomicInteger maxLeafDocs = new AtomicInteger(); - private final List skipStarNodeCreationInDims; + private final Set skipStarNodeCreationInDims; + + public AtomicInteger getMaxLeafDocs() { + return maxLeafDocs; + } + + public Set getSkipStarNodeCreationInDims() { + return skipStarNodeCreationInDims; + } + + public StarTreeBuildMode getBuildMode() { + return buildMode; + } + private final StarTreeBuildMode buildMode; - public StarTreeFieldSpec(int maxLeafDocs, List skipStarNodeCreationInDims, StarTreeBuildMode buildMode) { + public StarTreeFieldSpec(int maxLeafDocs, Set skipStarNodeCreationInDims, StarTreeBuildMode buildMode) { this.maxLeafDocs.set(maxLeafDocs); this.skipStarNodeCreationInDims = skipStarNodeCreationInDims; this.buildMode = buildMode; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/DataType.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/DataType.java new file mode 100644 index 0000000000000..5dc04d4196659 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/DataType.java @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.startree.aggregators; + +/** + * Data type of doc values + * @opensearch.internal + */ +public enum DataType { + INT(Integer.BYTES, true), + LONG(Long.BYTES, true), + FLOAT(Float.BYTES, true), + DOUBLE(Double.BYTES, true); + + private final int size; + private final boolean numeric; + + DataType(int size, boolean numeric) { + this.size = size; + this.numeric = numeric; + } + + /** + * Returns the number of bytes needed to store the data type. + */ + public int size() { + if (size >= 0) { + return size; + } + throw new IllegalStateException("Cannot get number of bytes for: " + this); + } + + /** + * Returns {@code true} if the data type is numeric (INT, LONG, FLOAT, DOUBLE, BIG_DECIMAL), + * {@code false} otherwise. + */ + public boolean isNumeric() { + return numeric; + } + + /** + * Converts the given string value to the data type. Returns byte[] for BYTES. + */ + public Object convert(String value) { + try { + switch (this) { + case INT: + return Integer.valueOf(value); + case LONG: + return Long.valueOf(value); + case FLOAT: + return Float.valueOf(value); + case DOUBLE: + return Double.valueOf(value); + default: + throw new IllegalStateException(); + } + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/MetricTypeFieldPair.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/MetricTypeFieldPair.java new file mode 100644 index 0000000000000..f5e39c0610119 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/MetricTypeFieldPair.java @@ -0,0 +1,115 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.startree.aggregators; + +import org.opensearch.index.compositeindex.MetricType; + +import java.util.Comparator; + +/** + * Builds aggregation function and doc values field pair to support various aggregations + * @opensearch.experimental + */ +public class MetricTypeFieldPair implements Comparable { + + public static final String DELIMITER = "__"; + public static final String STAR = "*"; + public static final MetricTypeFieldPair COUNT_STAR = new MetricTypeFieldPair(MetricType.COUNT, STAR); + + private final MetricType functionType; + private final String field; + + /** + * Constructor for MetricTypeFieldPair + */ + public MetricTypeFieldPair(MetricType functionType, String field) { + this.functionType = functionType; + if (functionType == MetricType.COUNT) { + this.field = STAR; + } else { + this.field = field; + } + } + + /** + * @return Metric Type + */ + public MetricType getFunctionType() { + return functionType; + } + + /** + * @return field Name + */ + public String getField() { + return field; + } + + /** + * @return field name with function type and field + */ + public String toFieldName() { + return toFieldName(functionType, field); + } + + /** + * Builds field name with function type and field + */ + public static String toFieldName(MetricType functionType, String field) { + return functionType.getTypeName() + DELIMITER + field; + } + + /** + * Builds MetricTypeFieldPair from field name + */ + public static MetricTypeFieldPair fromFieldName(String fieldName) { + String[] parts = fieldName.split(DELIMITER, 2); + return fromFunctionAndFieldName(parts[0], parts[1]); + } + + /** + * Builds MetricTypeFieldPair from function and field name + */ + private static MetricTypeFieldPair fromFunctionAndFieldName(String functionName, String fieldName) { + MetricType functionType = MetricType.fromTypeName(functionName); + if (functionType == MetricType.COUNT) { + return COUNT_STAR; + } else { + return new MetricTypeFieldPair(functionType, fieldName); + } + } + + @Override + public int hashCode() { + return 31 * functionType.hashCode() + field.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj instanceof MetricTypeFieldPair) { + MetricTypeFieldPair anotherPair = (MetricTypeFieldPair) obj; + return functionType == anotherPair.functionType && field.equals(anotherPair.field); + } + return false; + } + + @Override + public String toString() { + return toFieldName(); + } + + @Override + public int compareTo(MetricTypeFieldPair other) { + return Comparator.comparing((MetricTypeFieldPair o) -> o.field) + .thenComparing((MetricTypeFieldPair o) -> o.functionType) + .compare(this, other); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/SumValueAggregator.java new file mode 100644 index 0000000000000..c3bf11f1775fa --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/SumValueAggregator.java @@ -0,0 +1,63 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.startree.aggregators; + +import org.opensearch.index.compositeindex.MetricType; + +/** + * Sum value aggregator for star tree + * @opensearch.internal + */ +public class SumValueAggregator implements ValueAggregator { + public static final DataType AGGREGATED_VALUE_TYPE = DataType.DOUBLE; + + @Override + public MetricType getAggregationType() { + return MetricType.SUM; + } + + @Override + public DataType getAggregatedValueType() { + return AGGREGATED_VALUE_TYPE; + } + + @Override + public Double getInitialAggregatedValue(Number rawValue) { + return rawValue.doubleValue(); + } + + @Override + public Double applyRawValue(Double value, Number rawValue) { + return value + rawValue.doubleValue(); + } + + @Override + public Double applyAggregatedValue(Double value, Double aggregatedValue) { + return value + aggregatedValue; + } + + @Override + public Double cloneAggregatedValue(Double value) { + return value; + } + + @Override + public int getMaxAggregatedValueByteSize() { + return Double.BYTES; + } + + @Override + public byte[] serializeAggregatedValue(Double value) { + throw new UnsupportedOperationException(); + } + + @Override + public Double deserializeAggregatedValue(byte[] bytes) { + throw new UnsupportedOperationException(); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregator.java new file mode 100644 index 0000000000000..cc3410f3e3e7e --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregator.java @@ -0,0 +1,62 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.startree.aggregators; + +import org.opensearch.index.compositeindex.MetricType; + +/** + * A value aggregator that pre-aggregates on the input values for a specific type of aggregation. + * @opensearch.experimental + */ +public interface ValueAggregator { + + /** + * Returns the type of the aggregation. + */ + MetricType getAggregationType(); + + /** + * Returns the data type of the aggregated value. + */ + DataType getAggregatedValueType(); + + /** + * Returns the initial aggregated value. + */ + A getInitialAggregatedValue(R rawValue); + + /** + * Applies a raw value to the current aggregated value. + */ + A applyRawValue(A value, R rawValue); + + /** + * Applies an aggregated value to the current aggregated value. + */ + A applyAggregatedValue(A value, A aggregatedValue); + + /** + * Clones an aggregated value. + */ + A cloneAggregatedValue(A value); + + /** + * Returns the maximum size in bytes of the aggregated values seen so far. + */ + int getMaxAggregatedValueByteSize(); + + /** + * Serializes an aggregated value into a byte array. + */ + byte[] serializeAggregatedValue(A value); + + /** + * De-serializes an aggregated value from a byte array. + */ + A deserializeAggregatedValue(byte[] bytes); +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregatorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregatorFactory.java new file mode 100644 index 0000000000000..20a054b30f64d --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregatorFactory.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.startree.aggregators; + +import org.opensearch.index.compositeindex.MetricType; + +/** + * Value aggregator factory for a given aggregation type + * @opensearch.experimental + */ +public class ValueAggregatorFactory { + private ValueAggregatorFactory() {} + + /** + * Returns a new instance of value aggregator for the given aggregation type. + * + * @param aggregationType Aggregation type + * @return Value aggregator + */ + public static ValueAggregator getValueAggregator(MetricType aggregationType) { + switch (aggregationType) { + // other metric types (count, min, max, avg) will be supported in the future + case SUM: + return new SumValueAggregator(); + default: + throw new IllegalStateException("Unsupported aggregation type: " + aggregationType); + } + } + + /** + * Returns the data type of the aggregated value for the given aggregation type. + * + * @param aggregationType Aggregation type + * @return Data type of the aggregated value + */ + public static DataType getAggregatedValueType(MetricType aggregationType) { + switch (aggregationType) { + // other metric types (count, min, max, avg) will be supported in the future + case SUM: + return SumValueAggregator.AGGREGATED_VALUE_TYPE; + default: + throw new IllegalStateException("Unsupported aggregation type: " + aggregationType); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/package-info.java new file mode 100644 index 0000000000000..ec6841979699f --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Aggregators for Composite Index Star Tree + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.startree.aggregators; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/CompositeFieldWriter.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/CompositeFieldWriter.java new file mode 100644 index 0000000000000..84a197fdcf1df --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/CompositeFieldWriter.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.startree.builder; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; + +/** + * A star-tree builder that builds a single star-tree. + * @opensearch.experimental + */ +public interface CompositeFieldWriter extends Closeable { + + /** + * Builds the data structure for the given composite index config. + */ + void build() throws Exception; + + /** + * Builds the star-tree during segment merges + */ + void build(List starTreeDocValues) throws IOException; + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/DocValuesIteratorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/DocValuesIteratorFactory.java new file mode 100644 index 0000000000000..12b2d1cb0cedc --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/DocValuesIteratorFactory.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.startree.builder; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * An interface to support iterators for various doc values types. + * @opensearch.experimental + */ +public interface DocValuesIteratorFactory { + + /** + * Creates an iterator for the given doc values type and field using the doc values producer + */ + DocIdSetIterator createIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException; + + /** + * Returns the next value for the given iterator + */ + long getNextValue(DocIdSetIterator iterator) throws IOException; + + /** + * Returns the doc id for the next document from the given iterator + */ + int nextDoc(DocIdSetIterator iterator) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/StarTreeDocValues.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/StarTreeDocValues.java new file mode 100644 index 0000000000000..abebd21444cc4 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/StarTreeDocValues.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.startree.builder; + +import org.apache.lucene.index.SortedNumericDocValues; +import org.opensearch.index.compositeindex.startree.node.StarTree; + +import java.util.Map; + +/** + * Star tree aggregated values holder for reader / query + * @opensearch.experimental + */ +public class StarTreeDocValues { + public StarTree starTree; + + // Based on the implementation, these NEED to be INORDER or implementation of LinkedHashMap + // We use SortedNumericDocValues because essentially everything is stored as long + public Map dimensionValues; + public Map metricValues; + + public StarTreeDocValues( + StarTree starTree, + Map dimensionValues, + Map metricValues + ) { + this.starTree = starTree; + this.dimensionValues = dimensionValues; + this.metricValues = metricValues; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/StarTreeDocValuesIteratorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/StarTreeDocValuesIteratorFactory.java new file mode 100644 index 0000000000000..1d107ffd3836d --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/StarTreeDocValuesIteratorFactory.java @@ -0,0 +1,54 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.startree.builder; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * A factory class to return respective doc values iterator based on the doc volues type. + * @opensearch.experimental + */ +public class StarTreeDocValuesIteratorFactory implements DocValuesIteratorFactory { + + @Override + public DocIdSetIterator createIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException { + switch (type) { + case SORTED_SET: + return producer.getSortedSet(field); + case SORTED_NUMERIC: + return producer.getSortedNumeric(field); + default: + throw new IllegalArgumentException("Unsupported DocValuesType: " + type); + } + } + + @Override + public long getNextValue(DocIdSetIterator iterator) throws IOException { + if (iterator instanceof SortedSetDocValues) { + return ((SortedSetDocValues) iterator).nextOrd(); + } else if (iterator instanceof SortedNumericDocValues) { + return ((SortedNumericDocValues) iterator).nextValue(); + } else { + throw new IllegalArgumentException("Unsupported Iterator: " + iterator.toString()); + } + } + + @Override + public int nextDoc(DocIdSetIterator iterator) throws IOException { + return iterator.nextDoc(); + } + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/package-info.java new file mode 100644 index 0000000000000..cc7c95eca62ed --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Builders for Composite Index Star Tree + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.startree.builder; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/node/StarTree.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/node/StarTree.java new file mode 100644 index 0000000000000..1b3a87c07c3c7 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/node/StarTree.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.startree.node; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Interface for star tree + * @opensearch.experimental + */ +public interface StarTree { + + /** Get the root node of the star tree. */ + StarTreeNode getRoot(); + + /** + * Get a list of all dimension names. The node dimension id is the index of the dimension name in + * this list. + */ + List getDimensionNames(); + + /** Prints the tree for its visual representation. */ + void printTree(Map> dictionaryMap) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/node/StarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/node/StarTreeNode.java new file mode 100644 index 0000000000000..c3ad5406a2872 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/node/StarTreeNode.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.startree.node; + +import java.io.IOException; +import java.util.Iterator; + +/** + * Class representing each node in star tree + * @opensearch.experimental + */ +public interface StarTreeNode { + long ALL = -1l; + + /** Get the index of the dimension. */ + int getDimensionId() throws IOException; + + /** Get the value (dictionary id) of the dimension. */ + long getDimensionValue() throws IOException; + + /** Get the child dimension id. */ + int getChildDimensionId() throws IOException; + + /** Get the index of the start document. */ + int getStartDocId() throws IOException; + + /** Get the index of the end document (exclusive). */ + int getEndDocId() throws IOException; + + /** Get the index of the aggregated document. */ + int getAggregatedDocId() throws IOException; + + /** Get the number of children nodes. */ + int getNumChildren() throws IOException; + + /** Return true if the node is a leaf node, false otherwise. */ + boolean isLeaf(); + + /** + * Get the child node corresponding to the given dimension value (dictionary id), or null if such + * child does not exist. + */ + StarTreeNode getChildForDimensionValue(long dimensionValue) throws IOException; + + /** Get the iterator over all children nodes. */ + Iterator getChildrenIterator() throws IOException; +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/node/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/node/package-info.java new file mode 100644 index 0000000000000..6a320a0a37428 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/node/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Node for Composite Index Star Tree + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.startree.node; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/utils/StarTreeBuilderUtils.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/utils/StarTreeBuilderUtils.java new file mode 100644 index 0000000000000..5f86492149c23 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/utils/StarTreeBuilderUtils.java @@ -0,0 +1,131 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.startree.utils; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.IndexOutput; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Util class for building star tree + * @opensearch.experimental + */ +public class StarTreeBuilderUtils { + + private static final Logger logger = LogManager.getLogger(StarTreeBuilderUtils.class); + + // TODO: To be moved to off heap star tree implementation + public static final int NUM_INT_SERIALIZABLE_FIELDS = 6; + public static final int NUM_LONG_SERIALIZABLE_FIELDS = 1; + public static final long SERIALIZABLE_SIZE_IN_BYTES = (Integer.BYTES * NUM_INT_SERIALIZABLE_FIELDS) + (Long.BYTES + * NUM_LONG_SERIALIZABLE_FIELDS); + + private StarTreeBuilderUtils() {} + + public static final int INVALID_ID = -1; + public static final long MAGIC_MARKER = 0xBADDA55B00DAD00DL; + + /** Tree node representation */ + public static class TreeNode { + public int dimensionId = INVALID_ID; + public long dimensionValue = INVALID_ID; + public int startDocId = INVALID_ID; + public int endDocId = INVALID_ID; + public int aggregatedDocId = INVALID_ID; + public int childDimensionId = INVALID_ID; + public Map children; + } + + /** Serializes the tree */ + public static void serializeTree(IndexOutput indexOutput, TreeNode rootNode, String[] dimensions, int numNodes) throws IOException { + int headerSizeInBytes = computeHeaderByteSize(dimensions); + long totalSizeInBytes = headerSizeInBytes + (long) numNodes * SERIALIZABLE_SIZE_IN_BYTES; + + logger.info("Star tree size in bytes : {}", totalSizeInBytes); + + writeHeader(indexOutput, headerSizeInBytes, dimensions, numNodes); + writeNodes(indexOutput, rootNode); + } + + /** Computes the size of the header for the tree */ + static int computeHeaderByteSize(String[] dimensions) { + // Magic marker (8), version (4), size of header (4) and number of dimensions (4) + int headerSizeInBytes = 20; + + for (String dimension : dimensions) { + headerSizeInBytes += Integer.BYTES; // For dimension index + headerSizeInBytes += Integer.BYTES; // For length of dimension name + headerSizeInBytes += dimension.getBytes(UTF_8).length; // For dimension name + } + + headerSizeInBytes += Integer.BYTES; // For number of nodes. + return headerSizeInBytes; + } + + /** Writes the header of the tree */ + static void writeHeader(IndexOutput output, int headerSizeInBytes, String[] dimensions, int numNodes) throws IOException { + output.writeLong(MAGIC_MARKER); + output.writeInt(1); + output.writeInt(headerSizeInBytes); + output.writeInt(dimensions.length); + for (int i = 0; i < dimensions.length; i++) { + output.writeInt(i); + output.writeString(dimensions[i]); + } + output.writeInt(numNodes); + } + + /** Writes the nodes of the tree */ + static void writeNodes(IndexOutput output, TreeNode rootNode) throws IOException { + Queue queue = new LinkedList<>(); + queue.add(rootNode); + + int currentNodeId = 0; + while (!queue.isEmpty()) { + TreeNode node = queue.remove(); + + if (node.children == null) { + writeNode(output, node, INVALID_ID, INVALID_ID); + } else { + // Sort all children nodes based on dimension value + List sortedChildren = new ArrayList<>(node.children.values()); + sortedChildren.sort(Comparator.comparingLong(o -> o.dimensionValue)); + + int firstChildId = currentNodeId + queue.size() + 1; + int lastChildId = firstChildId + sortedChildren.size() - 1; + writeNode(output, node, firstChildId, lastChildId); + + queue.addAll(sortedChildren); + } + + currentNodeId++; + } + } + + /** Writes a node of the tree */ + private static void writeNode(IndexOutput output, TreeNode node, int firstChildId, int lastChildId) throws IOException { + output.writeInt(node.dimensionId); + output.writeLong(node.dimensionValue); + output.writeInt(node.startDocId); + output.writeInt(node.endDocId); + output.writeInt(node.aggregatedDocId); + output.writeInt(firstChildId); + output.writeInt(lastChildId); + } + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/utils/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/utils/package-info.java new file mode 100644 index 0000000000000..42b1dfa9ee518 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/utils/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Utility to support Composite Index Star Tree + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.startree.utils; diff --git a/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/MetricTypeFieldPairTests.java b/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/MetricTypeFieldPairTests.java new file mode 100644 index 0000000000000..0827d86162754 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/MetricTypeFieldPairTests.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.startree.aggregators; + +import org.opensearch.index.compositeindex.MetricType; +import org.opensearch.test.OpenSearchTestCase; + +public class MetricTypeFieldPairTests extends OpenSearchTestCase { + + public void testConstructor() { + MetricTypeFieldPair pair = new MetricTypeFieldPair(MetricType.SUM, "column1"); + assertEquals(MetricType.SUM, pair.getFunctionType()); + assertEquals("column1", pair.getField()); + } + + public void testCountStarConstructor() { + MetricTypeFieldPair pair = new MetricTypeFieldPair(MetricType.COUNT, "anything"); + assertEquals(MetricType.COUNT, pair.getFunctionType()); + assertEquals("*", pair.getField()); + } + + public void testToFieldName() { + MetricTypeFieldPair pair = new MetricTypeFieldPair(MetricType.AVG, "column2"); + assertEquals("avg__column2", pair.toFieldName()); + } + + public void testFromFieldName() { + MetricTypeFieldPair pair = MetricTypeFieldPair.fromFieldName("max__column3"); + assertEquals(MetricType.MAX, pair.getFunctionType()); + assertEquals("column3", pair.getField()); + } + + public void testCountStarFromFieldName() { + MetricTypeFieldPair pair = MetricTypeFieldPair.fromFieldName("count__*"); + assertEquals(MetricType.COUNT, pair.getFunctionType()); + assertEquals("*", pair.getField()); + assertSame(MetricTypeFieldPair.COUNT_STAR, pair); + } + + public void testEquals() { + MetricTypeFieldPair pair1 = new MetricTypeFieldPair(MetricType.SUM, "column1"); + MetricTypeFieldPair pair2 = new MetricTypeFieldPair(MetricType.SUM, "column1"); + assertEquals(pair1, pair2); + assertNotEquals(pair1, new MetricTypeFieldPair(MetricType.AVG, "column1")); + assertNotEquals(pair1, new MetricTypeFieldPair(MetricType.SUM, "column2")); + } + + public void testHashCode() { + MetricTypeFieldPair pair1 = new MetricTypeFieldPair(MetricType.SUM, "column1"); + MetricTypeFieldPair pair2 = new MetricTypeFieldPair(MetricType.SUM, "column1"); + assertEquals(pair1.hashCode(), pair2.hashCode()); + } + + public void testCompareTo() { + MetricTypeFieldPair pair1 = new MetricTypeFieldPair(MetricType.SUM, "column1"); + MetricTypeFieldPair pair2 = new MetricTypeFieldPair(MetricType.SUM, "column2"); + MetricTypeFieldPair pair3 = new MetricTypeFieldPair(MetricType.AVG, "column1"); + assertTrue(pair1.compareTo(pair2) < 0); + assertTrue(pair2.compareTo(pair1) > 0); + assertTrue(pair1.compareTo(pair3) > 0); + assertTrue(pair3.compareTo(pair1) < 0); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/SumValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/SumValueAggregatorTests.java new file mode 100644 index 0000000000000..ad9dd91112ec9 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/SumValueAggregatorTests.java @@ -0,0 +1,63 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.startree.aggregators; + +import org.opensearch.index.compositeindex.MetricType; +import org.opensearch.test.OpenSearchTestCase; + +public class SumValueAggregatorTests extends OpenSearchTestCase { + + public void testGetAggregationType() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals(MetricType.SUM.getTypeName(), aggregator.getAggregationType().getTypeName()); + } + + public void testGetAggregatedValueType() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals(SumValueAggregator.AGGREGATED_VALUE_TYPE, aggregator.getAggregatedValueType()); + } + + public void testGetInitialAggregatedValue() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals(1.0, aggregator.getInitialAggregatedValue(1), 0.0); + assertEquals(3.14, aggregator.getInitialAggregatedValue(3.14), 0.0); + } + + public void testApplyRawValue() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals(5.0, aggregator.applyRawValue(2.0, 3), 0.0); + assertEquals(7.28, aggregator.applyRawValue(3.14, 4.14), 0.0000001); + } + + public void testApplyAggregatedValue() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals(5.0, aggregator.applyAggregatedValue(2.0, 3.0), 0.0); + assertEquals(7.28, aggregator.applyAggregatedValue(3.14, 4.14), 0.0000001); + } + + public void testCloneAggregatedValue() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals(3.14, aggregator.cloneAggregatedValue(3.14), 0.0); + } + + public void testGetMaxAggregatedValueByteSize() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals(Double.BYTES, aggregator.getMaxAggregatedValueByteSize()); + } + + public void testSerializeAggregatedValue() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertThrows(UnsupportedOperationException.class, () -> aggregator.serializeAggregatedValue(3.14)); + } + + public void testDeserializeAggregatedValue() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertThrows(UnsupportedOperationException.class, () -> aggregator.deserializeAggregatedValue(new byte[0])); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregatorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregatorFactoryTests.java new file mode 100644 index 0000000000000..60129e0a7dc96 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregatorFactoryTests.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.startree.aggregators; + +import org.opensearch.index.compositeindex.MetricType; +import org.opensearch.test.OpenSearchTestCase; + +public class ValueAggregatorFactoryTests extends OpenSearchTestCase { + + public void testGetValueAggregatorForSumType() { + ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricType.SUM); + assertNotNull(aggregator); + assertEquals(SumValueAggregator.class, aggregator.getClass()); + } + + public void testGetValueAggregatorForUnsupportedType() { + IllegalStateException exception = expectThrows( + IllegalStateException.class, + () -> ValueAggregatorFactory.getValueAggregator(MetricType.UNSUPPORTED) + ); + assertEquals("Unsupported aggregation type: UNSUPPORTED", exception.getMessage()); + } + + public void testGetAggregatedValueTypeForSumType() { + DataType dataType = ValueAggregatorFactory.getAggregatedValueType(MetricType.SUM); + assertEquals(SumValueAggregator.AGGREGATED_VALUE_TYPE, dataType); + } + + public void testGetAggregatedValueTypeForUnsupportedType() { + IllegalStateException exception = expectThrows( + IllegalStateException.class, + () -> ValueAggregatorFactory.getAggregatedValueType(MetricType.UNSUPPORTED) + ); + assertEquals("Unsupported aggregation type: UNSUPPORTED", exception.getMessage()); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/startree/builder/StarTreeDocValuesIteratorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/startree/builder/StarTreeDocValuesIteratorFactoryTests.java new file mode 100644 index 0000000000000..1354fc996a47d --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/startree/builder/StarTreeDocValuesIteratorFactoryTests.java @@ -0,0 +1,113 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.startree.builder; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.Collections; + +import org.mockito.Mockito; + +import static org.mockito.Mockito.when; + +public class StarTreeDocValuesIteratorFactoryTests extends OpenSearchTestCase { + + private static StarTreeDocValuesIteratorFactory factory; + private static FieldInfo mockFieldInfo; + + @BeforeClass + public static void setup() { + factory = new StarTreeDocValuesIteratorFactory(); + mockFieldInfo = new FieldInfo( + "field", + 1, + false, + false, + true, + IndexOptions.NONE, + DocValuesType.NONE, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + } + + public void testCreateIterator_SortedSet() throws IOException { + DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); + SortedSetDocValues iterator = Mockito.mock(SortedSetDocValues.class); + when(producer.getSortedSet(mockFieldInfo)).thenReturn(iterator); + DocIdSetIterator result = factory.createIterator(DocValuesType.SORTED_SET, mockFieldInfo, producer); + assertEquals(iterator.getClass(), result.getClass()); + } + + public void testCreateIterator_SortedNumeric() throws IOException { + DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); + SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); + when(producer.getSortedNumeric(mockFieldInfo)).thenReturn(iterator); + DocIdSetIterator result = factory.createIterator(DocValuesType.SORTED_NUMERIC, mockFieldInfo, producer); + assertEquals(iterator.getClass(), result.getClass()); + } + + public void testCreateIterator_UnsupportedType() { + DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { + factory.createIterator(DocValuesType.BINARY, mockFieldInfo, producer); + }); + assertEquals("Unsupported DocValuesType: BINARY", exception.getMessage()); + } + + public void testGetNextValue_SortedSet() throws IOException { + SortedSetDocValues iterator = Mockito.mock(SortedSetDocValues.class); + when(iterator.nextOrd()).thenReturn(42L); + + long result = factory.getNextValue(iterator); + assertEquals(42L, result); + } + + public void testGetNextValue_SortedNumeric() throws IOException { + SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); + when(iterator.nextValue()).thenReturn(123L); + + long result = factory.getNextValue(iterator); + assertEquals(123L, result); + } + + public void testGetNextValue_UnsupportedIterator() { + DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { factory.getNextValue(iterator); }); + assertEquals("Unsupported Iterator: " + iterator.toString(), exception.getMessage()); + } + + public void testNextDoc() throws IOException { + DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); + when(iterator.nextDoc()).thenReturn(5); + + int result = factory.nextDoc(iterator); + assertEquals(5, result); + } +} From ff50df03478cc70f8665504e6059dbf0c61e5a1e Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Wed, 12 Jun 2024 09:52:04 +0530 Subject: [PATCH 4/4] Refactoring Signed-off-by: Sarthak Aggarwal --- ...er.java => BaseSingleStarTreeBuilder.java} | 153 ++++++------- .../aggregators/MetricTypeFieldPair.java | 40 ++-- .../aggregators/SumValueAggregator.java | 1 + .../startree/aggregators/ValueAggregator.java | 1 + .../aggregators/ValueAggregatorFactory.java | 1 + ...ieldWriter.java => SingleTreeBuilder.java} | 4 +- .../{aggregators => data}/DataType.java | 2 +- .../{builder => data}/StarTreeDocValues.java | 2 +- .../startree/data/StarTreeDocument.java | 29 +++ .../startree/node/StarTreeNode.java | 1 + .../startree/utils/StarTreeBuilderUtils.java | 16 +- .../aggregators/MetricTypeFieldPairTests.java | 8 +- .../ValueAggregatorFactoryTests.java | 1 + .../BaseSingleStarTreeBuilderTests.java | 212 ++++++++++++++++++ 14 files changed, 349 insertions(+), 122 deletions(-) rename server/src/main/java/org/apache/lucene/index/{BaseCompositeFieldStarTreeBuilder.java => BaseSingleStarTreeBuilder.java} (87%) rename server/src/main/java/org/opensearch/index/compositeindex/startree/builder/{CompositeFieldWriter.java => SingleTreeBuilder.java} (84%) rename server/src/main/java/org/opensearch/index/compositeindex/startree/{aggregators => data}/DataType.java (96%) rename server/src/main/java/org/opensearch/index/compositeindex/startree/{builder => data}/StarTreeDocValues.java (94%) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/startree/data/StarTreeDocument.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/startree/builder/BaseSingleStarTreeBuilderTests.java diff --git a/server/src/main/java/org/apache/lucene/index/BaseCompositeFieldStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java similarity index 87% rename from server/src/main/java/org/apache/lucene/index/BaseCompositeFieldStarTreeBuilder.java rename to server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java index 4c78976c0d4a7..f0feec9f1bd98 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseCompositeFieldStarTreeBuilder.java +++ b/server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java @@ -9,15 +9,12 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; -import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.compositeindex.CompositeField; import org.opensearch.index.compositeindex.DateDimension; import org.opensearch.index.compositeindex.Dimension; @@ -27,9 +24,10 @@ import org.opensearch.index.compositeindex.startree.aggregators.MetricTypeFieldPair; import org.opensearch.index.compositeindex.startree.aggregators.ValueAggregator; import org.opensearch.index.compositeindex.startree.aggregators.ValueAggregatorFactory; -import org.opensearch.index.compositeindex.startree.builder.CompositeFieldWriter; -import org.opensearch.index.compositeindex.startree.builder.StarTreeDocValues; +import org.opensearch.index.compositeindex.startree.builder.SingleTreeBuilder; import org.opensearch.index.compositeindex.startree.builder.StarTreeDocValuesIteratorFactory; +import org.opensearch.index.compositeindex.startree.data.StarTreeDocValues; +import org.opensearch.index.compositeindex.startree.data.StarTreeDocument; import org.opensearch.index.compositeindex.startree.node.StarTreeNode; import org.opensearch.index.compositeindex.startree.utils.StarTreeBuilderUtils; import org.opensearch.index.mapper.NumberFieldMapper; @@ -47,35 +45,36 @@ /** * Base class for star tree builder */ -public abstract class BaseCompositeFieldStarTreeBuilder implements CompositeFieldWriter { +public abstract class BaseSingleStarTreeBuilder implements SingleTreeBuilder { // TODO: STAR_TREE_CODEC will be moved to CodecService once the Star Tree Codec is defined public static final String STAR_TREE_CODEC = "startreecodec"; - private static final Logger logger = LogManager.getLogger(BaseCompositeFieldStarTreeBuilder.class); + private static final Logger logger = LogManager.getLogger(BaseSingleStarTreeBuilder.class); public static final int STAR_IN_DOC_VALUES_INDEX = 0; - public final String[] dimensionsSplitOrder; - public final Set skipStarNodeCreationForDimensions; - public final String[] metrics; + protected final String[] dimensionsSplitOrder; + protected final Set skipStarNodeCreationForDimensions; + protected final String[] metrics; - public final int numMetrics; - public final int numDimensions; - public int numDocs; - public int totalDocs; - public int numNodes; - public final int maxLeafDocuments; + protected final int numMetrics; + protected final int numDimensions; + protected int numDocs; + protected int totalDocs; + protected int numNodes; + protected final int maxLeafDocuments; - public final StarTreeBuilderUtils.TreeNode rootNode = getNewNode(); + protected final StarTreeBuilderUtils.TreeNode rootNode = getNewNode(); - public IndexOutput indexOutput; - public DocIdSetIterator[] dimensionReaders; - public DocIdSetIterator[] metricReaders; + // TODO: This will be initialized with OnHeap / OffHeap Implementations (Commented it's occurrences for now) + // private IndexOutput indexOutput; + protected DocIdSetIterator[] dimensionReaders; + protected DocIdSetIterator[] metricReaders; - public ValueAggregator[] valueAggregators; - public DocValuesConsumer docValuesConsumer; - public DocValuesProducer docValuesProducer; + protected ValueAggregator[] valueAggregators; + protected DocValuesConsumer docValuesConsumer; + protected DocValuesProducer docValuesProducer; private final StarTreeDocValuesIteratorFactory starTreeDocValuesIteratorFactory; private final CompositeField compositeField; @@ -90,7 +89,7 @@ public abstract class BaseCompositeFieldStarTreeBuilder implements CompositeFiel * @param docValuesConsumer to consume the new aggregated metrics during flush * @param state stores the segment state */ - protected BaseCompositeFieldStarTreeBuilder( + protected BaseSingleStarTreeBuilder( CompositeField compositeField, DocValuesProducer docValuesProducer, DocValuesConsumer docValuesConsumer, @@ -99,11 +98,11 @@ protected BaseCompositeFieldStarTreeBuilder( logger.info("Building in base star tree builder"); - String docFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, "stttree"); - logger.info("Star tree file name : {}", docFileName); + // String docFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, "stttree"); + // logger.info("Star tree file name : {}", docFileName); - indexOutput = state.directory.createOutput(docFileName, state.context); - CodecUtil.writeIndexHeader(indexOutput, STAR_TREE_CODEC, 0, state.segmentInfo.getId(), state.segmentSuffix); + // indexOutput = state.directory.createOutput(docFileName, state.context); + // CodecUtil.writeIndexHeader(indexOutput, STAR_TREE_CODEC, 0, state.segmentInfo.getId(), state.segmentSuffix); starTreeDocValuesIteratorFactory = new StarTreeDocValuesIteratorFactory(); this.compositeField = compositeField; @@ -145,7 +144,7 @@ protected BaseCompositeFieldStarTreeBuilder( int index = 0; for (MetricTypeFieldPair metricTypeFieldPair : metricTypeFieldPairs) { metrics[index] = metricTypeFieldPair.toFieldName(); - valueAggregators[index] = ValueAggregatorFactory.getValueAggregator(metricTypeFieldPair.getFunctionType()); + valueAggregators[index] = ValueAggregatorFactory.getValueAggregator(metricTypeFieldPair.getMetricType()); // Ignore the column for COUNT aggregation function if (valueAggregators[index].getAggregationType() != MetricType.COUNT) { String metricName = metricTypeFieldPair.getField(); @@ -165,7 +164,7 @@ protected BaseCompositeFieldStarTreeBuilder( /** * Generates the AggregationFunctionColumnPairs for all the metrics on a field */ - private List generateAggregationFunctionColumnPairs() { + public List generateAggregationFunctionColumnPairs() { List metricTypeFieldPairs = new ArrayList<>(); for (Metric metric : this.compositeField.getMetrics()) { for (MetricType metricType : metric.getMetrics()) { @@ -293,17 +292,17 @@ protected StarTreeDocument mergeSegmentStarTreeDocument( ) { // TODO: HANDLE KEYWORDS LATER! if (aggregatedStarTreeDocument == null) { - long[] dimensions = Arrays.copyOf(segmentStarTreeDocument._dimensions, numDimensions); + long[] dimensions = Arrays.copyOf(segmentStarTreeDocument.dimensions, numDimensions); Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { - metrics[i] = valueAggregators[i].getInitialAggregatedValue(segmentStarTreeDocument._metrics[i]); + metrics[i] = valueAggregators[i].getInitialAggregatedValue(segmentStarTreeDocument.metrics[i]); } return new StarTreeDocument(dimensions, metrics); } else { for (int i = 0; i < numMetrics; i++) { - aggregatedStarTreeDocument._metrics[i] = valueAggregators[i].applyRawValue( - aggregatedStarTreeDocument._metrics[i], - segmentStarTreeDocument._metrics[i] + aggregatedStarTreeDocument.metrics[i] = valueAggregators[i].applyRawValue( + aggregatedStarTreeDocument.metrics[i], + segmentStarTreeDocument.metrics[i] ); } return aggregatedStarTreeDocument; @@ -319,22 +318,19 @@ protected StarTreeDocument mergeSegmentStarTreeDocument( * @param starTreeStarTreeDocument Star-tree Document * @return Merged starTreeDocument */ - protected StarTreeDocument mergeStarTreeDocument( - StarTreeDocument aggregatedStarTreeDocument, - StarTreeDocument starTreeStarTreeDocument - ) { + public StarTreeDocument mergeStarTreeDocument(StarTreeDocument aggregatedStarTreeDocument, StarTreeDocument starTreeStarTreeDocument) { if (aggregatedStarTreeDocument == null) { - long[] dimensions = Arrays.copyOf(starTreeStarTreeDocument._dimensions, numDimensions); + long[] dimensions = Arrays.copyOf(starTreeStarTreeDocument.dimensions, numDimensions); Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { - metrics[i] = valueAggregators[i].cloneAggregatedValue((Long) starTreeStarTreeDocument._metrics[i]); + metrics[i] = valueAggregators[i].cloneAggregatedValue(starTreeStarTreeDocument.metrics[i]); } return new StarTreeDocument(dimensions, metrics); } else { for (int i = 0; i < numMetrics; i++) { - aggregatedStarTreeDocument._metrics[i] = valueAggregators[i].applyAggregatedValue( - (Long) starTreeStarTreeDocument._metrics[i], - (Long) aggregatedStarTreeDocument._metrics[i] + aggregatedStarTreeDocument.metrics[i] = valueAggregators[i].applyAggregatedValue( + starTreeStarTreeDocument.metrics[i], + aggregatedStarTreeDocument.metrics[i] ); } return aggregatedStarTreeDocument; @@ -366,7 +362,7 @@ public void build(Iterator starTreeDocumentIterator) throws IO logger.info("Generated star tree docs : [{}] from segment docs : [{}]", numStarTreeDocument, numSegmentStarTreeDocument); if (numDocs == 0) { - StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); + // StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); return; } @@ -386,7 +382,7 @@ public void build(Iterator starTreeDocumentIterator) throws IO createSortedDocValuesIndices(docValuesConsumer); // Serialize and save in disk - StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); + // StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); // TODO: Write star tree metadata for off heap implementation @@ -502,7 +498,7 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node } assert aggregatedStarTreeDocument != null; for (int i = node.dimensionId + 1; i < numDimensions; i++) { - aggregatedStarTreeDocument._dimensions[i] = STAR_IN_DOC_VALUES_INDEX; + aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; } node.aggregatedDocId = numDocs; appendToStarTree(aggregatedStarTreeDocument); @@ -526,7 +522,7 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node } assert aggregatedStarTreeDocument != null; for (int i = node.dimensionId + 1; i < numDimensions; i++) { - aggregatedStarTreeDocument._dimensions[i] = STAR_IN_DOC_VALUES_INDEX; + aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; } node.aggregatedDocId = numDocs; appendToStarTree(aggregatedStarTreeDocument); @@ -608,8 +604,8 @@ private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer) t Map> ordinalToSortedSetDocValueMap = new HashMap<>(); for (int docId = 0; docId < numDocs; docId++) { StarTreeDocument starTreeDocument = getStarTreeDocument(docId); - for (int i = 0; i < starTreeDocument._dimensions.length; i++) { - long val = starTreeDocument._dimensions[i]; + for (int i = 0; i < starTreeDocument.dimensions.length; i++) { + long val = starTreeDocument.dimensions[i]; StarTreeDocValuesWriter starTreeDocValuesWriter = dimensionWriters.get(i); switch (starTreeDocValuesWriter.getDocValuesType()) { case SORTED_SET: @@ -626,9 +622,9 @@ private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer) t throw new IllegalStateException("Unsupported doc values type"); } } - for (int i = 0; i < starTreeDocument._metrics.length; i++) { + for (int i = 0; i < starTreeDocument.metrics.length; i++) { try { - Number parse = NumberFieldMapper.NumberType.LONG.parse(starTreeDocument._metrics[i], true); + Number parse = NumberFieldMapper.NumberType.LONG.parse(starTreeDocument.metrics[i], true); StarTreeDocValuesWriter starTreeDocValuesWriter = metricWriters.get(i); ((SortedNumericDocValuesWriter) starTreeDocValuesWriter.getDocValuesWriter()).addValue(docId, parse.longValue()); } catch (IllegalArgumentException e) { @@ -686,7 +682,7 @@ public SortedSetDocValues getSortedSet(FieldInfo field) { /** * Returns the respective doc values writer based on doc values type */ - private DocValuesWriter getDocValuesWriter(DocValuesType docValuesType, FieldInfo fi, Counter counter) { + DocValuesWriter getDocValuesWriter(DocValuesType docValuesType, FieldInfo fi, Counter counter) { final ByteBlockPool.Allocator byteBlockAllocator = new ByteBlockPool.DirectTrackingAllocator(counter); final ByteBlockPool docValuesBytePool = new ByteBlockPool(byteBlockAllocator); switch (docValuesType) { @@ -708,40 +704,23 @@ private long handleDateDimension(final String fieldName, final long val) { } public void close() throws IOException { - boolean success = false; - try { - if (indexOutput != null) { - indexOutput.writeInt(-1); - CodecUtil.writeFooter(indexOutput); // write checksum - } - success = true; - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - if (success) { - IOUtils.close(indexOutput); - } else { - IOUtils.closeWhileHandlingException(indexOutput); - } - indexOutput = null; - } + // boolean success = false; + // try { + // if (indexOutput != null) { + // indexOutput.writeInt(-1); + // CodecUtil.writeFooter(indexOutput); // write checksum + // } + // success = true; + // } catch (Exception e) { + // throw new RuntimeException(e); + // } finally { + // if (success) { + // IOUtils.close(indexOutput); + // } else { + // IOUtils.closeWhileHandlingException(indexOutput); + // } + // indexOutput = null; + // } } - /** - * Star tree document - */ - public static class StarTreeDocument { - public final long[] _dimensions; - public final Object[] _metrics; - - public StarTreeDocument(long[] dimensions, Object[] metrics) { - _dimensions = dimensions; - _metrics = metrics; - } - - @Override - public String toString() { - return Arrays.toString(_dimensions) + " | " + Arrays.toString(_metrics); - } - } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/MetricTypeFieldPair.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/MetricTypeFieldPair.java index f5e39c0610119..ca7caaa5535ee 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/MetricTypeFieldPair.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/MetricTypeFieldPair.java @@ -21,15 +21,15 @@ public class MetricTypeFieldPair implements Comparable { public static final String STAR = "*"; public static final MetricTypeFieldPair COUNT_STAR = new MetricTypeFieldPair(MetricType.COUNT, STAR); - private final MetricType functionType; + private final MetricType metricType; private final String field; /** * Constructor for MetricTypeFieldPair */ - public MetricTypeFieldPair(MetricType functionType, String field) { - this.functionType = functionType; - if (functionType == MetricType.COUNT) { + public MetricTypeFieldPair(MetricType metricType, String field) { + this.metricType = metricType; + if (metricType == MetricType.COUNT) { this.field = STAR; } else { this.field = field; @@ -39,8 +39,8 @@ public MetricTypeFieldPair(MetricType functionType, String field) { /** * @return Metric Type */ - public MetricType getFunctionType() { - return functionType; + public MetricType getMetricType() { + return metricType; } /** @@ -51,17 +51,17 @@ public String getField() { } /** - * @return field name with function type and field + * @return field name with metric type and field */ public String toFieldName() { - return toFieldName(functionType, field); + return toFieldName(metricType, field); } /** - * Builds field name with function type and field + * Builds field name with metric type and field */ - public static String toFieldName(MetricType functionType, String field) { - return functionType.getTypeName() + DELIMITER + field; + public static String toFieldName(MetricType metricType, String field) { + return metricType.getTypeName() + DELIMITER + field; } /** @@ -69,24 +69,24 @@ public static String toFieldName(MetricType functionType, String field) { */ public static MetricTypeFieldPair fromFieldName(String fieldName) { String[] parts = fieldName.split(DELIMITER, 2); - return fromFunctionAndFieldName(parts[0], parts[1]); + return fromMetricAndFieldName(parts[0], parts[1]); } /** - * Builds MetricTypeFieldPair from function and field name + * Builds MetricTypeFieldPair from metric and field name */ - private static MetricTypeFieldPair fromFunctionAndFieldName(String functionName, String fieldName) { - MetricType functionType = MetricType.fromTypeName(functionName); - if (functionType == MetricType.COUNT) { + private static MetricTypeFieldPair fromMetricAndFieldName(String metricName, String fieldName) { + MetricType metricType = MetricType.fromTypeName(metricName); + if (metricType == MetricType.COUNT) { return COUNT_STAR; } else { - return new MetricTypeFieldPair(functionType, fieldName); + return new MetricTypeFieldPair(metricType, fieldName); } } @Override public int hashCode() { - return 31 * functionType.hashCode() + field.hashCode(); + return 31 * metricType.hashCode() + field.hashCode(); } @Override @@ -96,7 +96,7 @@ public boolean equals(Object obj) { } if (obj instanceof MetricTypeFieldPair) { MetricTypeFieldPair anotherPair = (MetricTypeFieldPair) obj; - return functionType == anotherPair.functionType && field.equals(anotherPair.field); + return metricType == anotherPair.metricType && field.equals(anotherPair.field); } return false; } @@ -109,7 +109,7 @@ public String toString() { @Override public int compareTo(MetricTypeFieldPair other) { return Comparator.comparing((MetricTypeFieldPair o) -> o.field) - .thenComparing((MetricTypeFieldPair o) -> o.functionType) + .thenComparing((MetricTypeFieldPair o) -> o.metricType) .compare(this, other); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/SumValueAggregator.java index c3bf11f1775fa..9fbfdd26df57c 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/SumValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/SumValueAggregator.java @@ -8,6 +8,7 @@ package org.opensearch.index.compositeindex.startree.aggregators; import org.opensearch.index.compositeindex.MetricType; +import org.opensearch.index.compositeindex.startree.data.DataType; /** * Sum value aggregator for star tree diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregator.java index cc3410f3e3e7e..bfbbc05254d5e 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregator.java @@ -8,6 +8,7 @@ package org.opensearch.index.compositeindex.startree.aggregators; import org.opensearch.index.compositeindex.MetricType; +import org.opensearch.index.compositeindex.startree.data.DataType; /** * A value aggregator that pre-aggregates on the input values for a specific type of aggregation. diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregatorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregatorFactory.java index 20a054b30f64d..bb1df58ca6b2b 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregatorFactory.java @@ -8,6 +8,7 @@ package org.opensearch.index.compositeindex.startree.aggregators; import org.opensearch.index.compositeindex.MetricType; +import org.opensearch.index.compositeindex.startree.data.DataType; /** * Value aggregator factory for a given aggregation type diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/CompositeFieldWriter.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/SingleTreeBuilder.java similarity index 84% rename from server/src/main/java/org/opensearch/index/compositeindex/startree/builder/CompositeFieldWriter.java rename to server/src/main/java/org/opensearch/index/compositeindex/startree/builder/SingleTreeBuilder.java index 84a197fdcf1df..2d485db89b474 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/CompositeFieldWriter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/SingleTreeBuilder.java @@ -8,6 +8,8 @@ package org.opensearch.index.compositeindex.startree.builder; +import org.opensearch.index.compositeindex.startree.data.StarTreeDocValues; + import java.io.Closeable; import java.io.IOException; import java.util.List; @@ -16,7 +18,7 @@ * A star-tree builder that builds a single star-tree. * @opensearch.experimental */ -public interface CompositeFieldWriter extends Closeable { +public interface SingleTreeBuilder extends Closeable { /** * Builds the data structure for the given composite index config. diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/DataType.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/data/DataType.java similarity index 96% rename from server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/DataType.java rename to server/src/main/java/org/opensearch/index/compositeindex/startree/data/DataType.java index 5dc04d4196659..c15697d722563 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/startree/aggregators/DataType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/data/DataType.java @@ -5,7 +5,7 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -package org.opensearch.index.compositeindex.startree.aggregators; +package org.opensearch.index.compositeindex.startree.data; /** * Data type of doc values diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/StarTreeDocValues.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/data/StarTreeDocValues.java similarity index 94% rename from server/src/main/java/org/opensearch/index/compositeindex/startree/builder/StarTreeDocValues.java rename to server/src/main/java/org/opensearch/index/compositeindex/startree/data/StarTreeDocValues.java index abebd21444cc4..bdedcefb97a61 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/startree/builder/StarTreeDocValues.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/data/StarTreeDocValues.java @@ -5,7 +5,7 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -package org.opensearch.index.compositeindex.startree.builder; +package org.opensearch.index.compositeindex.startree.data; import org.apache.lucene.index.SortedNumericDocValues; import org.opensearch.index.compositeindex.startree.node.StarTree; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/data/StarTreeDocument.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/data/StarTreeDocument.java new file mode 100644 index 0000000000000..44b211123c19f --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/data/StarTreeDocument.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.startree.data; + +import java.util.Arrays; + +/** + * Star tree document + */ +public class StarTreeDocument { + public final long[] dimensions; + public final Object[] metrics; + + public StarTreeDocument(long[] dimensions, Object[] metrics) { + this.dimensions = dimensions; + this.metrics = metrics; + } + + @Override + public String toString() { + return Arrays.toString(dimensions) + " | " + Arrays.toString(metrics); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/node/StarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/node/StarTreeNode.java index c3ad5406a2872..ea33f817a689b 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/startree/node/StarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/node/StarTreeNode.java @@ -12,6 +12,7 @@ /** * Class representing each node in star tree + * The interface is implemented by build-mode based nodes * @opensearch.experimental */ public interface StarTreeNode { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/startree/utils/StarTreeBuilderUtils.java b/server/src/main/java/org/opensearch/index/compositeindex/startree/utils/StarTreeBuilderUtils.java index 5f86492149c23..5c6b1dcdda149 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/startree/utils/StarTreeBuilderUtils.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/startree/utils/StarTreeBuilderUtils.java @@ -37,17 +37,17 @@ public class StarTreeBuilderUtils { private StarTreeBuilderUtils() {} - public static final int INVALID_ID = -1; + public static final int ALL = -1; public static final long MAGIC_MARKER = 0xBADDA55B00DAD00DL; /** Tree node representation */ public static class TreeNode { - public int dimensionId = INVALID_ID; - public long dimensionValue = INVALID_ID; - public int startDocId = INVALID_ID; - public int endDocId = INVALID_ID; - public int aggregatedDocId = INVALID_ID; - public int childDimensionId = INVALID_ID; + public int dimensionId = ALL; + public long dimensionValue = ALL; + public int startDocId = ALL; + public int endDocId = ALL; + public int aggregatedDocId = ALL; + public int childDimensionId = ALL; public Map children; } @@ -100,7 +100,7 @@ static void writeNodes(IndexOutput output, TreeNode rootNode) throws IOException TreeNode node = queue.remove(); if (node.children == null) { - writeNode(output, node, INVALID_ID, INVALID_ID); + writeNode(output, node, ALL, ALL); } else { // Sort all children nodes based on dimension value List sortedChildren = new ArrayList<>(node.children.values()); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/MetricTypeFieldPairTests.java b/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/MetricTypeFieldPairTests.java index 0827d86162754..1dca82708a19c 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/MetricTypeFieldPairTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/MetricTypeFieldPairTests.java @@ -15,13 +15,13 @@ public class MetricTypeFieldPairTests extends OpenSearchTestCase { public void testConstructor() { MetricTypeFieldPair pair = new MetricTypeFieldPair(MetricType.SUM, "column1"); - assertEquals(MetricType.SUM, pair.getFunctionType()); + assertEquals(MetricType.SUM, pair.getMetricType()); assertEquals("column1", pair.getField()); } public void testCountStarConstructor() { MetricTypeFieldPair pair = new MetricTypeFieldPair(MetricType.COUNT, "anything"); - assertEquals(MetricType.COUNT, pair.getFunctionType()); + assertEquals(MetricType.COUNT, pair.getMetricType()); assertEquals("*", pair.getField()); } @@ -32,13 +32,13 @@ public void testToFieldName() { public void testFromFieldName() { MetricTypeFieldPair pair = MetricTypeFieldPair.fromFieldName("max__column3"); - assertEquals(MetricType.MAX, pair.getFunctionType()); + assertEquals(MetricType.MAX, pair.getMetricType()); assertEquals("column3", pair.getField()); } public void testCountStarFromFieldName() { MetricTypeFieldPair pair = MetricTypeFieldPair.fromFieldName("count__*"); - assertEquals(MetricType.COUNT, pair.getFunctionType()); + assertEquals(MetricType.COUNT, pair.getMetricType()); assertEquals("*", pair.getField()); assertSame(MetricTypeFieldPair.COUNT_STAR, pair); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregatorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregatorFactoryTests.java index 60129e0a7dc96..3ed8d0a8858e9 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregatorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/startree/aggregators/ValueAggregatorFactoryTests.java @@ -9,6 +9,7 @@ package org.opensearch.index.compositeindex.startree.aggregators; import org.opensearch.index.compositeindex.MetricType; +import org.opensearch.index.compositeindex.startree.data.DataType; import org.opensearch.test.OpenSearchTestCase; public class ValueAggregatorFactoryTests extends OpenSearchTestCase { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/startree/builder/BaseSingleStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/startree/builder/BaseSingleStarTreeBuilderTests.java new file mode 100644 index 0000000000000..c476f429a0dd2 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/startree/builder/BaseSingleStarTreeBuilderTests.java @@ -0,0 +1,212 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.startree.builder; + +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.index.BaseSingleStarTreeBuilder; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.Version; +import org.opensearch.index.compositeindex.CompositeField; +import org.opensearch.index.compositeindex.Dimension; +import org.opensearch.index.compositeindex.Metric; +import org.opensearch.index.compositeindex.MetricType; +import org.opensearch.index.compositeindex.StarTreeFieldSpec; +import org.opensearch.index.compositeindex.startree.aggregators.MetricTypeFieldPair; +import org.opensearch.index.compositeindex.startree.data.StarTreeDocValues; +import org.opensearch.index.compositeindex.startree.data.StarTreeDocument; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import static org.mockito.Mockito.mock; + +public class BaseSingleStarTreeBuilderTests extends OpenSearchTestCase { + + private static BaseSingleStarTreeBuilder builder; + private static List dimensionsOrder; + private static List fields = List.of( + "field1", + "field2", + "field3", + "field4", + "field5", + "field6", + "field7", + "field8", + "field9", + "field10" + ); + private static List metrics; + private static Directory directory; + private static FieldInfo[] fieldsInfo; + + @BeforeClass + public static void setup() throws IOException { + + dimensionsOrder = List.of(new Dimension("field1"), new Dimension("field3"), new Dimension("field5"), new Dimension("field8")); + metrics = List.of(new Metric("field2", List.of(MetricType.SUM)), new Metric("field4", List.of(MetricType.SUM))); + + CompositeField compositeField = new CompositeField( + "test", + dimensionsOrder, + metrics, + new StarTreeFieldSpec(1, Set.of("field8"), StarTreeFieldSpec.StarTreeBuildMode.ON_HEAP) + ); + DocValuesConsumer docValuesConsumer = mock(DocValuesConsumer.class); + DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); + directory = newFSDirectory(createTempDir()); + SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LUCENE_9_11_0, + "test_segment", + 5, + false, + false, + new Lucene99Codec(), + new HashMap<>(), + UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), + new HashMap<>(), + null + ); + + fieldsInfo = new FieldInfo[fields.size()]; + + for (int i = 0; i < fieldsInfo.length; i++) { + fieldsInfo[i] = new FieldInfo( + fields.get(i), + i, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + } + FieldInfos fieldInfos = new FieldInfos(fieldsInfo); + final SegmentWriteState state = new SegmentWriteState( + InfoStream.getDefault(), + segmentInfo.dir, + segmentInfo, + fieldInfos, + null, + newIOContext(random()) + ); + + builder = new BaseSingleStarTreeBuilder(compositeField, docValuesProducer, docValuesConsumer, state) { + @Override + public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException {} + + @Override + public StarTreeDocument getStarTreeDocument(int docId) throws IOException { + return null; + } + + @Override + public long getDimensionValue(int docId, int dimensionId) throws IOException { + return 0; + } + + @Override + public Iterator sortAndAggregateSegmentStarTreeDocument(int numDocs) throws IOException { + return null; + } + + @Override + public Iterator generateStarTreeForStarNode(int startDocId, int endDocId, int dimensionId) + throws IOException { + return null; + } + + @Override + public void build(List starTreeDocValues) throws IOException { + + } + }; + } + + public void test_generateAggregationFunctionColumnPairs() throws IOException { + List metricTypeFieldPairs = builder.generateAggregationFunctionColumnPairs(); + List expectedMetricTypeFieldPairs = List.of( + new MetricTypeFieldPair(MetricType.SUM, "field2"), + new MetricTypeFieldPair(MetricType.SUM, "field4") + ); + assertEquals(metricTypeFieldPairs, expectedMetricTypeFieldPairs); + } + + public void test_mergeStarTreeDocument() { + StarTreeDocument starTreeDocument1 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 4.0, 8.0 }); + StarTreeDocument starTreeDocument2 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 10.0, 6.0 }); + + StarTreeDocument expectedeMergedStarTreeDocument = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 14.0, 14.0 }); + StarTreeDocument mergedStarTreeDocument = builder.mergeStarTreeDocument(starTreeDocument1, starTreeDocument2); + + assertEquals(mergedStarTreeDocument.metrics[0], expectedeMergedStarTreeDocument.metrics[0]); + assertEquals(mergedStarTreeDocument.metrics[1], expectedeMergedStarTreeDocument.metrics[1]); + } + + public void test_mergeStarTreeDocument_nullAggregatedStarTreeDocument() { + StarTreeDocument starTreeDocument = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 10.0, 6.0 }); + + StarTreeDocument expectedeMergedStarTreeDocument = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 10.0, 6.0 }); + StarTreeDocument mergedStarTreeDocument = builder.mergeStarTreeDocument(null, starTreeDocument); + + assertEquals(mergedStarTreeDocument.metrics[0], expectedeMergedStarTreeDocument.metrics[0]); + assertEquals(mergedStarTreeDocument.metrics[1], expectedeMergedStarTreeDocument.metrics[1]); + } + + // Test could not be added due to package private classes + // public void test_getDocValuesWriter() { + // assertTrue( + // builder.getDocValuesWriter(DocValuesType.SORTED_SET, fieldsInfo[0], Counter.newCounter()) instanceof SortedSetDocValuesWriter + // ); + // assertTrue( + // builder.getDocValuesWriter( + // DocValuesType.SORTED_NUMERIC, + // fieldsInfo[0], + // Counter.newCounter() + // ) instanceof SortedNumericDocValuesWriter + // ); + // } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + directory.close(); + } +}