-
Notifications
You must be signed in to change notification settings - Fork 1.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Create SystemIndexRegistry with helper method matchesSystemIndex (#14415
) (#14692) * Create new extension point in SystemIndexPlugin for a single plugin to get registered system indices * Add to CHANGELOG * WIP on system indices from IndexNameExpressionResolver * Add test in IndexNameExpressionResolverTests * Remove changes in SystemIndexPlugin * Add method in IndexNameExpressionResolver to get matching system indices * Show how resolver can be chained to get system indices * Fix forbiddenApis check * Update CHANGELOG * Make SystemIndices internal * Remove unneeded changes * Fix CI failures * Fix precommit errors * Use Regex instead of WildcardMatcher * Address code review feedback * Allow caller to pass index expressions * Create SystemIndexRegistry * Update CHANGELOG * Remove singleton limitation * Add javadoc * Add @experimentalapi annotation --------- (cherry picked from commit bf56227) Signed-off-by: Craig Perkins <cwperx@amazon.com> Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
- Loading branch information
1 parent
7040df2
commit a4a6bb8
Showing
6 changed files
with
242 additions
and
97 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
130 changes: 130 additions & 0 deletions
130
server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,130 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.indices; | ||
|
|
||
| import org.apache.lucene.util.automaton.Automaton; | ||
| import org.apache.lucene.util.automaton.Operations; | ||
| import org.opensearch.common.annotation.ExperimentalApi; | ||
| import org.opensearch.common.collect.Tuple; | ||
| import org.opensearch.common.regex.Regex; | ||
| import org.opensearch.tasks.TaskResultsService; | ||
|
|
||
| import java.util.Arrays; | ||
| import java.util.Collection; | ||
| import java.util.Collections; | ||
| import java.util.Comparator; | ||
| import java.util.HashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| import static java.util.Collections.singletonList; | ||
| import static java.util.Collections.singletonMap; | ||
| import static java.util.Collections.unmodifiableMap; | ||
| import static org.opensearch.tasks.TaskResultsService.TASK_INDEX; | ||
|
|
||
| /** | ||
| * This class holds the {@link SystemIndexDescriptor} objects that represent system indices the | ||
| * node knows about. This class also contains static methods that identify if index expressions match | ||
| * registered system index patterns | ||
| * | ||
| * @opensearch.api | ||
| */ | ||
| @ExperimentalApi | ||
| public class SystemIndexRegistry { | ||
| private static final SystemIndexDescriptor TASK_INDEX_DESCRIPTOR = new SystemIndexDescriptor(TASK_INDEX + "*", "Task Result Index"); | ||
| private static final Map<String, Collection<SystemIndexDescriptor>> SERVER_SYSTEM_INDEX_DESCRIPTORS = singletonMap( | ||
| TaskResultsService.class.getName(), | ||
| singletonList(TASK_INDEX_DESCRIPTOR) | ||
| ); | ||
|
|
||
| private volatile static String[] SYSTEM_INDEX_PATTERNS = new String[0]; | ||
| volatile static Collection<SystemIndexDescriptor> SYSTEM_INDEX_DESCRIPTORS = Collections.emptyList(); | ||
|
|
||
| static void register(Map<String, Collection<SystemIndexDescriptor>> pluginAndModulesDescriptors) { | ||
| final Map<String, Collection<SystemIndexDescriptor>> descriptorsMap = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors); | ||
| checkForOverlappingPatterns(descriptorsMap); | ||
| List<SystemIndexDescriptor> descriptors = pluginAndModulesDescriptors.values() | ||
| .stream() | ||
| .flatMap(Collection::stream) | ||
| .collect(Collectors.toList()); | ||
| descriptors.add(TASK_INDEX_DESCRIPTOR); | ||
|
|
||
| SYSTEM_INDEX_DESCRIPTORS = descriptors.stream().collect(Collectors.toUnmodifiableList()); | ||
| SYSTEM_INDEX_PATTERNS = descriptors.stream().map(SystemIndexDescriptor::getIndexPattern).toArray(String[]::new); | ||
| } | ||
|
|
||
| public static List<String> matchesSystemIndexPattern(String... indexExpressions) { | ||
| return Arrays.stream(indexExpressions) | ||
| .filter(pattern -> Regex.simpleMatch(SYSTEM_INDEX_PATTERNS, pattern)) | ||
| .collect(Collectors.toList()); | ||
| } | ||
|
|
||
| /** | ||
| * Given a collection of {@link SystemIndexDescriptor}s and their sources, checks to see if the index patterns of the listed | ||
| * descriptors overlap with any of the other patterns. If any do, throws an exception. | ||
| * | ||
| * @param sourceToDescriptors A map of source (plugin) names to the SystemIndexDescriptors they provide. | ||
| * @throws IllegalStateException Thrown if any of the index patterns overlaps with another. | ||
| */ | ||
| static void checkForOverlappingPatterns(Map<String, Collection<SystemIndexDescriptor>> sourceToDescriptors) { | ||
| List<Tuple<String, SystemIndexDescriptor>> sourceDescriptorPair = sourceToDescriptors.entrySet() | ||
| .stream() | ||
| .flatMap(entry -> entry.getValue().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor))) | ||
| .sorted(Comparator.comparing(d -> d.v1() + ":" + d.v2().getIndexPattern())) // Consistent ordering -> consistent error message | ||
| .collect(Collectors.toList()); | ||
|
|
||
| // This is O(n^2) with the number of system index descriptors, and each check is quadratic with the number of states in the | ||
| // automaton, but the absolute number of system index descriptors should be quite small (~10s at most), and the number of states | ||
| // per pattern should be low as well. If these assumptions change, this might need to be reworked. | ||
| sourceDescriptorPair.forEach(descriptorToCheck -> { | ||
| List<Tuple<String, SystemIndexDescriptor>> descriptorsMatchingThisPattern = sourceDescriptorPair.stream() | ||
|
|
||
| .filter(d -> descriptorToCheck.v2() != d.v2()) // Exclude the pattern currently being checked | ||
| .filter(d -> overlaps(descriptorToCheck.v2(), d.v2())) | ||
| .collect(Collectors.toList()); | ||
| if (descriptorsMatchingThisPattern.isEmpty() == false) { | ||
| throw new IllegalStateException( | ||
| "a system index descriptor [" | ||
| + descriptorToCheck.v2() | ||
| + "] from [" | ||
| + descriptorToCheck.v1() | ||
| + "] overlaps with other system index descriptors: [" | ||
| + descriptorsMatchingThisPattern.stream() | ||
| .map(descriptor -> descriptor.v2() + " from [" + descriptor.v1() + "]") | ||
| .collect(Collectors.joining(", ")) | ||
| ); | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| private static boolean overlaps(SystemIndexDescriptor a1, SystemIndexDescriptor a2) { | ||
| Automaton a1Automaton = Regex.simpleMatchToAutomaton(a1.getIndexPattern()); | ||
| Automaton a2Automaton = Regex.simpleMatchToAutomaton(a2.getIndexPattern()); | ||
| return Operations.isEmpty(Operations.intersection(a1Automaton, a2Automaton)) == false; | ||
| } | ||
|
|
||
| private static Map<String, Collection<SystemIndexDescriptor>> buildSystemIndexDescriptorMap( | ||
| Map<String, Collection<SystemIndexDescriptor>> pluginAndModulesMap | ||
| ) { | ||
| final Map<String, Collection<SystemIndexDescriptor>> map = new HashMap<>( | ||
| pluginAndModulesMap.size() + SERVER_SYSTEM_INDEX_DESCRIPTORS.size() | ||
| ); | ||
| map.putAll(pluginAndModulesMap); | ||
| // put the server items last since we expect less of them | ||
| SERVER_SYSTEM_INDEX_DESCRIPTORS.forEach((source, descriptors) -> { | ||
| if (map.putIfAbsent(source, descriptors) != null) { | ||
| throw new IllegalArgumentException( | ||
| "plugin or module attempted to define the same source [" + source + "] as a built-in system index" | ||
| ); | ||
| } | ||
| }); | ||
| return unmodifiableMap(map); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.