Align dot prefix validation with Serverless (elastic#116266)

This aligns the deprecation warnings for on-prem dot-prefixed indices to
be the same as the Serverless validation. It adds exemptions for the
`.entities…` indices, and makes the list a dynamic setting.

(cherry picked from commit 72aa17a)

docs/changelog/116266.yaml

@@ -0,0 +1,5 @@
+pr: 116266
+summary: Align dot prefix validation with Serverless
+area: Indices APIs
+type: bug
+issues: []

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidationPlugin.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.action.support.MappedActionFilter;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -45,4 +46,9 @@ public Collection<?> createComponents(PluginServices services) {
     public Collection<MappedActionFilter> getMappedActionFilters() {
         return actionFilters.get();
     }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return List.of(DotPrefixValidator.VALIDATE_DOT_PREFIXES, DotPrefixValidator.IGNORED_INDEX_PATTERNS_SETTING);
+    }
 }

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java

@@ -23,9 +23,12 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.tasks.Task;
 
+import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+import java.util.stream.Collectors;
 
 /**
  * DotPrefixValidator provides an abstract class implementing a mapped action filter.
@@ -39,7 +42,7 @@
  * method, which subclasses must implement.
  *
  * Some built-in index names and patterns are also elided from the check, as defined in
- * {@link #IGNORED_INDEX_NAMES} and {@link #IGNORED_INDEX_PATTERNS}.
+ * {@link #IGNORED_INDEX_NAMES} and {@link #IGNORED_INDEX_PATTERNS_SETTING}.
  */
 public abstract class DotPrefixValidator<RequestType> implements MappedActionFilter {
     public static final Setting<Boolean> VALIDATE_DOT_PREFIXES = Setting.boolSetting(
@@ -64,20 +67,43 @@ public abstract class DotPrefixValidator<RequestType> implements MappedActionFil
         ".ml-state",
         ".ml-anomalies-unrelated"
     );
-    private static Set<Pattern> IGNORED_INDEX_PATTERNS = Set.of(
-        Pattern.compile("\\.ml-state-\\d+"),
-        Pattern.compile("\\.slo-observability\\.sli-v\\d+.*"),
-        Pattern.compile("\\.slo-observability\\.summary-v\\d+.*")
+    public static Setting<List<String>> IGNORED_INDEX_PATTERNS_SETTING = Setting.stringListSetting(
+        "cluster.indices.validate_ignored_dot_patterns",
+        List.of(
+            "\\.ml-state-\\d+",
+            "\\.slo-observability\\.sli-v\\d+.*",
+            "\\.slo-observability\\.summary-v\\d+.*",
+            "\\.entities\\.v\\d+\\.latest\\..*"
+        ),
+        (patternList) -> patternList.forEach(pattern -> {
+            try {
+                Pattern.compile(pattern);
+            } catch (PatternSyntaxException e) {
+                throw new IllegalArgumentException("invalid dot validation exception pattern: [" + pattern + "]", e);
+            }
+        }),
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
     );
 
     DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DotPrefixValidator.class);
 
     private final ThreadContext threadContext;
     private final boolean isEnabled;
+    private volatile Set<Pattern> ignoredIndexPatterns;
 
     public DotPrefixValidator(ThreadContext threadContext, ClusterService clusterService) {
         this.threadContext = threadContext;
         this.isEnabled = VALIDATE_DOT_PREFIXES.get(clusterService.getSettings());
+        this.ignoredIndexPatterns = IGNORED_INDEX_PATTERNS_SETTING.get(clusterService.getSettings())
+            .stream()
+            .map(Pattern::compile)
+            .collect(Collectors.toSet());
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(IGNORED_INDEX_PATTERNS_SETTING, this::updateIgnoredIndexPatterns);
+    }
+
+    private void updateIgnoredIndexPatterns(List<String> patterns) {
+        this.ignoredIndexPatterns = patterns.stream().map(Pattern::compile).collect(Collectors.toSet());
     }
 
     protected abstract Set<String> getIndicesFromRequest(RequestType request);
@@ -108,7 +134,7 @@ void validateIndices(@Nullable Set<String> indices) {
                         if (IGNORED_INDEX_NAMES.contains(strippedName)) {
                             return;
                         }
-                        if (IGNORED_INDEX_PATTERNS.stream().anyMatch(p -> p.matcher(strippedName).matches())) {
+                        if (this.ignoredIndexPatterns.stream().anyMatch(p -> p.matcher(strippedName).matches())) {
                             return;
                         }
                         deprecationLogger.warn(

modules/dot-prefix-validation/src/test/java/org/elasticsearch/validation/DotPrefixValidatorTests.java

@@ -10,16 +10,17 @@
 package org.elasticsearch.validation;
 
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.BeforeClass;
 
-import java.util.HashSet;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Set;
 
 import static org.mockito.Mockito.mock;
@@ -28,23 +29,24 @@
 public class DotPrefixValidatorTests extends ESTestCase {
     private final OperatorValidator<?> opV = new OperatorValidator<>();
     private final NonOperatorValidator<?> nonOpV = new NonOperatorValidator<>();
-    private static final Set<Setting<?>> settings;
 
     private static ClusterService clusterService;
-    private static ClusterSettings clusterSettings;
-
-    static {
-        Set<Setting<?>> cSettings = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        cSettings.add(DotPrefixValidator.VALIDATE_DOT_PREFIXES);
-        settings = cSettings;
-    }
 
     @BeforeClass
     public static void beforeClass() {
+        List<String> allowed = new ArrayList<>(DotPrefixValidator.IGNORED_INDEX_PATTERNS_SETTING.getDefault(Settings.EMPTY));
+        // Add a new allowed pattern for testing
+        allowed.add("\\.potato\\d+");
+        Settings settings = Settings.builder()
+            .put(DotPrefixValidator.IGNORED_INDEX_PATTERNS_SETTING.getKey(), Strings.collectionToCommaDelimitedString(allowed))
+            .build();
         clusterService = mock(ClusterService.class);
-        clusterSettings = new ClusterSettings(Settings.EMPTY, Sets.newHashSet(DotPrefixValidator.VALIDATE_DOT_PREFIXES));
+        ClusterSettings clusterSettings = new ClusterSettings(
+            settings,
+            Sets.newHashSet(DotPrefixValidator.VALIDATE_DOT_PREFIXES, DotPrefixValidator.IGNORED_INDEX_PATTERNS_SETTING)
+        );
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
-        when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
+        when(clusterService.getSettings()).thenReturn(settings);
         when(clusterService.threadPool()).thenReturn(mock(ThreadPool.class));
     }
 
@@ -74,6 +76,13 @@ public void testValidation() {
         nonOpV.validateIndices(Set.of(".slo-observability.summary-v2.3"));
         nonOpV.validateIndices(Set.of(".slo-observability.summary-v2.3-2024-01-01"));
         nonOpV.validateIndices(Set.of("<.slo-observability.summary-v3.3.{2024-10-16||/M{yyyy-MM-dd|UTC}}>"));
+        nonOpV.validateIndices(Set.of(".entities.v1.latest.builtin_services_from_ecs_data"));
+        nonOpV.validateIndices(Set.of(".entities.v92.latest.eggplant.potato"));
+        nonOpV.validateIndices(Set.of("<.entities.v12.latest.eggplant-{M{yyyy-MM-dd|UTC}}>"));
+
+        // Test pattern added to the settings
+        nonOpV.validateIndices(Set.of(".potato5"));
+        nonOpV.validateIndices(Set.of("<.potato5>"));
     }
 
     private void assertFails(Set<String> indices) {
@@ -85,7 +94,7 @@ private void assertFails(Set<String> indices) {
         );
     }
 
-    private class NonOperatorValidator<R> extends DotPrefixValidator<R> {
+    private static class NonOperatorValidator<R> extends DotPrefixValidator<R> {
 
         private NonOperatorValidator() {
             super(new ThreadContext(Settings.EMPTY), clusterService);
@@ -107,7 +116,7 @@ boolean isInternalRequest() {
         }
     }
 
-    private class OperatorValidator<R> extends NonOperatorValidator<R> {
+    private static class OperatorValidator<R> extends NonOperatorValidator<R> {
         @Override
         boolean isInternalRequest() {
             return true;