[7.16] Script: Implicit context cache (#81552) (#81560)

* [7.16] Script: Implicit context cache (#81552)

The script context cache is deprecated.  In 7.16 the default
value of `script.max_compilation_rate` was switched from `"use-context"`,
to `"150/5m".  That means uses would have to explicitly set "use-context"
to use any of the context cache families of settings:
`script.context.*.max_compilations_rate`
`script.context.*.cache_max_size`
`script.context.*.cache_expire`

On upgrades to 7.16, if the customer was using the default and had
set any of those settings without setting
`script.max_compilation_rate: "use-context"`, the upgrade would be
rejected.

To avoid an unintentional breaking change, the script service will
now **implicitly** use the script context cache if `script.max_compilation_rate`
is **unset** and any of the context cache family of settings is set.

The context cache will also be used if
`script.max_compilation_rate: "use-context"`, as before.

Fixes: #81486
Backport: ba6c17d

docs/changelog/81552.yaml

@@ -0,0 +1,5 @@
+pr: 81552
+summary: "Script: Implicit context cache"
+area: Infra/Scripting
+type: bug
+issues: []

server/src/main/java/org/elasticsearch/script/ScriptService.java

@@ -277,6 +277,14 @@ public static boolean isUseContextCacheSet(Settings settings) {
         return SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings).equals(USE_CONTEXT_RATE_VALUE);
     }
 
+    public static boolean isImplicitContextCacheSet(Settings settings) {
+        return new ScriptService.ContextSettings(settings).implicitContextCache();
+    }
+
+    public static String contextDeprecationMessage(Settings settings) {
+        return new ScriptService.ContextSettings(settings).deprecationMessage();
+    }
+
     /**
      * This is overridden in tests to disable compilation rate limiting.
      */
@@ -323,53 +331,26 @@ void registerClusterSettingsListeners(ClusterSettings clusterSettings) {
      * when using the general cache.
      */
     void validateCacheSettings(Settings settings) {
-        boolean useContext = SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings).equals(USE_CONTEXT_RATE_VALUE);
-        if (useContext) {
+        ContextSettings contextSettings = new ContextSettings(settings, contexts.keySet());
+        if (contextSettings.useContextSet) {
             deprecationLogger.warn(DeprecationCategory.SCRIPTING, "scripting-context-cache", USE_CONTEXT_RATE_KEY_DEPRECATION_MESSAGE);
+        } else if (contextSettings.hasContextSettings()) {
+            deprecationLogger.warn(DeprecationCategory.SCRIPTING, "scripting-context-cache", contextSettings.deprecationMessage());
         }
-        List<Setting.AffixSetting<?>> affixes = Arrays.asList(
-            SCRIPT_MAX_COMPILATIONS_RATE_SETTING,
-            SCRIPT_CACHE_EXPIRE_SETTING,
-            SCRIPT_CACHE_SIZE_SETTING
-        );
-        List<String> customRates = new ArrayList<>();
-        List<String> keys = new ArrayList<>();
-        for (Setting.AffixSetting<?> affix : affixes) {
-            for (String context : affix.getAsMap(settings).keySet()) {
-                String s = affix.getConcreteSettingForNamespace(context).getKey();
-                if (contexts.containsKey(context) == false) {
-                    throw new IllegalArgumentException("Context [" + context + "] doesn't exist for setting [" + s + "]");
-                }
-                keys.add(s);
-                if (affix.equals(SCRIPT_MAX_COMPILATIONS_RATE_SETTING)) {
-                    customRates.add(s);
-                }
-            }
-        }
-        if (useContext == false && keys.isEmpty() == false) {
-            keys.sort(Comparator.naturalOrder());
-            throw new IllegalArgumentException(
-                "Context cache settings ["
-                    + String.join(", ", keys)
-                    + "] requires ["
-                    + SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey()
-                    + "] to be ["
-                    + USE_CONTEXT_RATE_KEY
-                    + "]"
-            );
+        if (contextSettings.incompatibleSettings()) {
+            throw new IllegalArgumentException(contextSettings.incompatibleSettingsMessage());
         }
         if (SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING.get(settings)) {
-            if (customRates.size() > 0) {
-                customRates.sort(Comparator.naturalOrder());
+            if (contextSettings.compilationContexts.size() > 0) {
                 throw new IllegalArgumentException(
                     "Cannot set custom context compilation rates ["
-                        + String.join(", ", customRates)
+                        + String.join(", ", contextSettings.contextCompilationKeys())
                         + "] if compile rates disabled via ["
                         + SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING.getKey()
                         + "]"
                 );
             }
-            if (useContext == false && SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.exists(settings)) {
+            if (contextSettings.useContextSet == false && contextSettings.isGeneralCompilationRateSet) {
                 throw new IllegalArgumentException(
                     "Cannot set custom general compilation rates ["
                         + SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey()
@@ -383,6 +364,144 @@ void validateCacheSettings(Settings settings) {
         }
     }
 
+    /**
+     * Collect settings related to script context and general caches.
+     *
+     * The general cache is used by default.
+     * The context cache is used if {@code script.max_compilations_rate} is {@code "use-context"}, a deprecated value.
+     * The context cache is used implicitly if {@code script.max_compilations_rate} is unset and any of the context
+     * cache family of settings is used:
+     *   {@code script.context.*.max_compilations_rate}, {@link ScriptService#SCRIPT_MAX_COMPILATIONS_RATE_SETTING}
+     *   {@code script.context.*.cache_max_size}, {@link ScriptService#SCRIPT_CACHE_SIZE_SETTING}
+     *   {@code script.context.*.cache_expire}, {@link ScriptService#SCRIPT_CACHE_EXPIRE_SETTING}
+     */
+    public static class ContextSettings {
+        public final Settings settings;
+        public final boolean useContextSet;
+        public final boolean isGeneralCompilationRateSet;
+        public final ScriptCache.CompilationRate generalCompilationRate;
+        public final List<String> compilationContexts;
+        public final List<String> sizeContexts;
+        public final List<String> expireContexts;
+
+        public ContextSettings(Settings settings, Set<String> contexts) {
+            this.settings = settings;
+            generalCompilationRate = SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings);
+            useContextSet = generalCompilationRate.equals(USE_CONTEXT_RATE_VALUE);
+            isGeneralCompilationRateSet = SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.exists(settings);
+            compilationContexts = getContexts(SCRIPT_MAX_COMPILATIONS_RATE_SETTING, settings, contexts);
+            sizeContexts = getContexts(SCRIPT_CACHE_SIZE_SETTING, settings, contexts);
+            expireContexts = getContexts(SCRIPT_CACHE_EXPIRE_SETTING, settings, contexts);
+        }
+
+        public ContextSettings(Settings settings) {
+            this(settings, Collections.emptySet());
+        }
+
+        protected static List<String> getContexts(Setting.AffixSetting<?> setting, Settings settings, Set<String> contexts) {
+            List<String> contextSettings = new ArrayList<>();
+            for (String context : setting.getAsMap(settings).keySet()) {
+                if (contexts.isEmpty() == false && contexts.contains(context) == false) {
+                    String settingKey = setting.getConcreteSettingForNamespace(context).getKey();
+                    throw new IllegalArgumentException("Context [" + context + "] doesn't exist for setting [" + settingKey + "]");
+                }
+                contextSettings.add(context);
+            }
+            contextSettings.sort(Comparator.naturalOrder());
+            return contextSettings;
+        }
+
+        /** Are there any context specific settings */
+        public boolean hasContextSettings() {
+            return compilationContexts.isEmpty() == false || expireContexts.isEmpty() == false || sizeContexts.isEmpty() == false;
+        }
+
+        /** deprecation message for implicitly using the context cache */
+        public String deprecationMessage() {
+            // Implicitly using the script context cache is deprecated, remove the following deprecated settings to use the script general
+            // cache.
+            if (hasContextSettings() == false) {
+                return "";
+            }
+            List<String> settingsKeys = new ArrayList<>();
+            settingsKeys.addAll(fullKeys(SCRIPT_MAX_COMPILATIONS_RATE_SETTING, compilationContexts));
+            settingsKeys.addAll(fullKeys(SCRIPT_CACHE_SIZE_SETTING, sizeContexts));
+            settingsKeys.addAll(fullKeys(SCRIPT_CACHE_EXPIRE_SETTING, expireContexts));
+            settingsKeys.sort(Comparator.naturalOrder());
+            return "Implicitly using the script context cache is deprecated, remove settings "
+                + "["
+                + String.join(", ", settingsKeys)
+                + "] to use the script general cache.";
+        }
+
+        /** the context specific max compilation keys */
+        public List<String> contextCompilationKeys() {
+            return fullKeys(SCRIPT_MAX_COMPILATIONS_RATE_SETTING, compilationContexts);
+        }
+
+        /** the full keys for the contexts in the context affix setting */
+        protected static List<String> fullKeys(Setting.AffixSetting<?> affix, List<String> contexts) {
+            return contexts.stream().map(ctx -> affix.getConcreteSettingForNamespace(ctx).getKey()).collect(Collectors.toList());
+        }
+
+        /**
+         *  Should the context cache be used?  This is true if "use-context" is set explicitly or implicitly, see above for implicit
+         *  definition.
+         */
+        public boolean useContextCache() {
+            return useContextSet || implicitContextCache();
+        }
+
+        /**
+         * Implicitly use the script context cache.  False if context cache is explicitly used as well as context cache is unused.
+         */
+        public boolean implicitContextCache() {
+            return useContextSet == false && hasContextSettings() && isGeneralCompilationRateSet == false;
+        }
+
+        /**
+         * Is the set of settings incompatible?  This is the case if:
+         * 1) {@code script.max_compilations_rate}, {@link ScriptService#SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING} is set but not
+         *    set to "use-context".
+         * 2) Any of the context cache family of settings is set.
+         */
+        public boolean incompatibleSettings() {
+            return useContextSet == false && hasContextSettings() && isGeneralCompilationRateSet;
+        }
+
+        /**
+         * All context specific settings
+         */
+        public List<String> contextSettings() {
+            List<String> contextSettings = fullKeys(SCRIPT_MAX_COMPILATIONS_RATE_SETTING, compilationContexts);
+            contextSettings.addAll(fullKeys(SCRIPT_CACHE_SIZE_SETTING, sizeContexts));
+            contextSettings.addAll(fullKeys(SCRIPT_CACHE_EXPIRE_SETTING, expireContexts));
+            return contextSettings;
+        }
+
+        /**
+         * Error message if there are incompatible settings.
+         */
+        public String incompatibleSettingsMessage() {
+            if (incompatibleSettings() == false) {
+                return "";
+            }
+            List<String> incompatible = contextSettings();
+            return "Context cache settings ["
+                + String.join(",", incompatible)
+                + "] are incompatible with ["
+                + SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey()
+                + "] set to non-default value ["
+                + generalCompilationRate
+                + "]."
+                + " Either remove the incompatible settings (recommended) or set ["
+                + SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey()
+                + "] to ["
+                + USE_CONTEXT_RATE_KEY
+                + "] to use per-context settings";
+        }
+    }
+
     @Override
     public void close() throws IOException {
         IOUtils.close(engines.values());
@@ -670,10 +789,10 @@ public void applyClusterState(ClusterChangedEvent event) {
 
     void setCacheHolder(Settings settings) {
         CacheHolder current = cacheHolder.get();
-        boolean useContext = SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings).equals(USE_CONTEXT_RATE_VALUE);
+        ContextSettings contextSettings = new ContextSettings(settings, contexts.keySet());
 
         if (current == null) {
-            if (useContext) {
+            if (contextSettings.useContextCache()) {
                 cacheHolder.set(contextCacheHolder(settings));
             } else {
                 cacheHolder.set(generalCacheHolder(settings));
@@ -682,7 +801,7 @@ void setCacheHolder(Settings settings) {
         }
 
         // Update
-        if (useContext) {
+        if (contextSettings.useContextCache()) {
             if (current.general != null) {
                 // Flipping to context specific
                 cacheHolder.set(contextCacheHolder(settings));