Deprecate the behaviour of implicitly disabling file/native realm (elastic#69320)

As a precursor for elastic#50892, this PR deprecate the behaviour of file and/or
native realm being implicitly disabled when there are other explicitly
configured realms.

With this change, the recommend way of disabling file/native realm is to
explicitly set enabled to false, e.g.:

xpack.security.authc.realms.file.default_file.enabled: false

This PR ensures that a warning is generated whenever file and/or native realm
is implicitly disabled.

This change also brings a question about the order parameter. Currently, the
order parameter is mandatory in 8.0 and gets a warning message if it is missing
in 7.x. However, it makes sense to not specify the order parameter if the realm
is disabled. So I also updated the order parameter related code to do just
that.

Author: ywangd

docs/reference/migration/index.asciidoc

@@ -28,6 +28,7 @@ For more information about {minor-version},
 see the <<release-highlights>> and <<es-release-notes>>.
 For information about how to upgrade your cluster, see <<setup-upgrade>>.
 
+* <<breaking-changes-7.13,Migrating to 7.13>>
 * <<breaking-changes-7.12,Migrating to 7.12>>
 * <<breaking-changes-7.11,Migrating to 7.11>>
 * <<breaking-changes-7.10,Migrating to 7.10>>
@@ -44,6 +45,7 @@ For information about how to upgrade your cluster, see <<setup-upgrade>>.
 
 --
 
+include::migrate_7_13.asciidoc[]
 include::migrate_7_12.asciidoc[]
 include::migrate_7_11.asciidoc[]
 include::migrate_7_10.asciidoc[]

docs/reference/migration/migrate_7_13.asciidoc

@@ -0,0 +1,67 @@
+[[migrating-7.13]]
+== Migrating to 7.13
+++++
+<titleabbrev>7.13</titleabbrev>
+++++
+
+This section discusses the changes that you need to be aware of when migrating
+your application to {es} 7.13.
+
+See also <<release-highlights>> and <<es-release-notes>>.
+
+// * <<breaking_713_blah_changes>>
+// * <<breaking_713_blah_changes>>
+
+//NOTE: The notable-breaking-changes tagged regions are re-used in the
+//Installation and Upgrade Guide
+
+//tag::notable-breaking-changes[]
+
+[discrete]
+[[breaking-changes-7.13]]
+=== Breaking changes
+
+The following changes in {es} 7.13 might affect your applications
+and prevent them from operating normally.
+Before upgrading to 7.13, review these changes and take the described steps
+to mitigate the impact.
+
+NOTE: Breaking changes introduced in minor versions are
+normally limited to security and bug fixes.
+Significant changes in behavior are deprecated in a minor release and
+the old behavior is supported until the next major release.
+To find out if you are using any deprecated functionality,
+enable <<deprecation-logging, deprecation logging>>.
+
+
+[discrete]
+[[deprecated-7.13]]
+=== Deprecations
+
+The following functionality has been deprecated in {es} 7.13
+and will be removed in 8.0
+While this won't have an immediate impact on your applications,
+we strongly encourage you take the described steps to update your code
+after upgrading to 7.13.
+
+NOTE: Significant changes in behavior are deprecated in a minor release and
+the old behavior is supported until the next major release.
+To find out if you are using any deprecated functionality,
+enable <<deprecation-logging, deprecation logging>>.
+
+[discrete]
+[[breaking_713_security_changes]]
+==== Security deprecations
+
+[[implicitly-disabled-basic-realms]]
+Currently, the file and native realms have following implicit behaviours:
+
+* If file and native realms are not configured, they are implicitly disabled
+if there are other explicitly configured realms.
+* If no realm is available due to either unconfigured, explicitly disabled
+or disallowed by the license, the file and native realms are always enabled
+even when they are explicitly disabled.
+
+Both of the above behaviours are deprecated. In version 8.0.0, the file and
+native realms will always be enabled unless explicitly disabled. If they are
+explicitly disabled, they remain disabled at all times.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmSettings.java

@@ -30,9 +30,10 @@
 public class RealmSettings {
 
     public static final String PREFIX = "xpack.security.authc.realms.";
+    public static final String ENABLED_SETTING_KEY = "enabled";
     public static final String ORDER_SETTING_KEY = "order";
 
-    public static final Function<String, Setting.AffixSetting<Boolean>> ENABLED_SETTING = affixSetting("enabled",
+    public static final Function<String, Setting.AffixSetting<Boolean>> ENABLED_SETTING = affixSetting(ENABLED_SETTING_KEY,
             key -> Setting.boolSetting(key, true, Setting.Property.NodeScope));
     public static final Function<String, Setting.AffixSetting<Integer>> ORDER_SETTING = affixSetting(ORDER_SETTING_KEY,
             key -> Setting.intSetting(key, Integer.MAX_VALUE, Setting.Property.NodeScope));

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java

@@ -47,6 +47,7 @@ private DeprecationChecks() {
             NodeDeprecationChecks::checkProcessors,
             NodeDeprecationChecks::checkMissingRealmOrders,
             NodeDeprecationChecks::checkUniqueRealmOrders,
+            NodeDeprecationChecks::checkImplicitlyDisabledBasicRealms,
             (settings, pluginsAndModules) -> NodeDeprecationChecks.checkThreadPoolListenerQueueSize(settings),
             (settings, pluginsAndModules) -> NodeDeprecationChecks.checkThreadPoolListenerSize(settings),
             NodeDeprecationChecks::checkClusterRemoteConnectSetting,

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java

@@ -9,18 +9,24 @@
 
 import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
 import org.elasticsearch.bootstrap.JavaVersion;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.FixedExecutorBuilder;
 import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
+import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
+import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
 
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -52,6 +58,7 @@ static DeprecationIssue checkMissingRealmOrders(final Settings settings, final P
         final Set<String> orderNotConfiguredRealms = RealmSettings.getRealmSettings(settings).entrySet()
                 .stream()
                 .filter(e -> false == e.getValue().hasValue(RealmSettings.ORDER_SETTING_KEY))
+                .filter(e -> e.getValue().getAsBoolean(RealmSettings.ENABLED_SETTING_KEY, true))
                 .map(e -> RealmSettings.realmSettingPrefix(e.getKey()) + RealmSettings.ORDER_SETTING_KEY)
                 .collect(Collectors.toSet());
 
@@ -104,6 +111,57 @@ static DeprecationIssue checkUniqueRealmOrders(final Settings settings, final Pl
         );
     }
 
+    static DeprecationIssue checkImplicitlyDisabledBasicRealms(final Settings settings, final PluginsAndModules pluginsAndModules) {
+        final Map<RealmConfig.RealmIdentifier, Settings> realmSettings = RealmSettings.getRealmSettings(settings);
+        if (realmSettings.isEmpty()) {
+            return null;
+        }
+
+        boolean anyRealmEnabled = false;
+        final Set<String> unconfiguredBasicRealms =
+            new HashSet<>(org.elasticsearch.common.collect.Set.of(FileRealmSettings.TYPE, NativeRealmSettings.TYPE));
+        for (Map.Entry<RealmConfig.RealmIdentifier, Settings> realmSetting: realmSettings.entrySet()) {
+            anyRealmEnabled = anyRealmEnabled || realmSetting.getValue().getAsBoolean(RealmSettings.ENABLED_SETTING_KEY, true);
+            unconfiguredBasicRealms.remove(realmSetting.getKey().getType());
+        }
+
+        final String details;
+        if (false == anyRealmEnabled) {
+            final List<String> explicitlyDisabledBasicRealms =
+                Sets.difference(org.elasticsearch.common.collect.Set.of(FileRealmSettings.TYPE, NativeRealmSettings.TYPE),
+                    unconfiguredBasicRealms).stream().sorted().collect(Collectors.toList());
+            if (explicitlyDisabledBasicRealms.isEmpty()) {
+                return null;
+            }
+            details = String.format(
+                Locale.ROOT,
+                "Found explicitly disabled basic %s: [%s]. But %s will be enabled because no other realms are configured or enabled. " +
+                    "In next major release, explicitly disabled basic realms will remain disabled.",
+                explicitlyDisabledBasicRealms.size() == 1 ? "realm" : "realms",
+                Strings.collectionToDelimitedString(explicitlyDisabledBasicRealms, ","),
+                explicitlyDisabledBasicRealms.size() == 1 ? "it" : "they"
+                );
+        } else {
+            if (unconfiguredBasicRealms.isEmpty()) {
+                return null;
+            }
+            details = String.format(
+                Locale.ROOT,
+                "Found implicitly disabled basic %s: [%s]. %s disabled because there are other explicitly configured realms." +
+                    "In next major release, basic realms will always be enabled unless explicitly disabled.",
+                unconfiguredBasicRealms.size() == 1 ? "realm" : "realms",
+                Strings.collectionToDelimitedString(unconfiguredBasicRealms, ","),
+                unconfiguredBasicRealms.size() == 1 ? "It is" : "They are");
+        }
+        return new DeprecationIssue(
+            DeprecationIssue.Level.WARNING,
+            "File and/or native realms are enabled by default in next major release.",
+            "https://www.elastic.co/guide/en/elasticsearch/reference/7.13/deprecated-7.13.html#implicitly-disabled-basic-realms",
+            details
+        );
+
+    }
+
     static DeprecationIssue checkThreadPoolListenerQueueSize(final Settings settings) {
         return checkThreadPoolListenerSetting("thread_pool.listener.queue_size", settings);
     }