Metadata of API key authentication must have roleDescriptor keys (#84203)

This PR removes conditional check for metadata keys related to API key
role descriptors. API key authentication must always have these keys for
it to work. The PR adds assertions for these keys and fixes relevant
tests.

Relates: #82639

Author: ywangd

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java

@@ -696,43 +696,39 @@ private static Map<String, Object> maybeRewriteMetadataForApiKeyRoleDescriptors(
         Map<String, Object> metadata = authentication.getMetadata();
         // If authentication type is API key, regardless whether it has run-as, the metadata must contain API key role descriptors
         if (authentication.isAuthenticatedWithApiKey()) {
+            assert metadata.containsKey(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY)
+                : "metadata must contain role descriptor for API key authentication";
+            assert metadata.containsKey(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)
+                : "metadata must contain limited role descriptor for API key authentication";
             if (authentication.getVersion().onOrAfter(VERSION_API_KEY_ROLES_AS_BYTES)
                 && streamVersion.before(VERSION_API_KEY_ROLES_AS_BYTES)) {
                 metadata = new HashMap<>(metadata);
-                if (metadata.containsKey(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY)) {
+                metadata.put(
+                    AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY,
+                    convertRoleDescriptorsBytesToMap((BytesReference) metadata.get(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY))
+                );
+                metadata.put(
+                    AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY,
+                    convertRoleDescriptorsBytesToMap(
+                        (BytesReference) metadata.get(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)
+                    )
+                );
+            } else if (authentication.getVersion().before(VERSION_API_KEY_ROLES_AS_BYTES)
+                && streamVersion.onOrAfter(VERSION_API_KEY_ROLES_AS_BYTES)) {
+                    metadata = new HashMap<>(metadata);
                     metadata.put(
                         AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY,
-                        convertRoleDescriptorsBytesToMap((BytesReference) metadata.get(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY))
+                        convertRoleDescriptorsMapToBytes(
+                            (Map<String, Object>) metadata.get(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY)
+                        )
                     );
-                }
-                if (metadata.containsKey(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)) {
                     metadata.put(
                         AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY,
-                        convertRoleDescriptorsBytesToMap(
-                            (BytesReference) metadata.get(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)
+                        convertRoleDescriptorsMapToBytes(
+                            (Map<String, Object>) metadata.get(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)
                         )
                     );
                 }
-            } else if (authentication.getVersion().before(VERSION_API_KEY_ROLES_AS_BYTES)
-                && streamVersion.onOrAfter(VERSION_API_KEY_ROLES_AS_BYTES)) {
-                    metadata = new HashMap<>(metadata);
-                    if (metadata.containsKey(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY)) {
-                        metadata.put(
-                            AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY,
-                            convertRoleDescriptorsMapToBytes(
-                                (Map<String, Object>) metadata.get(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY)
-                            )
-                        );
-                    }
-                    if (metadata.containsKey(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)) {
-                        metadata.put(
-                            AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY,
-                            convertRoleDescriptorsMapToBytes(
-                                (Map<String, Object>) metadata.get(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)
-                            )
-                        );
-                    }
-                }
         }
         return metadata;
     }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java

@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.core.security.authc;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
@@ -414,6 +415,9 @@ public static Authentication randomApiKeyAuthentication(User user, String apiKey
         final HashMap<String, Object> metadata = new HashMap<>();
         metadata.put(AuthenticationField.API_KEY_ID_KEY, apiKeyId);
         metadata.put(AuthenticationField.API_KEY_NAME_KEY, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 16));
+        metadata.put(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY, new BytesArray("{}"));
+        metadata.put(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, new BytesArray("""
+            {"x":{"cluster":["all"],"indices":[{"names":["index*"],"privileges":["all"]}]}}"""));
         return Authentication.newApiKeyAuthentication(AuthenticationResult.success(user, metadata), randomAlphaOfLengthBetween(3, 8))
             .maybeRewriteForOlderVersion(version);
     }