feat(core): Add experimental account storage API (spinnaker#5594)

* feat(core): Add experimental account storage API

This adds an experimental API for storing and loading account
credentials definitions from an external durable store such as a SQL
database. Secrets can be referenced through the existing Kork
SecretEngine API which will be fetched on demand. Initial support is for
Kubernetes accounts given the lack of existing Kubernetes cluster
federation standards compared to other cloud providers, though this API
is made generic to allow for other cloud provider APIs to participate in
this system.

* Combine AccountController with CredentialsController

* Fix account permissions and add comments

* Add alpha annotations

* Use correct annotations for rest APIs

clouddriver-api/src/main/java/com/netflix/spinnaker/clouddriver/security/AccountDefinitionRepository.java

@@ -0,0 +1,140 @@
+/*
+ * Copyright 2021 Apple Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.security;
+
+import com.netflix.spinnaker.credentials.definition.CredentialsDefinition;
+import com.netflix.spinnaker.kork.annotations.Alpha;
+import com.netflix.spinnaker.kork.annotations.NonnullByDefault;
+import java.util.List;
+import javax.annotation.Nullable;
+
+/** Provides CRUD persistence operations for account {@link CredentialsDefinition} instances. */
+@Alpha
+@NonnullByDefault
+public interface AccountDefinitionRepository {
+  /**
+   * Looks up an account definition using the account name.
+   *
+   * @param name account name to look up
+   * @return the configured account definition or null if none is found
+   */
+  @Nullable
+  CredentialsDefinition getByName(String name);
+
+  /**
+   * Lists account definitions for a given account type. This API allows for infinite scrolling
+   * style pagination using the {@code limit} and {@code startingAccountName} parameters.
+   *
+   * @param typeName account type to search for (the value of the @JsonTypeName annotation on the
+   *     corresponding CredentialsDefinition class
+   * @param limit max number of entries to return
+   * @param startingAccountName where to begin results list if specified or start at the beginning
+   *     if null
+   * @return list of stored account definitions matching the given account type name (sorted
+   *     alphabetically)
+   */
+  List<? extends CredentialsDefinition> listByType(
+      String typeName, int limit, @Nullable String startingAccountName);
+
+  /**
+   * Lists account definitions for a given account type. Account types correspond to the value in
+   * the {@code @JsonTypeName} annotation present on the corresponding {@code CredentialsDefinition}
+   * class.
+   *
+   * @param typeName account type to search for
+   * @return list of all stored account definitions matching the given account type name
+   */
+  List<? extends CredentialsDefinition> listByType(String typeName);
+
+  /**
+   * Creates a new account definition using the provided data. Secrets should use {@code
+   * SecretEngine} encrypted URIs (e.g., {@code
+   * encrypted:secrets-manager!r:us-west-2!s:my-account-credentials}) when the underlying storage
+   * provider does not support row-level encryption or equivalent security features. Encrypted URIs
+   * will only be decrypted when loading account definitions, not when storing them. Note that
+   * account definitions correspond to the JSON representation of the underlying {@link
+   * CredentialsDefinition} object along with a JSON type discriminator field with the key
+   * {@code @type} and value of the corresponding {@code @JsonTypeName} annotation.
+   *
+   * @param definition account definition to store as a new account
+   */
+  void create(CredentialsDefinition definition);
+
+  /**
+   * Updates an existing account definition using the provided data. See details in {@link
+   * #create(CredentialsDefinition)} for details on the format.
+   *
+   * @param definition updated account definition to replace an existing account
+   * @see #create(CredentialsDefinition)
+   */
+  void update(CredentialsDefinition definition);
+
+  /**
+   * Deletes an account by name.
+   *
+   * @param name name of account to delete
+   */
+  void delete(String name);
+
+  /**
+   * Looks up the revision history of an account given its name. Keys correspond to ascending
+   * version numbers for each account definition update.
+   *
+   * @param name account name to look up history for
+   * @return history of account updates for the given account name
+   */
+  List<Revision> revisionHistory(String name);
+
+  /**
+   * Provides metadata for an account definition revision when making updates to an account via
+   * {@link AccountDefinitionRepository} APIs.
+   */
+  class Revision {
+    private final int version;
+    private final long timestamp;
+    private final @Nullable CredentialsDefinition account;
+
+    /** Constructs a revision entry with a version and account definition. */
+    public Revision(int version, long timestamp, @Nullable CredentialsDefinition account) {
+      this.version = version;
+      this.timestamp = timestamp;
+      this.account = account;
+    }
+
+    /** Returns the version number of this revision. Versions start at 1 and increase from there. */
+    public int getVersion() {
+      return version;
+    }
+
+    /**
+     * Returns the timestamp (in millis since the epoch) corresponding to when this revision was
+     * made.
+     */
+    public long getTimestamp() {
+      return timestamp;
+    }
+
+    /**
+     * Returns the account definition used in this revision. Returns {@code null} when this revision
+     * corresponds to a deletion.
+     */
+    @Nullable
+    public CredentialsDefinition getAccount() {
+      return account;
+    }
+  }
+}

clouddriver-core/clouddriver-core.gradle

@@ -32,6 +32,7 @@ dependencies {
   implementation "io.spinnaker.kork:kork-web"
   implementation "io.spinnaker.kork:kork-annotations"
   implementation "io.spinnaker.kork:kork-moniker"
+  implementation "io.spinnaker.kork:kork-secrets"
   implementation "com.squareup.okhttp:okhttp"
   implementation "com.squareup.okhttp:okhttp-apache"
   implementation "com.squareup.okhttp:okhttp-urlconnection"
@@ -59,4 +60,5 @@ dependencies {
   testImplementation "org.junit.jupiter:junit-jupiter-engine"
   testImplementation "org.junit.jupiter:junit-jupiter-params"
   testImplementation "org.mockito:mockito-core"
+  testImplementation "org.springframework.boot:spring-boot-starter-test"
 }

clouddriver-core/src/main/java/com/netflix/spinnaker/clouddriver/config/AccountDefinitionConfiguration.java

@@ -0,0 +1,148 @@
+/*
+ * Copyright 2021 Apple Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.config;
+
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.deser.std.StringDeserializer;
+import com.netflix.spinnaker.clouddriver.jackson.AccountDefinitionModule;
+import com.netflix.spinnaker.clouddriver.security.AccountDefinitionMapper;
+import com.netflix.spinnaker.credentials.definition.CredentialsDefinition;
+import com.netflix.spinnaker.kork.secrets.EncryptedSecret;
+import com.netflix.spinnaker.kork.secrets.SecretManager;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import lombok.Data;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.beans.factory.config.BeanDefinition;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.ResourceLoader;
+import org.springframework.core.type.filter.AssignableTypeFilter;
+import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
+import org.springframework.util.ClassUtils;
+
+/**
+ * Provides configuration settings related to managing account credential definitions at runtime.
+ *
+ * @see Properties
+ */
+@Configuration
+@EnableConfigurationProperties(AccountDefinitionConfiguration.Properties.class)
+@Log4j2
+public class AccountDefinitionConfiguration {
+
+  /**
+   * Creates a mapper that can convert between JSON and {@link CredentialsDefinition} classes that
+   * are annotated with {@link JsonTypeName}. Account definition classes are scanned in {@code
+   * com.netflix.spinnaker.clouddriver} and any additional packages configured in {@link
+   * Properties#setAdditionalScanPackages(List)}. Only eligible account definition classes are used
+   * with a fresh {@link com.fasterxml.jackson.databind.ObjectMapper} configured with a custom
+   * string deserializer to transparently load and decrypt {@link EncryptedSecret} strings. These
+   * encrypted secrets must reference a configured {@link
+   * com.netflix.spinnaker.kork.secrets.SecretEngine}.
+   *
+   * @see com.netflix.spinnaker.kork.jackson.ObjectMapperSubtypeConfigurer
+   */
+  @Bean
+  public AccountDefinitionMapper accountDefinitionMapper(
+      Jackson2ObjectMapperBuilder mapperBuilder,
+      SecretManager secretManager,
+      AccountDefinitionModule accountDefinitionModule) {
+    mapperBuilder.deserializers(createSecretDecryptingDeserializer(secretManager));
+    var mapper = mapperBuilder.build();
+    return new AccountDefinitionMapper(mapper, accountDefinitionModule.getTypeMap());
+  }
+
+  @Bean
+  public AccountDefinitionModule accountDefinitionModule(
+      ResourceLoader loader, Properties properties) {
+    return new AccountDefinitionModule(findAccountDefinitionTypes(loader, properties));
+  }
+
+  private static StringDeserializer createSecretDecryptingDeserializer(
+      SecretManager secretManager) {
+    return new StringDeserializer() {
+      @Override
+      public String deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
+        var string = super.deserialize(p, ctxt);
+        if (EncryptedSecret.isEncryptedSecret(string)) {
+          return EncryptedSecret.isEncryptedFile(string)
+              ? secretManager.decryptAsFile(string).toString()
+              : secretManager.decrypt(string);
+        } else {
+          return string;
+        }
+      }
+    };
+  }
+
+  @SuppressWarnings("unchecked")
+  private static Class<? extends CredentialsDefinition>[] findAccountDefinitionTypes(
+      ResourceLoader loader, Properties properties) {
+    var provider = new ClassPathScanningCandidateComponentProvider(false);
+    provider.setResourceLoader(loader);
+    provider.addIncludeFilter(new AssignableTypeFilter(CredentialsDefinition.class));
+    List<String> scanPackages = new ArrayList<>(properties.additionalScanPackages);
+    scanPackages.add(0, "com.netflix.spinnaker.clouddriver");
+    return scanPackages.stream()
+        .flatMap(packageName -> provider.findCandidateComponents(packageName).stream())
+        .map(BeanDefinition::getBeanClassName)
+        .filter(Objects::nonNull)
+        .map(AccountDefinitionConfiguration::loadCredentialsDefinitionType)
+        .filter(type -> type.isAnnotationPresent(JsonTypeName.class))
+        .toArray(Class[]::new);
+  }
+
+  private static Class<? extends CredentialsDefinition> loadCredentialsDefinitionType(
+      String className) {
+    try {
+      return ClassUtils.forName(className, null).asSubclass(CredentialsDefinition.class);
+    } catch (ClassNotFoundException e) {
+      throw new IllegalStateException(
+          String.format("Unable to load CredentialsDefinition type `%s`", className), e);
+    }
+  }
+
+  @ConfigurationProperties("account.storage")
+  @ConditionalOnProperty("account.storage.enabled")
+  @Data
+  public static class Properties {
+    /**
+     * Indicates whether to enable durable storage for account definitions. When enabled with an
+     * implementation of {@link
+     * com.netflix.spinnaker.clouddriver.security.AccountDefinitionRepository}, account definitions
+     * can be stored and retrieved by a durable storage provider.
+     */
+    private boolean enabled;
+
+    /**
+     * Additional packages to scan for {@link
+     * com.netflix.spinnaker.credentials.definition.CredentialsDefinition} implementation classes
+     * that may be annotated with {@link JsonTypeName} to participate in the account management
+     * system. These packages are in addition to the default scan package from within Clouddriver.
+     */
+    private List<String> additionalScanPackages = List.of();
+  }
+}

clouddriver-core/src/main/java/com/netflix/spinnaker/clouddriver/jackson/AccountDefinitionModule.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2021 Apple Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.jackson;
+
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.netflix.spinnaker.clouddriver.jackson.mixins.CredentialsDefinitionMixin;
+import com.netflix.spinnaker.clouddriver.security.AccountDefinitionMapper;
+import com.netflix.spinnaker.credentials.definition.CredentialsDefinition;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Jackson module to register {@link CredentialsDefinition} type discriminators for the provided
+ * account definition types.
+ */
+public class AccountDefinitionModule extends SimpleModule {
+
+  private final Class<? extends CredentialsDefinition>[] accountDefinitionTypes;
+
+  public AccountDefinitionModule(Class<? extends CredentialsDefinition>[] accountDefinitionTypes) {
+    super("Clouddriver Account Definition API");
+    this.accountDefinitionTypes = accountDefinitionTypes;
+  }
+
+  @Override
+  public void setupModule(SetupContext context) {
+    super.setupModule(context);
+    context.setMixInAnnotations(CredentialsDefinition.class, CredentialsDefinitionMixin.class);
+    context.registerSubtypes(accountDefinitionTypes);
+  }
+
+  public Map<String, Class<? extends CredentialsDefinition>> getTypeMap() {
+    var typeMap =
+        new HashMap<String, Class<? extends CredentialsDefinition>>(accountDefinitionTypes.length);
+    for (Class<? extends CredentialsDefinition> type : accountDefinitionTypes) {
+      typeMap.put(AccountDefinitionMapper.getJsonTypeName(type), type);
+    }
+    return typeMap;
+  }
+}

clouddriver-core/src/main/java/com/netflix/spinnaker/clouddriver/jackson/mixins/CredentialsDefinitionMixin.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2021 Apple Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.jackson.mixins;
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+/**
+ * Jackson mixin to add a polymorphic type name value. When a {@link
+ * com.netflix.spinnaker.credentials.definition.CredentialsDefinition} implementation class is
+ * annotated with {@link com.fasterxml.jackson.annotation.JsonTypeName}, then the value of that
+ * annotation is used as the {@code @type} property value when marshalling and unmarshalling
+ * CredentialsDefinition classes. It is recommended that the corresponding cloud provider name for
+ * the credentials be used here.
+ */
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME)
+public interface CredentialsDefinitionMixin {}