feat(core): Add experimental account storage API

clouddriver-api/src/main/java/com/netflix/spinnaker/clouddriver/security/AccountDefinitionRepository.java

@@ -0,0 +1,140 @@
+/*
+ * Copyright 2021 Apple Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.security;
+
+import com.netflix.spinnaker.credentials.definition.CredentialsDefinition;
+import com.netflix.spinnaker.kork.annotations.Alpha;
+import com.netflix.spinnaker.kork.annotations.NonnullByDefault;
+import java.util.List;
+import javax.annotation.Nullable;
+
+/** Provides CRUD persistence operations for account {@link CredentialsDefinition} instances. */
+@Alpha
+@NonnullByDefault
+public interface AccountDefinitionRepository {
+  /**
+   * Looks up an account definition using the account name.
+   *
+   * @param name account name to look up
+   * @return the configured account definition or null if none is found
+   */
+  @Nullable
+  CredentialsDefinition getByName(String name);
+
+  /**
+   * Lists account definitions for a given account type. This API allows for infinite scrolling
+   * style pagination using the {@code limit} and {@code startingAccountName} parameters.
+   *
+   * @param typeName account type to search for (the value of the @JsonTypeName annotation on the
+   *     corresponding CredentialsDefinition class
+   * @param limit max number of entries to return
+   * @param startingAccountName where to begin results list if specified or start at the beginning
+   *     if null
+   * @return list of stored account definitions matching the given account type name (sorted
+   *     alphabetically)
+   */
+  List<? extends CredentialsDefinition> listByType(
+      String typeName, int limit, @Nullable String startingAccountName);
+
+  /**
+   * Lists account definitions for a given account type. Account types correspond to the value in
+   * the {@code @JsonTypeName} annotation present on the corresponding {@code CredentialsDefinition}
+   * class.
+   *
+   * @param typeName account type to search for
+   * @return list of all stored account definitions matching the given account type name
+   */
+  List<? extends CredentialsDefinition> listByType(String typeName);
+
+  /**
+   * Creates a new account definition using the provided data. Secrets should use {@code
+   * SecretEngine} encrypted URIs (e.g., {@code
+   * encrypted:secrets-manager!r:us-west-2!s:my-account-credentials}) when the underlying storage
+   * provider does not support row-level encryption or equivalent security features. Encrypted URIs
+   * will only be decrypted when loading account definitions, not when storing them. Note that
+   * account definitions correspond to the JSON representation of the underlying {@link
+   * CredentialsDefinition} object along with a JSON type discriminator field with the key
+   * {@code @type} and value of the corresponding {@code @JsonTypeName} annotation.
+   *
+   * @param definition account definition to store as a new account
+   */
+  void create(CredentialsDefinition definition);
+
+  /**
+   * Updates an existing account definition using the provided data. See details in {@link
+   * #create(CredentialsDefinition)} for details on the format.
+   *
+   * @param definition updated account definition to replace an existing account
+   * @see #create(CredentialsDefinition)
+   */
+  void update(CredentialsDefinition definition);
+
+  /**
+   * Deletes an account by name.
+   *
+   * @param name name of account to delete
+   */
+  void delete(String name);
+
+  /**
+   * Looks up the revision history of an account given its name. Keys correspond to ascending
+   * version numbers for each account definition update.
+   *
+   * @param name account name to look up history for
+   * @return history of account updates for the given account name
+   */
+  List<Revision> revisionHistory(String name);
+
+  /**
+   * Provides metadata for an account definition revision when making updates to an account via
+   * {@link AccountDefinitionRepository} APIs.
+   */
+  class Revision {
+    private final int version;
+    private final long timestamp;
+    private final @Nullable CredentialsDefinition account;
+
+    /** Constructs a revision entry with a version and account definition. */
+    public Revision(int version, long timestamp, @Nullable CredentialsDefinition account) {
+      this.version = version;
+      this.timestamp = timestamp;
+      this.account = account;
+    }
+
+    /** Returns the version number of this revision. Versions start at 1 and increase from there. */
+    public int getVersion() {
+      return version;
+    }
+
+    /**
+     * Returns the timestamp (in millis since the epoch) corresponding to when this revision was
+     * made.
+     */
+    public long getTimestamp() {
+      return timestamp;
+    }
+
+    /**
+     * Returns the account definition used in this revision. Returns {@code null} when this revision
+     * corresponds to a deletion.
+     */
+    @Nullable
+    public CredentialsDefinition getAccount() {
+      return account;
+    }
+  }
+}

clouddriver-core/clouddriver-core.gradle

@@ -31,6 +31,7 @@ dependencies {
   implementation "io.spinnaker.kork:kork-web"
   implementation "io.spinnaker.kork:kork-annotations"
   implementation "io.spinnaker.kork:kork-moniker"
+  implementation "io.spinnaker.kork:kork-secrets"
   implementation "com.squareup.okhttp:okhttp"
   implementation "com.squareup.okhttp:okhttp-apache"
   implementation "com.squareup.okhttp:okhttp-urlconnection"
@@ -58,4 +59,5 @@ dependencies {
   testImplementation "org.junit.jupiter:junit-jupiter-engine"
   testImplementation "org.junit.jupiter:junit-jupiter-params"
   testImplementation "org.mockito:mockito-core"
+  testImplementation "org.springframework.boot:spring-boot-starter-test"
 }

clouddriver-core/src/main/java/com/netflix/spinnaker/clouddriver/config/AccountDefinitionConfiguration.java

@@ -0,0 +1,148 @@
+/*
+ * Copyright 2021 Apple Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.config;
+
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.deser.std.StringDeserializer;
+import com.netflix.spinnaker.clouddriver.jackson.AccountDefinitionModule;
+import com.netflix.spinnaker.clouddriver.security.AccountDefinitionMapper;
+import com.netflix.spinnaker.credentials.definition.CredentialsDefinition;
+import com.netflix.spinnaker.kork.secrets.EncryptedSecret;
+import com.netflix.spinnaker.kork.secrets.SecretManager;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import lombok.Data;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.beans.factory.config.BeanDefinition;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.ResourceLoader;
+import org.springframework.core.type.filter.AssignableTypeFilter;
+import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
+import org.springframework.util.ClassUtils;
+
+/**
+ * Provides configuration settings related to managing account credential definitions at runtime.
+ *
+ * @see Properties
+ */
+@Configuration
+@EnableConfigurationProperties(AccountDefinitionConfiguration.Properties.class)
+@Log4j2
+public class AccountDefinitionConfiguration {
+
+  /**
+   * Creates a mapper that can convert between JSON and {@link CredentialsDefinition} classes that
+   * are annotated with {@link JsonTypeName}. Account definition classes are scanned in {@code
+   * com.netflix.spinnaker.clouddriver} and any additional packages configured in {@link
+   * Properties#setAdditionalScanPackages(List)}. Only eligible account definition classes are used
+   * with a fresh {@link com.fasterxml.jackson.databind.ObjectMapper} configured with a custom
+   * string deserializer to transparently load and decrypt {@link EncryptedSecret} strings. These
+   * encrypted secrets must reference a configured {@link
+   * com.netflix.spinnaker.kork.secrets.SecretEngine}.
+   *
+   * @see com.netflix.spinnaker.kork.jackson.ObjectMapperSubtypeConfigurer
+   */
+  @Bean
+  public AccountDefinitionMapper accountDefinitionMapper(
+      Jackson2ObjectMapperBuilder mapperBuilder,
+      SecretManager secretManager,
+      AccountDefinitionModule accountDefinitionModule) {
+    mapperBuilder.deserializers(createSecretDecryptingDeserializer(secretManager));
+    var mapper = mapperBuilder.build();
+    return new AccountDefinitionMapper(mapper, accountDefinitionModule.getTypeMap());
+  }
+
+  @Bean
+  public AccountDefinitionModule accountDefinitionModule(
+      ResourceLoader loader, Properties properties) {
+    return new AccountDefinitionModule(findAccountDefinitionTypes(loader, properties));
+  }
+
+  private static StringDeserializer createSecretDecryptingDeserializer(
+      SecretManager secretManager) {
+    return new StringDeserializer() {
+      @Override
+      public String deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
+        var string = super.deserialize(p, ctxt);
+        if (EncryptedSecret.isEncryptedSecret(string)) {
+          return EncryptedSecret.isEncryptedFile(string)
+              ? secretManager.decryptAsFile(string).toString()
+              : secretManager.decrypt(string);
+        } else {
+          return string;
+        }
+      }
+    };
+  }
+
+  @SuppressWarnings("unchecked")
+  private static Class<? extends CredentialsDefinition>[] findAccountDefinitionTypes(
+      ResourceLoader loader, Properties properties) {
+    var provider = new ClassPathScanningCandidateComponentProvider(false);
+    provider.setResourceLoader(loader);
+    provider.addIncludeFilter(new AssignableTypeFilter(CredentialsDefinition.class));
+    List<String> scanPackages = new ArrayList<>(properties.additionalScanPackages);
+    scanPackages.add(0, "com.netflix.spinnaker.clouddriver");
+    return scanPackages.stream()
+        .flatMap(packageName -> provider.findCandidateComponents(packageName).stream())
+        .map(BeanDefinition::getBeanClassName)
+        .filter(Objects::nonNull)
+        .map(AccountDefinitionConfiguration::loadCredentialsDefinitionType)
+        .filter(type -> type.isAnnotationPresent(JsonTypeName.class))
+        .toArray(Class[]::new);
+  }
+
+  private static Class<? extends CredentialsDefinition> loadCredentialsDefinitionType(
+      String className) {
+    try {
+      return ClassUtils.forName(className, null).asSubclass(CredentialsDefinition.class);
+    } catch (ClassNotFoundException e) {
+      throw new IllegalStateException(
+          String.format("Unable to load CredentialsDefinition type `%s`", className), e);
+    }
+  }
+
+  @ConfigurationProperties("account.storage")
+  @ConditionalOnProperty("account.storage.enabled")
+  @Data
+  public static class Properties {
+    /**
+     * Indicates whether to enable durable storage for account definitions. When enabled with an
+     * implementation of {@link
+     * com.netflix.spinnaker.clouddriver.security.AccountDefinitionRepository}, account definitions
+     * can be stored and retrieved by a durable storage provider.
+     */
+    private boolean enabled;
+
+    /**
+     * Additional packages to scan for {@link
+     * com.netflix.spinnaker.credentials.definition.CredentialsDefinition} implementation classes
+     * that may be annotated with {@link JsonTypeName} to participate in the account management
+     * system. These packages are in addition to the default scan package from within Clouddriver.
+     */
+    private List<String> additionalScanPackages = List.of();
+  }
+}

clouddriver-core/src/main/java/com/netflix/spinnaker/clouddriver/jackson/AccountDefinitionModule.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2021 Apple Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.jackson;
+
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.netflix.spinnaker.clouddriver.jackson.mixins.CredentialsDefinitionMixin;
+import com.netflix.spinnaker.clouddriver.security.AccountDefinitionMapper;
+import com.netflix.spinnaker.credentials.definition.CredentialsDefinition;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Jackson module to register {@link CredentialsDefinition} type discriminators for the provided
+ * account definition types.
+ */
+public class AccountDefinitionModule extends SimpleModule {
+
+  private final Class<? extends CredentialsDefinition>[] accountDefinitionTypes;
+
+  public AccountDefinitionModule(Class<? extends CredentialsDefinition>[] accountDefinitionTypes) {
+    super("Clouddriver Account Definition API");
+    this.accountDefinitionTypes = accountDefinitionTypes;
+  }
+
+  @Override
+  public void setupModule(SetupContext context) {
+    super.setupModule(context);
+    context.setMixInAnnotations(CredentialsDefinition.class, CredentialsDefinitionMixin.class);
+    context.registerSubtypes(accountDefinitionTypes);
+  }
+
+  public Map<String, Class<? extends CredentialsDefinition>> getTypeMap() {
+    var typeMap =
+        new HashMap<String, Class<? extends CredentialsDefinition>>(accountDefinitionTypes.length);
+    for (Class<? extends CredentialsDefinition> type : accountDefinitionTypes) {
+      typeMap.put(AccountDefinitionMapper.getJsonTypeName(type), type);
+    }
+    return typeMap;
+  }
+}

clouddriver-core/src/main/java/com/netflix/spinnaker/clouddriver/jackson/mixins/CredentialsDefinitionMixin.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2021 Apple Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.jackson.mixins;
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+/**
+ * Jackson mixin to add a polymorphic type name value. When a {@link
+ * com.netflix.spinnaker.credentials.definition.CredentialsDefinition} implementation class is
+ * annotated with {@link com.fasterxml.jackson.annotation.JsonTypeName}, then the value of that
+ * annotation is used as the {@code @type} property value when marshalling and unmarshalling
+ * CredentialsDefinition classes. It is recommended that the corresponding cloud provider name for
+ * the credentials be used here.
+ */
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME)
+public interface CredentialsDefinitionMixin {}