feat: Adding ability to add repos.

Author: chrsdietz

package-lock.json

@@ -26,7 +26,7 @@
     "@types/request-promise-native": "1.0.15",
     "@types/sinon": "7.0.3",
     "@types/sinon-chai": "3.2.2",
-    "@xapp/serverless-plugin-type-definitions": "0.1.20",
+    "@xapp/serverless-plugin-type-definitions": "0.1.22",
     "chai": "4.2.0",
     "mocha": "5.2.0",
     "nyc": "13.1.0",

package.json

@@ -1,5 +1,12 @@
 import { CloudFormation } from "aws-sdk";
+import FeatureNotSupportedError from "./FeatureNotSupportedError";
+import ResourceNotFoundError from "./ResourceNotFoundError";
 
+/**
+ * Returns the value of the CloudFormation exported item.
+ * @param cf The CloudFormation object to use.
+ * @param exportName The name of the exported item.
+ */
 export function findCloudformationExport(cf: CloudFormation, exportName: string): Promise<string> {
     const find = (NextToken?: string): Promise<string> => {
         return cf.listExports({ NextToken }).promise()
@@ -10,3 +17,94 @@ export function findCloudformationExport(cf: CloudFormation, exportName: string)
     };
     return find();
 }
+
+export interface ConfigObject {
+    [key: string]: any | CloudFormationObject;
+}
+
+/**
+ * Will recursively filter through a configuration object and look for CloudFormation items to parse
+ * like "Ref:" and "Fn::*" functions.
+ * @param cf
+ * @param stackName
+ * @param configObject
+ */
+export async function parseConfigObject(cf: CloudFormation, stackName: string, configObject: any | ConfigObject) {
+    if (isNotDefined(configObject)) {
+        return configObject;
+    }
+    if (typeof configObject !== "object") {
+        return configObject;
+    }
+    const keys = Object.keys(configObject);
+    const newObject: ConfigObject = Array.isArray(configObject) ? configObject.slice() : { ...configObject };
+    for (const key of keys) {
+        const item = configObject[key];
+        if (!isNotDefined(item)) {
+            if (Array.isArray(item)) {
+                newObject[key] = await parseConfigObject(cf, stackName, item);
+            } else if (typeof item === "object") {
+                const internalObjectKeys = Object.keys(item);
+                if (internalObjectKeys.length === 1 && cloudFormationObjectKeys.indexOf(internalObjectKeys[0] as keyof CloudFormationObject) >= 0) {
+                    // Possibly a Cloudformation object like Ref or Fn::*
+                    newObject[key] = await retrieveCloudFormationValue(cf, stackName, item);
+                } else {
+                    newObject[key] = await parseConfigObject(cf, stackName, item);
+                }
+            } // Else it's already what it should be.
+        }
+    }
+    return newObject;
+}
+
+const cloudFormationObjectKeys: (keyof CloudFormationObject)[] = Object.keys({
+    Ref: ""
+} as Required<CloudFormationObject>) as (keyof CloudFormationObject)[];
+
+export interface CloudFormationObject {
+    Ref?: string;
+}
+
+export function retrieveCloudFormationValue(cf: CloudFormation, stackName: string, value: CloudFormationObject): Promise<boolean | number | string> {
+    if (isNotDefined(value)) {
+        return Promise.resolve(value as string); // It's undefined but Typescript doesn't know or care so we're just going to lie.
+    }
+    if (typeof value !== "object" || Object.keys(value).length !== 1) {
+        return Promise.reject(new Error("Value is not a CloudFormation parsable object."));
+    }
+    if (!isNotDefined(value.Ref)) {
+        return findPhysicalID(cf, stackName, value.Ref);
+    }
+    return Promise.reject(new FeatureNotSupportedError(`CloudFormation value ${Object.keys(value)[0]} not currently supported.`));
+}
+
+/**
+ * Returns the *physical ID* of a resource in a stack from the given ref.
+ *
+ * Throws an error if the ID is not found at the stack.
+ * @param cf
+ * @param stackName
+ * @param ref
+ */
+export function findPhysicalID(cf: CloudFormation, stackName: string, ref: string): Promise<string> {
+    const find = (NextToken?: string): Promise<string> => {
+        return cf.listStackResources({ StackName: stackName, NextToken }).promise()
+            .then((results): string | Promise<string> => {
+                const item = results.StackResourceSummaries.find(resource => resource.LogicalResourceId === ref);
+                return (item) ? item.PhysicalResourceId : (results.NextToken) ? find(results.NextToken) : undefined;
+            });
+    };
+    return find()
+        .then((physicalId) => {
+            if (isNotDefined(physicalId)) {
+                throw new ResourceNotFoundError(`Physical ID not found for ref "${ref}" in stack "${stackName}".`);
+            }
+            return physicalId;
+        });
+}
+
+function isNotDefined(item: any) {
+    // tslint:disable:no-null-keyword Checking for null with double equals checks both undefined and null
+    return item == null;
+    // tslint:enable:no-null-keyword
+}

src/AwsUtils.ts

@@ -11,9 +11,27 @@ export interface Template {
 export type S3RepositoryType = "s3";
 
 export interface S3RepositorySettings {
+    /**
+     * The s3 bucket to send backups to.
+     */
     bucket: string;
+    /**
+     * The region that the s3 bucket is in.
+     */
     region: string;
-    role_arn: string;
+    /**
+     * The ARN that Elasticsearch will assume to send items.
+     * Either this or `role_name` must be set.
+     */
+    role_arn?: string;
+    /**
+     * The ARN that Elasticsearch will assume to send items.
+     * Either this or `role_arn` must be set.
+     */
+    role_name?: string;
+    /**
+     * Whether or not the repository is encrypted on server.
+     */
     server_side_encryption?: boolean;
 }
 
@@ -55,11 +73,6 @@ export interface PluginConfig {
      * The repositories to be set for Elasticsearch
      */
     "repositories"?: Repository[];
-
-    /**
-     * The AWS Profile credentials that is to be used to send information to the Elasticsearch server.
-     */
-    "aws-profile"?: string;
 }
 
 export default PluginConfig;

src/Config.ts

@@ -0,0 +1,7 @@
+export class FeatureNotSupported extends Error {
+    constructor(msg: string) {
+        super(msg);
+    }
+}
+
+export default FeatureNotSupported;

src/FeatureNotSupportedError.ts

@@ -1,11 +1,12 @@
-import { CLI, Hooks, Serverless, ServerlessPlugin} from "@xapp/serverless-plugin-type-definitions";
-import { CloudFormation, SharedIniFileCredentials } from "aws-sdk";
+import { CLI, Hooks, Serverless, ServerlessPlugin, ServerlessProvider } from "@xapp/serverless-plugin-type-definitions";
+import { CloudFormation, config as AWSConfig, SharedIniFileCredentials, STS } from "aws-sdk";
 import * as Path from "path";
 import { AWSOptions } from "request";
 import * as Request from "request-promise-native";
-import * as AwsUtils from "./AwsUtils";
-import Config, { Index, Repository, Template } from "./Config";
-import * as ServerlessUtils from "./ServerlessObjUtils";
+import { findCloudformationExport, parseConfigObject } from "./AwsUtils";
+import Config, { Index, Template } from "./Config";
+import { getProfile, getProviderName, getRegion, getStackName } from "./ServerlessObjUtils";
+import { setupRepo } from "./SetupRepo";
 
 interface Custom {
     elasticsearch?: Config;
@@ -15,27 +16,26 @@ class Plugin implements ServerlessPlugin {
 
     private serverless: Serverless<Custom>;
     private cli: CLI;
-    private config: Config;
     hooks: Hooks;
 
     constructor(serverless: Serverless<Custom>, context: any) {
         this.serverless = serverless;
         this.cli = serverless.cli;
 
         this.hooks = {
-            "before:aws:deploy:deploy:updateStack": this.create.bind(this),
+            "before:aws:deploy:deploy:updateStack": this.validate.bind(this),
             "after:aws:deploy:deploy:updateStack": this.setupElasticCache.bind(this)
         };
     }
 
     /**
      * Creates the plugin with the fully parsed Serverless object.
      */
-    private async create() {
+    private async validate() {
         const custom = this.serverless.service.custom || {};
-        this.config = custom.elasticsearch || {};
+        const config = custom.elasticsearch || {};
 
-        if (!this.config.endpoint && !this.config["cf-endpoint"]) {
+        if (!config.endpoint && !config["cf-endpoint"]) {
             throw new Error("Elasticsearch endpoint not specified.");
         }
     }
@@ -44,42 +44,67 @@ class Plugin implements ServerlessPlugin {
      * Sends the mapping information to elasticsearch.
      */
     private async setupElasticCache() {
-        let domain = this.config.endpoint;
-        if (this.config["cf-endpoint"]) {
-            const cloudFormation = new CloudFormation({
-                region: ServerlessUtils.getRegion(this.serverless),
-                credentials: new SharedIniFileCredentials({
-                    profile: this.config["aws-profile"] || ServerlessUtils.getProfile(this.serverless)
-                })
-            });
-            domain = await AwsUtils.findCloudformationExport(cloudFormation, this.config["cf-endpoint"]);
-            if (!domain) {
-                throw new Error("Endpoint not found at cloudformation export.");
-            }
-        }
-
-        const endpoint = domain.startsWith("http") ? domain : `https://${domain}`;
+        const serviceName = await getProviderName(this.serverless);
+        const profile = getProfile(this.serverless);
+        const region = getRegion(this.serverless);
 
         const requestOptions: Partial<Request.Options> = {};
-        if (this.config["aws-profile"]) {
-            const sharedIni = new SharedIniFileCredentials({ profile: this.config["aws-profile"] });
+        if (serviceName === "aws") {
+            AWSConfig.credentials = new SharedIniFileCredentials({ profile });
+            AWSConfig.region = region;
             requestOptions.aws = {
-                key: sharedIni.accessKeyId,
-                secret: sharedIni.secretAccessKey,
+                key: AWSConfig.credentials.accessKeyId,
+                secret: AWSConfig.credentials.secretAccessKey,
                 sign_version: 4
             } as AWSOptions; // The typings are wrong.  It need to include "key" and "sign_version"
         }
 
+        const config = await parseConfig(this.serverless);
+        const endpoint = config.endpoint.startsWith("http") ? config.endpoint : `https://${config.endpoint}`;
+
         this.cli.log("Setting up templates...");
-        await setupTemplates(endpoint, this.config.templates, requestOptions);
+        await setupTemplates(endpoint, config.templates, requestOptions);
         this.cli.log("Setting up indices...");
-        await setupIndices(endpoint, this.config.indices, requestOptions);
+        await setupIndices(endpoint, config.indices, requestOptions);
         this.cli.log("Setting up repositories...");
-        await setupRepo(endpoint, this.config.repositories, requestOptions);
+        await setupRepo({
+            baseUrl: endpoint,
+            sts: new STS(),
+            repos: config.repositories,
+            requestOptions
+        });
         this.cli.log("Elasticsearch setup complete.");
     }
 }
 
+/**
+ * Parses the config object so all attributes are usable values.
+ *
+ * If the user has defined "cf-Endpoint" then the correct value will be moved to "endpoint".
+ *
+ * @param serverless
+ */
+async function parseConfig(serverless: Serverless<Custom>): Promise<Config> {
+    const provider = serverless.service.provider || {} as Partial<ServerlessProvider>;
+    const custom = serverless.service.custom || {};
+    let config = custom.elasticsearch || {} as Config;
+
+    if (provider.name === "aws" || config["cf-endpoint"]) {
+        const cloudFormation = new CloudFormation();
+
+        config = await parseConfigObject(cloudFormation, getStackName(serverless), config);
+
+        if (config["cf-endpoint"]) {
+            config.endpoint = await findCloudformationExport(cloudFormation, config["cf-endpoint"]);
+            if (!config.endpoint) {
+                throw new Error("Endpoint not found at cloudformation export.");
+            }
+        }
+    }
+
+    return config;
+}
+
 /**
  * Sets up all the indices in the given object.
  * @param baseUrl The elasticsearch URL
@@ -132,31 +157,6 @@ function validateTemplate(template: Template) {
     }
 }
 
-/**
- * Sets up all the repos.
- * @param baseUrl
- * @param repo
- */
-function setupRepo(baseUrl: string, repos: Repository[] = [], requestOptions: Partial<Request.Options>) {
-    const setupPromises: PromiseLike<Request.FullResponse>[] = repos.map((repo) => {
-        validateRepo(repo);
-        const { name, type, settings } = repo;
-        const url = `${baseUrl}/_snapshot/${name}`;
-        return esPut(url, { type, settings }, requestOptions);
-    });
-    return Promise.all(setupPromises);
-}
-
-function validateRepo(repo: Repository) {
-    if (!repo.name) {
-        throw new Error("Repo does not have a name.");
-    }
-    if (!repo.type) {
-        throw new Error("Repo does not have a type.");
-    }
-    // The settings will be validated by Elasticsearch.
-}
-
 function esPut(url: string, settings: object, requestOpts?: Partial<Request.Options>) {
     const headers = {
         "Content-Type": "application/json",

src/Plugin.ts

@@ -0,0 +1,7 @@
+export class ResourceNotFound extends Error {
+    constructor(msg: string) {
+        super(msg);
+    }
+}
+
+export default ResourceNotFound;