Clearer separation of dirname (#15)

Fail when no success

* Autofix: TypeScript header

[atomist:generated] [atomist:autofix=typescript_header]
* Clearer separation of dirname
* Move test to own file
* Autofix: tslint

[atomist:generated] [atomist:autofix=tslint]
* Autofix: TypeScript imports

[atomist:generated] [atomist:autofix=typescript_imports]
* Autofix: TypeScript header

[atomist:generated] [atomist:autofix=typescript_header]
* If no files are uploaded and warnings happened, fail the goal
* Nevermind, it doesn't seem to do anything
* Only increment file count if the upload succeeded
* Break up giant file

index.ts

@@ -15,3 +15,4 @@
  */
 
 export { PublishToS3 } from "./lib/publishToS3";
+export { PublishToS3Options } from "./lib/options";

lib/deleteS3.ts

@@ -0,0 +1,107 @@
+/*
+ * Copyright © 2019 Atomist, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ProgressLog } from "@atomist/sdm";
+import { S3 } from "aws-sdk";
+import { PublishToS3Options } from "./options";
+
+type QuantityDeleted = number;
+type SuccessfullyPushedKey = string;
+type Warning = string;
+
+export async function deleteKeys(
+    s3: S3,
+    log: ProgressLog,
+    params: PublishToS3Options,
+    keysToDelete: S3.ObjectIdentifier[]): Promise<[QuantityDeleted, Warning[]]> {
+    const { bucketName } = params;
+    let deleted = 0;
+    const warnings: Warning[] = [];
+    const maxItems = 1000;
+    for (let i = 0; i < keysToDelete.length; i += maxItems) {
+        const deleteNow = keysToDelete.slice(i, i + maxItems);
+        try {
+            const deleteObjectsResult = await s3.deleteObjects({
+                Bucket: bucketName,
+                Delete: {
+                    Objects: deleteNow,
+                },
+            }).promise();
+            deleted += deleteObjectsResult.Deleted.length;
+            const deletedString = deleteObjectsResult.Deleted.map(o => o.Key).join(",");
+            log.write(`Deleted objects (${deletedString}) in 's3://${bucketName}'`);
+            if (deleteObjectsResult.Errors && deleteObjectsResult.Errors.length > 0) {
+                deleteObjectsResult.Errors.forEach(e => {
+                    const msg = `Error deleting object '${e.Key}': ${e.Message}`;
+                    log.write(msg);
+                    warnings.push(msg);
+                });
+            }
+        } catch (e) {
+            const keysString = deleteNow.map(o => o.Key).join(",");
+            const msg = `Failed to delete objects (${keysString}) in 's3://${bucketName}': ${e.message}`;
+            log.write(msg);
+            warnings.push(msg);
+            break;
+        }
+    }
+    return [deleted, warnings];
+}
+
+export async function gatherKeysToDelete(
+    s3: S3,
+    log: ProgressLog,
+    keysToKeep: SuccessfullyPushedKey[],
+    params: PublishToS3Options): Promise<[S3.ObjectIdentifier[], Warning[]]> {
+
+    const { bucketName } = params;
+    const keysToDelete: S3.ObjectIdentifier[] = [];
+    const warnings: Warning[] = [];
+
+    let listObjectsResponse: S3.ListObjectsV2Output = {
+        IsTruncated: true,
+        NextContinuationToken: undefined,
+    };
+    const maxItems = 1000;
+    while (listObjectsResponse.IsTruncated) {
+        try {
+            listObjectsResponse = await s3.listObjectsV2({
+                Bucket: bucketName,
+                MaxKeys: maxItems,
+                ContinuationToken: listObjectsResponse.NextContinuationToken,
+            }).promise();
+            keysToDelete.push(...filterKeys(keysToKeep, listObjectsResponse.Contents));
+        } catch (e) {
+            const msg = `Failed to list objects in 's3://${bucketName}': ${e.message}`;
+            log.write(msg);
+            warnings.push(msg);
+            break;
+        }
+    }
+
+    return [keysToDelete, warnings];
+}
+
+/**
+ * Remove objects that either have no key or match a key in `keys`.
+ *
+ * @param keys Keys that should be removed from `objects`
+ * @param objects Array to filter
+ * @return Array of object identifiers
+ */
+export function filterKeys(keys: string[], objects: S3.Object[]): S3.ObjectIdentifier[] {
+    return objects.filter(o => o.Key && !keys.includes(o.Key)).map(o => ({ Key: o.Key }));
+}

lib/options.ts

@@ -0,0 +1,86 @@
+/*
+ * Copyright © 2019 Atomist, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GoalInvocation } from "@atomist/sdm";
+import { GlobPatterns } from "./publishToS3";
+
+/**
+ * Specify how to publish a project's output to S3.
+ */
+export interface PublishToS3Options {
+
+    /**
+     * Name of this publish operation. Make it unique per push.
+     */
+    uniqueName: string;
+
+    /**
+     * Name of the bucket. For example: docs.atomist.com
+     */
+    bucketName: string;
+
+    /**
+     * AWS region. For example: us-west-2
+     * This is used to construct a URL that the goal will link to
+     */
+    region: string;
+
+    /**
+     * Select the files to publish. This is an array of glob patterns.
+     * For example: [ "target/**\/*", "index.html" ],
+     */
+    filesToPublish: GlobPatterns;
+
+    /**
+     * Function from a file path within this project to a key (path
+     * within the bucket) where it belongs on S3.  You can use the
+     * invocation to see the SHA and branch, in case you want to
+     * upload to a branch- or commit-specific place inside the bucket.
+     */
+    pathTranslation?: (filePath: string, inv: GoalInvocation) => string;
+
+    /**
+     * The file or path within the project represents the root of the
+     * uploaded site.  This is a path within the project.  It will be
+     * passed to pathTranslation to get the path within the bucket
+     * when generating the link to the completed goal.  If it is not
+     * provided, no externalUrl is provided in the goal result.
+     */
+    pathToIndex?: string;
+
+    /**
+     * If true, delete objects from S3 bucket that do not map to files
+     * in the repository being copied to the bucket.  If false, files
+     * from the repository are copied to the bucket but no existing
+     * objects in the bucket are deleted.
+     */
+    sync?: boolean;
+
+    /**
+     * If set, look for hidden files with this extension (otherwise
+     * matching the names of files to be uploaded) for additional
+     * parameter properties to supply to the argument of S3.putObject.
+     *
+     * For example, if a file "X" is being uploaded and the value of
+     * `paramsExt` is ".s3params" and there is a file ".X.s3params" in
+     * the same directory, the contents of the ".X.s3params" file are
+     * parsed as JSON and merged into the parameters used as the
+     * argument to the S3.putObject function with the former taking
+     * precedence, i.e., the values in ".X.s3params" take override the
+     * default property values.
+     */
+    paramsExt?: string;
+}