Feat: bundle workflow changes

packages/netlify-cms-backend-bitbucket/package.json

@@ -22,7 +22,8 @@
   "dependencies": {
     "common-tags": "^1.8.0",
     "js-base64": "^2.5.1",
-    "semaphore": "^1.1.0"
+    "semaphore": "^1.1.0",
+    "what-the-diff": "^0.6.0"
   },
   "peerDependencies": {
     "@emotion/core": "^10.0.9",

packages/netlify-cms-backend-bitbucket/src/API.ts

@@ -1,4 +1,4 @@
-import { flow, get } from 'lodash';
+import { flow, get, has } from 'lodash';
 import {
   localForage,
   unsentRequest,
@@ -23,8 +23,12 @@ import {
   PreviewState,
   FetchError,
   parseContentKey,
+  COMBINE_PR_TITLE,
+  isBinaryFile,
+  isCombineKey,
 } from 'netlify-cms-lib-util';
 import { oneLine } from 'common-tags';
+import { parse } from 'what-the-diff';
 
 interface Config {
   apiRoot?: string;
@@ -447,18 +451,54 @@ export default class API {
     await this.addPullRequestComment(pullRequest, statusToLabel(status));
   }
 
+  async updatePullRequestTargetBranch(pullRequest: BitBucketPullRequest, branch: string) {
+    return this.requestJSON({
+      method: 'PUT',
+      url: `${this.repoURL}/pullrequests/${pullRequest.id}`,
+      headers: { 'Content-Type': APPLICATION_JSON },
+      body: JSON.stringify({
+        title: pullRequest.title,
+        destination: {
+          branch: {
+            name: branch,
+          },
+        },
+      }),
+    });
+  }
+
+  createBranch(branch: string, ref: string) {
+    return this.requestJSON({
+      method: 'POST',
+      url: `${this.repoURL}/refs/branches`,
+      headers: { 'Content-Type': APPLICATION_JSON },
+      body: JSON.stringify({
+        name: branch,
+        target: {
+          hash: ref,
+        },
+      }),
+    });
+  }
+
   async getDifferences(branch: string) {
-    const diff: BitBucketDiffStat = await this.requestJSON({
-      url: `${this.repoURL}/diffstat/${branch}..${this.branch}`,
+    const rawDiff = await this.requestText({
+      url: `${this.repoURL}/diff/${branch}..${this.branch}`,
       params: {
-        pagelen: 100,
+        binary: false,
       },
     });
-    return diff.values;
+    const diff = parse(rawDiff).map(d => ({
+      newPath: d.newPath.replace(/b\//, ''),
+      binary: d.binary,
+      status: d.status,
+    }));
+    return diff;
   }
 
   async editorialWorkflowGit(files: (Entry | AssetProxy)[], entry: Entry, options: PersistOptions) {
-    const contentKey = this.generateContentKey(options.collectionName as string, entry.slug);
+    const contentKey =
+      options.combineKey || this.generateContentKey(options.collectionName as string, entry.slug);
     const branch = this.branchFromContentKey(contentKey);
     const unpublished = options.unpublished || false;
     if (!unpublished) {
@@ -478,8 +518,8 @@ export default class API {
       const diffs = await this.getDifferences(branch);
       const toDelete: DeleteEntry[] = [];
       for (const diff of diffs) {
-        if (!files.some(file => file.path === diff.new.path)) {
-          toDelete.push({ path: diff.new.path, delete: true });
+        if (!files.some(file => file.path === diff.newPath) && isBinaryFile(diff)) {
+          toDelete.push({ path: diff.newPath, delete: true });
         }
       }
 
@@ -518,6 +558,10 @@ export default class API {
     return `${CMS_BRANCH_PREFIX}/${contentKey}`;
   }
 
+  getBranchName(collectionName: string, slug: string) {
+    return this.branchFromContentKey(this.generateContentKey(collectionName, slug));
+  }
+
   async isFileExists(path: string, branch: string) {
     const fileExists = await this.readFile(path, null, { branch })
       .then(() => true)
@@ -571,31 +615,78 @@ export default class API {
     const branch = this.branchFromContentKey(contentKey);
     const pullRequest = await this.getBranchPullRequest(branch);
     const diff = await this.getDifferences(branch);
-    const path = diff.find(d => d.new.path.includes(slug))?.new.path as string;
+    const path = diff.find(d => d.newPath.includes(slug))?.newPath as string;
     // TODO: get real file id
-    const mediaFiles = await Promise.all(
-      diff.filter(d => d.new.path !== path).map(d => ({ path: d.new.path, id: null })),
+    const otherFiles = await Promise.all(
+      diff
+        .filter(d => d.newPath !== path)
+        .map(d => ({
+          path: d.newPath,
+          ...(isBinaryFile(d) ? { id: null } : { newFile: d.status === 'added' }),
+        })),
     );
     const label = await this.getPullRequestLabel(pullRequest.id);
     const status = labelToStatus(label);
-    return { branch, collection, slug, path, status, mediaFiles };
+    const timeStamp = pullRequest.updated_on;
+    return { branch, collection, slug, path, status, timeStamp, otherFiles };
   }
 
-  async readUnpublishedBranchFile(contentKey: string) {
-    const { branch, collection, slug, path, status, mediaFiles } = await this.retrieveMetadata(
-      contentKey,
-    );
+  async readUnpublishedBranchFile(contentKey: string, loadEntryMediaFiles) {
+    const {
+      branch,
+      collection,
+      slug,
+      path,
+      status,
+      timeStamp,
+      otherFiles,
+    } = await this.retrieveMetadata(contentKey);
+
+    if (isCombineKey(collection, slug)) {
+      const mediaFiles = otherFiles.filter(f => has(f, 'id'));
+      const loadedMediaFiles =
+        loadEntryMediaFiles && (await loadEntryMediaFiles(branch, mediaFiles));
+      return await Promise.all(
+        otherFiles
+          .filter(f => has(f, 'newFile'))
+          .map(async file => {
+            const fileData = await this.readFile(file.path, null, { branch });
+            return {
+              file: { path: file.path, id: null },
+              metaData: {
+                branch,
+                objects: { entry: { path: file.path, mediaFiles } },
+                status,
+                timeStamp,
+              },
+              data: fileData,
+              isModification: !file.newFile,
+              combineKey: contentKey,
+              ...(loadedMediaFiles && { mediaFiles: loadedMediaFiles }),
+            };
+          }),
+      );
+    }
 
     const [fileData, isModification] = await Promise.all([
       this.readFile(path, null, { branch }) as Promise<string>,
       this.isFileExists(path, this.branch),
     ]);
+    const loadedMediaFiles = loadEntryMediaFiles && (await loadEntryMediaFiles(branch, otherFiles));
 
     return {
       slug,
-      metaData: { branch, collection, objects: { entry: { path, mediaFiles } }, status },
-      fileData,
+      file: { path, id: null },
+      metaData: {
+        branch,
+        collection,
+        objects: { entry: { path, mediaFiles: otherFiles } },
+        timeStamp,
+        status,
+      },
+      data: fileData,
       isModification,
+      ...(loadedMediaFiles && { mediaFiles: loadedMediaFiles }),
     };
   }
 
@@ -611,6 +702,23 @@ export default class API {
     return branches;
   }
 
+  async combineColletionEntry(combineArgs, entries) {
+    const combineBranch = this.getBranchName(combineArgs.collection, combineArgs.slug);
+    const [mergeEntry, refEntry] = entries;
+    if (!combineArgs.unpublished) {
+      const refBranch = this.getBranchName(refEntry.collection, refEntry.slug);
+      const ref = await this.branchCommitSha(refBranch);
+      await this.createBranch(combineBranch, ref);
+      await this.deleteUnpublishedEntry(refEntry.collection, refEntry.slug);
+      await this.createPullRequest(combineBranch, COMBINE_PR_TITLE, combineArgs.status);
+    }
+
+    const mergeBranch = this.getBranchName(mergeEntry.collection, mergeEntry.slug);
+    const pullRequest = await this.getBranchPullRequest(mergeBranch);
+    await this.updatePullRequestTargetBranch(pullRequest, combineBranch);
+    await this.mergePullRequest(pullRequest);
+  }
+
   async updateUnpublishedEntryStatus(collection: string, slug: string, newStatus: string) {
     const contentKey = this.generateContentKey(collection, slug);
     const branch = this.branchFromContentKey(contentKey);
@@ -676,8 +784,8 @@ export default class API {
     return statuses.values;
   }
 
-  async getStatuses(collectionName: string, slug: string) {
-    const contentKey = this.generateContentKey(collectionName, slug);
+  async getStatuses(collectionName: string, slug: string, combineKey: string) {
+    const contentKey = combineKey || this.generateContentKey(collectionName, slug);
     const branch = this.branchFromContentKey(contentKey);
     const pullRequest = await this.getBranchPullRequest(branch);
     const statuses = await this.getPullRequestStatuses(pullRequest);

packages/netlify-cms-backend-bitbucket/src/implementation.ts

@@ -451,28 +451,29 @@ export default class BitbucketBackend implements Implementation {
   async unpublishedEntry(
     collection: string,
     slug: string,
+    combineKey: string | undefined,
     {
       loadEntryMediaFiles = (branch: string, files: UnpublishedEntryMediaFile[]) =>
         this.loadEntryMediaFiles(branch, files),
     } = {},
   ) {
-    const contentKey = this.api!.generateContentKey(collection, slug);
-    const data = await this.api!.readUnpublishedBranchFile(contentKey);
-    const mediaFiles = await loadEntryMediaFiles(
-      data.metaData.branch,
-      // TODO: fix this
-      // eslint-disable-next-line @typescript-eslint/ban-ts-ignore
-      // @ts-ignore
-      data.metaData.objects.entry.mediaFiles,
+    const contentKey = combineKey || this.api!.generateContentKey(collection, slug);
+    return await this.api!.readUnpublishedBranchFile(contentKey, loadEntryMediaFiles);
+  }
+
+  async unpublishedCombineEntry(combineKey: string, path: string) {
+    return await this.unpublishedEntry('', '', combineKey).then(entries =>
+      entries.find(entry => entry.file.path === path),
+    );
+  }
+
+  async combineColletionEntry(combineArgs, entries) {
+    //  combineColletionEntry is a transactional operation
+    return runWithLock(
+      this.lock,
+      () => this.api!.combineColletionEntry(combineArgs, entries),
+      'Failed to acquire combine entry  lock',
     );
-    return {
-      slug,
-      file: { path: data.metaData.objects.entry.path, id: null },
-      data: data.fileData as string,
-      metaData: data.metaData,
-      mediaFiles,
-      isModification: data.isModification,
-    };
   }
 
   async updateUnpublishedEntryStatus(collection: string, slug: string, newStatus: string) {
@@ -502,9 +503,9 @@ export default class BitbucketBackend implements Implementation {
     );
   }
 
-  async getDeployPreview(collection: string, slug: string) {
+  async getDeployPreview(collection: string, slug: string, combineKey: string) {
     try {
-      const statuses = await this.api!.getStatuses(collection, slug);
+      const statuses = await this.api!.getStatuses(collection, slug, combineKey);
       const deployStatus = getPreviewStatus(statuses, this.previewContext);
 
       if (deployStatus) {

packages/netlify-cms-backend-git-gateway/src/implementation.ts

@@ -490,8 +490,8 @@ export default class GitGateway implements Implementation {
   deleteFile(path: string, commitMessage: string) {
     return this.backend!.deleteFile(path, commitMessage);
   }
-  async getDeployPreview(collection: string, slug: string) {
-    return this.backend!.getDeployPreview(collection, slug);
+  async getDeployPreview(collection: string, slug: string, combineKey: string | undefined) {
+    return this.backend!.getDeployPreview(collection, slug, combineKey);
   }
   unpublishedEntries() {
     return this.backend!.unpublishedEntries();
@@ -501,6 +501,12 @@ export default class GitGateway implements Implementation {
       loadEntryMediaFiles: (branch, files) => this.loadEntryMediaFiles(branch, files),
     });
   }
+  unpublishedCombineEntry(combineKey: string, path: string) {
+    return this.backend!.unpublishedCombineEntry(combineKey, path);
+  }
+  combineColletionEntry(combineArgs, entries) {
+    return this.backend!.combineColletionEntry(combineArgs, entries);
+  }
   updateUnpublishedEntryStatus(collection: string, slug: string, newStatus: string) {
     return this.backend!.updateUnpublishedEntryStatus(collection, slug, newStatus);
   }

packages/netlify-cms-backend-github/src/API.ts

@@ -497,7 +497,7 @@ export default class API {
       .catch(replace404WithEmptyArray);
   }
 
-  async readUnpublishedBranchFile(contentKey: string) {
+  async readUnpublishedBranchFile(contentKey: string, loadEntryMediaFiles) {
     try {
       const metaData = await this.retrieveMetadata(contentKey).then(data =>
         data.objects.entry.path ? data : Promise.reject(null),
@@ -516,12 +516,21 @@ export default class API {
         }) as Promise<string>,
         this.isUnpublishedEntryModification(metaData.objects.entry.path),
       ]);
+      const files = metaData.objects.files || [];
+      const loadedMediaFiles =
+        loadEntryMediaFiles &&
+        (await loadEntryMediaFiles(
+          metaData.branch,
+          files.map(({ sha: id, path }) => ({ id, path })),
+        ));
 
       return {
         metaData,
-        fileData,
+        file: { path: metaData.objects.entry.path, id: null },
+        data: fileData,
         isModification,
         slug: this.slugFromContentKey(contentKey, metaData.collection),
+        ...(loadedMediaFiles && { mediaFiles: loadedMediaFiles }),
       };
     } catch (e) {
       throw new EditorialWorkflowError('content is not under editorial workflow', true);

packages/netlify-cms-backend-github/src/implementation.tsx

@@ -427,20 +427,7 @@ export default class GitHub implements Implementation {
     } = {},
   ) {
     const contentKey = this.api!.generateContentKey(collection, slug);
-    const data = await this.api!.readUnpublishedBranchFile(contentKey);
-    const files = data.metaData.objects.files || [];
-    const mediaFiles = await loadEntryMediaFiles(
-      data.metaData.branch,
-      files.map(({ sha: id, path }) => ({ id, path })),
-    );
-    return {
-      slug,
-      file: { path: data.metaData.objects.entry.path, id: null },
-      data: data.fileData as string,
-      metaData: data.metaData,
-      mediaFiles,
-      isModification: data.isModification,
-    };
+    return await this.api!.readUnpublishedBranchFile(contentKey, loadEntryMediaFiles);
   }
 
   /**