Contentful/csv export

.github/workflows/botonic-plugin-contentful-tests.yml

@@ -25,7 +25,7 @@ jobs:
     - name: Install dev dependencies
       run: (cd ./packages/$PACKAGE && npm install -D)
     - name: Build
-      run: (cd ./packages/$PACKAGE && npm run build)
+      run: (cd ./packages/$PACKAGE && npm run build_with_tests)
     - name: Run tests
       env:
         CONTENTFUL_TEST_SPACE_ID: ${{ secrets.CONTENTFUL_TEST_SPACE_ID }}

.github/workflows/botonic-plugin-dynamo-tests.yml

@@ -25,7 +25,7 @@ jobs:
     - name: Install dev dependencies
       run: (cd ./packages/$PACKAGE && npm install -D)
     - name: Build
-      run: (cd ./packages/$PACKAGE && npm run build)
+      run: (cd ./packages/$PACKAGE && npm run build_with_tests)
     - name: Run tests
       env:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

package.json

@@ -22,7 +22,7 @@
     "ts-mockito": "^2.5.0",
     "ts-node": "^8.6.2",
     "tslib": "^1.10.0",
-    "typescript": "~3.7.5"
+    "typescript": "^3.8.3"
   },
   "engines": {
     "node": ">=8.0.0"

packages/botonic-plugin-contentful/jest.setup.js

@@ -1,2 +1,2 @@
 // uncomment to extend jest test timeout while debugging from IDE
-// jest.setTimeout(300000);
+// jest.setTimeout(3000000)

packages/botonic-plugin-contentful/package.json

@@ -25,11 +25,14 @@
   "dependencies": {
     "@babel/runtime": "^7.8.3",
     "contentful": "^7.13.1",
+    "csv-parse": "^4.8.6",
+    "csv-stringify": "^5.3.6",
     "escape-string-regexp": "^2.0.0",
     "memoizee": "^0.4.14",
     "moment": "^2.24.0",
     "moment-timezone": "^0.5.27",
-    "node-nlp": "^4.0.2"
+    "node-nlp": "^4.0.2",
+    "sort-stream": "^1.0.1"
   },
   "devDependencies": {
     "@types/memoizee": "^0.4.3",

packages/botonic-plugin-contentful/src/cms/callback.ts

@@ -1,4 +1,9 @@
-import { CMS, ContentType, MESSAGE_TYPES, MessageContentType } from './cms'
+import {
+  CMS,
+  ContentType,
+  MESSAGE_CONTENT_TYPES,
+  MessageContentType,
+} from './cms'
 import escapeStringRegexp from 'escape-string-regexp'
 import { Context } from './context'
 import { TopContent } from './contents'
@@ -54,7 +59,7 @@ export class ContentCallback extends Callback {
   }
 
   private static checkDeliverableModel(modelType: string): MessageContentType {
-    if (MESSAGE_TYPES.includes(modelType as MessageContentType)) {
+    if (MESSAGE_CONTENT_TYPES.includes(modelType as MessageContentType)) {
       return modelType as MessageContentType
     } else {
       throw new Error(

packages/botonic-plugin-contentful/src/cms/cms.ts

@@ -15,6 +15,7 @@ import {
   TopContent,
   Content,
 } from './contents'
+import { enumValues } from '../util/enums'
 
 export enum MessageContentType {
   CAROUSEL = 'carousel',
@@ -23,8 +24,10 @@ export enum MessageContentType {
   CHITCHAT = 'chitchat', //so far it's an alias for TEXT
   STARTUP = 'startUp',
 }
-export const MESSAGE_TYPES = Object.values(MessageContentType).map(
-  m => m as MessageContentType
+
+// CHITCHAT removed because it's an alias for texts
+export const MESSAGE_CONTENT_TYPES = enumValues(MessageContentType).filter(
+  m => m != MessageContentType.CHITCHAT
 )
 
 export enum NonMessageTopContentType {
@@ -39,19 +42,28 @@ export const TopContentType = {
   ...MessageContentType,
   ...NonMessageTopContentType,
 }
-export const TOPCONTENT_TYPES = Object.values(TopContentType).map(
-  m => m as TopContentType
-)
+export const TOP_CONTENT_TYPES = [
+  ...MESSAGE_CONTENT_TYPES,
+  ...enumValues(NonMessageTopContentType),
+]
 
 export enum SubContentType {
   BUTTON = 'button',
   ELEMENT = 'element',
 }
 export type ContentType = TopContentType | SubContentType
 export const ContentType = { ...TopContentType, ...SubContentType }
-export const CONTENT_TYPES = Object.values(ContentType).map(
-  m => m as ContentType
-)
+export const CONTENT_TYPES = [
+  ...TOP_CONTENT_TYPES,
+  ...enumValues(SubContentType),
+]
+
+export type BotonicContentType = MessageContentType | SubContentType
+export const BotonicContentType = { ...MessageContentType, ...SubContentType }
+export const BOTONIC_CONTENT_TYPES = [
+  ...MESSAGE_CONTENT_TYPES,
+  ...enumValues(SubContentType),
+]
 
 export function isSameModel(model1: ContentType, model2: ContentType): boolean {
   switch (model1) {

packages/botonic-plugin-contentful/src/cms/context.ts

@@ -9,4 +9,9 @@ export const DEFAULT_CONTEXT: Context = {}
 export interface Context {
   locale?: Locale
   callbacks?: CallbackMap
+  /**
+   * When set, empty fields will be blank even if they have a value for the fallback locale
+   * NOT applying it so far for assets because cms.Asset does not support blank assets
+   */
+  ignoreFallbackLocale?: boolean
 }

packages/botonic-plugin-contentful/src/contentful/contents/button.ts

@@ -12,7 +12,7 @@ import { TextFields } from './text'
 import { UrlFields } from './url'
 
 export class ButtonDelivery {
-  private static BUTTON_CONTENT_TYPE = 'button'
+  public static BUTTON_CONTENT_TYPE = 'button'
   private static PAYLOAD_CONTENT_TYPE = 'payload'
   constructor(private readonly delivery: DeliveryApi) {}
 

packages/botonic-plugin-contentful/src/contentful/contents/carousel.ts

@@ -36,7 +36,7 @@ export class CarouselDelivery extends DeliveryWithFollowUp {
     )
   }
 
-  private async elementFromEntry(
+  public async elementFromEntry(
     entry: contentful.Entry<ElementFields>,
     context: cms.Context
   ): Promise<cms.Element> {

packages/botonic-plugin-contentful/src/contentful/delivery-api.ts

@@ -35,6 +35,8 @@ export interface DeliveryApi {
     context: Context,
     query?: any
   ): Promise<contentful.EntryCollection<T>>
+
+  getContentType(id: string): Promise<contentful.ContentType>
 }
 
 /**
@@ -67,6 +69,15 @@ export class AdaptorDeliveryApi implements DeliveryApi {
     )
   }
 
+  async getContentType(id: string): Promise<contentful.ContentType> {
+    try {
+      return this.client.getContentType(id)
+    } catch (e) {
+      console.error(`ERROR in getContentType for id ${id}: ${e}`)
+      throw e
+    }
+  }
+
   private static queryFromContext(context: Context, query: any = {}): any {
     if (context.locale) {
       query['locale'] = context.locale

packages/botonic-plugin-contentful/src/contentful/delivery/cache.ts

@@ -1,12 +1,13 @@
 import * as contentful from 'contentful/index'
-import { Entry } from 'contentful/index'
+import { ContentType, Entry } from 'contentful/index'
 import memoize from 'memoizee'
 import { ReducedClientApi } from './client-api'
 
 export class CachedClientApi implements ReducedClientApi {
   readonly getAsset: (id: string, query?: any) => Promise<contentful.Asset>
   readonly getEntries: <T>(query: any) => Promise<contentful.EntryCollection<T>>
   readonly getEntry: <T>(id: string, query?: any) => Promise<Entry<T>>
+  readonly getContentType: (id: string) => Promise<ContentType>
 
   constructor(readonly client: ReducedClientApi, readonly cacheTtlMs = 10000) {
     const options = (length: number) =>
@@ -24,5 +25,6 @@ export class CachedClientApi implements ReducedClientApi {
     this.getAsset = memoize(client.getAsset, options(2))
     this.getEntries = memoize(client.getEntries, options(1))
     this.getEntry = memoize(client.getEntry, options(2))
+    this.getContentType = memoize(client.getContentType, options(1))
   }
 }

packages/botonic-plugin-contentful/src/contentful/delivery/client-api.ts

@@ -2,5 +2,5 @@ import { ContentfulClientApi } from 'contentful'
 
 export type ReducedClientApi = Pick<
   ContentfulClientApi,
-  'getAsset' | 'getEntries' | 'getEntry'
+  'getAsset' | 'getEntries' | 'getEntry' | 'getContentType'
 >

packages/botonic-plugin-contentful/src/contentful/ignore-fallback-decorator.ts

@@ -0,0 +1,118 @@
+import { Asset, ContentType, Entry, EntryCollection } from 'contentful'
+import { DeliveryApi } from './delivery-api'
+import { Context } from '../cms'
+import {
+  ContentfulVisitor,
+  I18nEntryTraverser,
+  I18nValue,
+  VisitedField,
+} from './traverser'
+
+export class IgnoreFallbackDecorator implements DeliveryApi {
+  constructor(private readonly api: DeliveryApi) {}
+
+  getContentType(id: string): Promise<ContentType> {
+    return this.api.getContentType(id)
+  }
+
+  async getEntries<T>(
+    context: Context,
+    query: any = {}
+  ): Promise<EntryCollection<T>> {
+    if (!context.ignoreFallbackLocale) {
+      return this.api.getEntries(context, query)
+    }
+    let entries = await this.api.getEntries<T>(this.i18nContext(context), query)
+
+    entries = { ...entries }
+    entries.items = await this.traverseEntries(context, entries.items)
+    return entries
+  }
+
+  async getEntry<T>(
+    id: string,
+    context: Context,
+    query: any = {}
+  ): Promise<Entry<T>> {
+    if (!context.ignoreFallbackLocale) {
+      return this.api.getEntry(id, context, query)
+    }
+    const entry = await this.api.getEntry<T>(
+      id,
+      this.i18nContext(context),
+      query
+    )
+    return (await this.traverseEntries(context, [entry]))[0]
+  }
+
+  async traverseEntries<T>(
+    context: Context,
+    entries: Entry<T>[]
+  ): Promise<Entry<T>[]> {
+    const visitor = new IgnoreFallbackVisitor(context)
+    return Promise.all(
+      entries.map(async item => {
+        const traverser = new I18nEntryTraverser(this.api, visitor)
+        return await traverser.traverse(item, context)
+      })
+    )
+  }
+
+  getAsset(id: string, query?: any): Promise<Asset> {
+    return this.api.getAsset(id, query)
+  }
+
+  private i18nContext(context: Context) {
+    return {
+      ...context,
+      locale: '*',
+    } as Context
+  }
+}
+
+class IgnoreFallbackVisitor implements ContentfulVisitor {
+  contextForContentful: Context
+  constructor(readonly context: Context) {
+    if (!context.locale) {
+      throw new Error(
+        'Context.ignoreFallbackLocale set but Context.locale not set'
+      )
+    }
+    this.contextForContentful = {
+      ...context,
+      locale: '*',
+    }
+  }
+
+  visitEntry<T>(entry: Entry<T>): Entry<T> {
+    return entry
+  }
+
+  visitStringField(vf: VisitedField<string>): I18nValue<string> {
+    return this.hackType(vf.value[vf.locale], '')
+  }
+  hackType<T>(t: T, defaultValue?: T): I18nValue<T> {
+    if (defaultValue != undefined) {
+      t = t ?? defaultValue
+    }
+    return (t as any) as I18nValue<T>
+  }
+
+  visitMultipleStringField(vf: VisitedField<string[]>): I18nValue<string[]> {
+    return this.hackType(vf.value[vf.locale], [])
+  }
+
+  visitSingleReference<T>(vf: VisitedField<Entry<T>>): I18nValue<Entry<T>> {
+    return this.hackType(vf.value[vf.locale], (undefined as any) as Entry<T>)
+  }
+
+  visitMultipleReference<T>(
+    vf: VisitedField<EntryCollection<T>>
+  ): I18nValue<EntryCollection<T>> {
+    return this.hackType(vf.value[vf.locale])
+  }
+
+  name(): string {
+    return 'ignoreFallbackLocale'
+  }
+}