feat(contentful): support locales which contain country

eg: es_ES

packages/botonic-plugin-contentful/src/index.ts

@@ -8,4 +8,9 @@ export * from './time'
 export * from './tools'
 export * from './util'
 
-export { default, CmsOptions, ContentfulOptions } from './plugin'
+export {
+  default,
+  CmsOptions,
+  ContentfulOptions,
+  ContentfulCredentials,
+} from './plugin'

packages/botonic-plugin-contentful/src/nlp/locales.ts

@@ -24,6 +24,10 @@ export function checkLocale(locale: Locale): Locale {
   return locale
 }
 
+export function rootLocale(locale: Locale): Locale {
+  return locale.substr(0, 2)
+}
+
 /**
  * Converts to lowercase, trims and removes accents
  */

packages/botonic-plugin-contentful/src/nlp/normalizer.ts

@@ -4,7 +4,7 @@ import {
   DEFAULT_STOP_WORDS,
   tokenizerPerLocale,
 } from './tokens'
-import { Locale } from './locales'
+import { Locale, rootLocale } from './locales'
 import { equalArrays } from '../util/arrays'
 
 /**
@@ -118,6 +118,7 @@ export class Normalizer {
    * @throws EmptyTextException if the text is empty or only contains separators
    */
   normalize(locale: Locale, raw: string): NormalizedUtterance {
+    locale = rootLocale(locale)
     let txt = raw.replace(this.separatorsRegex, ' ')
     txt = txt.trim().toLowerCase() // TODO use preprocess without normalization? move to NormalizedUtterance constructor?
     if (!txt) {

packages/botonic-plugin-contentful/src/nlp/stemmer.ts

@@ -5,6 +5,7 @@ import StemmerEs from '@nlpjs/lang-es/src/stemmer-es'
 import StemmerPt from '@nlpjs/lang-pt/src/stemmer-pt'
 import StemmerRu from '@nlpjs/lang-ru/src/stemmer-ru'
 import { StemmerPl } from './stemmers/polish-stemmer'
+import { Locale, rootLocale } from './locales'
 
 // see https://github.com/axa-group/nlp.js/blob/HEAD/docs/language-support.md
 // and https://stackoverflow.com/a/11210358/145289
@@ -21,8 +22,8 @@ export const stemmers: { [key: string]: BaseStemmer } = {
   //node-nlp does not support polish
 }
 
-export function stemmerFor(locale: string): BaseStemmer {
-  const stem = stemmers[locale]
+export function stemmerFor(locale: Locale): BaseStemmer {
+  const stem = stemmers[rootLocale(locale)]
   if (!stem) {
     throw new Error(`No stemmer configured for locale '${locale}'`)
   }

packages/botonic-plugin-contentful/src/nlp/tokens.ts

@@ -6,7 +6,7 @@ import TokenizerRu from '@nlpjs/lang-ru/src/tokenizer-ru'
 import { esDefaultStopWords } from './stopwords/stopwords-es'
 import { caDefaultStopWords } from './stopwords/stopwords-ca'
 import { enDefaultStopWords } from './stopwords/stopwords-en'
-import { Locale } from './locales'
+import { Locale, rootLocale } from './locales'
 import { plDefaultStopWords } from './stopwords/stopwords-pl'
 import { ptDefaultStopWords } from './stopwords/stopwords-pt'
 import { ruDefaultStopWords } from './stopwords/stopwords-ru'
@@ -78,8 +78,9 @@ const tokenizers: { [locale: string]: Tokenizer } = {
 }
 
 export function tokenizerPerLocale(locale: Locale): Tokenizer {
-  return tokenizers[locale]
+  return tokenizers[rootLocale(locale)]
 }
+
 export const DEFAULT_SEPARATORS = ';,./()!?" '
 export const DEFAULT_SEPARATORS_REGEX = new RegExp(
   '[' + DEFAULT_SEPARATORS + ']',

packages/botonic-plugin-contentful/src/search/search-by-keywords.ts

@@ -7,6 +7,7 @@ import {
   Normalizer,
   NormalizedUtterance,
   Word,
+  rootLocale,
 } from '../nlp'
 import { SearchCandidate, SearchResult } from './search-result'
 
@@ -25,7 +26,7 @@ export class SearchByKeywords {
     matchType: MatchType,
     context: cms.ContextWithLocale
   ): Promise<SearchResult[]> {
-    const locale = checkLocale(context.locale)
+    const locale = rootLocale(checkLocale(context.locale))
     const contentsWithKeywords = await this.cms.contentsWithKeywords(context)
     const kws = new KeywordsParser<SearchCandidate>(
       locale,

packages/botonic-plugin-contentful/tests/contentful/ignore-fallback-decorator.test.ts

@@ -22,8 +22,9 @@ import {
 export const TEST_BUTTON_BLANK_SPANISH = '40buQOqp9jbwoxmMZhFO16'
 
 function createIgnoreFallbackDecorator() {
+  const options = testContentfulOptions()
   return new IgnoreFallbackDecorator(
-    new AdaptorDeliveryApi(createContentfulClientApi(testContentfulOptions()))
+    new AdaptorDeliveryApi(createContentfulClientApi(options), options)
   )
 }
 

packages/botonic-plugin-contentful/tests/nlp/locales.test.ts

@@ -1,5 +1,10 @@
-import { preprocess } from '../../src/nlp/locales'
+import { preprocess, rootLocale } from '../../src/nlp/locales'
 
-test('TEST normalize', () => {
-  expect(preprocess('es', ' ÑÇáü òL·l ')).toEqual('ncau ol·l')
+test('TEST preprocess', () => {
+  expect(preprocess('es_ES', ' ÑÇáü òL·l ')).toEqual('ncau ol·l')
+})
+
+test('TEST rootLocale', () => {
+  expect(rootLocale('es_ES')).toEqual('es')
+  expect(rootLocale('en')).toEqual('en')
 })

packages/botonic-plugin-contentful/tests/nlp/normalizer.test.ts

@@ -10,7 +10,10 @@ import {
 
 test('TEST: sut.normalize stopWord', () => {
   const sut = new Normalizer(undefined, { es: ['stopWórd'] })
-  expect(sut.normalize('es', 'no digas STOPword').stems).toEqual(['no', 'dec'])
+  expect(sut.normalize('es_ES', 'no digas STOPword').stems).toEqual([
+    'no',
+    'dec',
+  ])
 })
 
 test.each<any>([