perf(customers): flatten customFieldsData, trackedData fields

close #774
erxes · May 6, 2020 · 934970b · 934970b
1 parent c5b4cb4
commit 934970b
Show file tree

Hide file tree

Showing 30 changed files with 434 additions and 170 deletions.
diff --git a/elkSyncer/main.py b/elkSyncer/main.py
@@ -14,6 +14,17 @@
 
 client = Elasticsearch([ELASTICSEARCH_URL])
 
+nestedType = {
+    "type" : "nested",
+    "properties" : {
+        "field" : { "type" : "keyword"},
+        "value" : { "type" : "text" },
+        "stringValue" : { "type" : "text" },
+        "numberValue" : { "type" : "float" },
+        "dateValue" : { "type" : "date" }
+    }
+}
+
 customer_mapping = {
     'state': {
         'type': 'keyword',
@@ -55,6 +66,8 @@
     'emailValidationStatus': {
         'type': 'keyword',
     },
+    "customFieldsData" : nestedType,
+    "trackedData" : nestedType,
 }
 
 company_mapping = {
@@ -89,6 +102,7 @@
     'businessType': {
         'type': 'keyword',
     },
+    "customFieldsData" : nestedType
 }
 
 event_mapping = {
@@ -101,6 +115,7 @@
     'customerId': {
         'type': 'keyword',
     },
+    "attributes" : nestedType
 }
 
 analysis = {

diff --git a/src/__tests__/companyDb.test.ts b/src/__tests__/companyDb.test.ts
@@ -116,7 +116,7 @@ describe('Companies model tests', () => {
     try {
       await Companies.createCompany({
         primaryName: 'name',
-        customFieldsData: { [field._id]: 'invalid number' },
+        customFieldsData: [{ field: field._id, value: 'invalid number' }],
       });
     } catch (e) {
       expect(e.message).toBe(`${field.text}: Invalid number`);
@@ -157,7 +157,7 @@ describe('Companies model tests', () => {
     try {
       await Companies.updateCompany(_company._id, {
         primaryName: 'name',
-        customFieldsData: { [field._id]: 'invalid number' },
+        customFieldsData: [{ field: field._id, value: 'invalid number' }],
       });
     } catch (e) {
       expect(e.message).toBe(`${field.text}: Invalid number`);

diff --git a/src/__tests__/companyMutations.test.ts b/src/__tests__/companyMutations.test.ts
@@ -69,7 +69,6 @@ describe('Companies mutations', () => {
       size: faker.random.number(),
       industry: 'Airlines',
       tagIds: company.tagIds,
-      customFieldsData: {},
       parentCompanyId: parent._id,
     };
 
@@ -102,7 +101,7 @@ describe('Companies mutations', () => {
     expect(result.size).toBe(args.size);
     expect(result.industry).toBe(args.industry);
     expect(expect.arrayContaining(result.tagIds)).toEqual(args.tagIds);
-    expect(result.customFieldsData).toEqual(args.customFieldsData);
+    expect(result.customFieldsData.length).toEqual(0);
     expect(result.parentCompanyId).toBe(parent._id);
   });
 
@@ -127,7 +126,6 @@ describe('Companies mutations', () => {
       emails: [faker.internet.email()],
       size: faker.random.number(),
       industry: faker.random.word(),
-      customFieldsData: {},
       ownerId: _user._id,
       parentCompanyId: parent._id,
       tagIds: [tag2._id],
@@ -166,7 +164,7 @@ describe('Companies mutations', () => {
     expect(result.size).toBe(args.size);
     expect(result.industry).toBe(args.industry);
     expect(expect.arrayContaining(result.tagIds)).toEqual(args.tagIds);
-    expect(result.customFieldsData).toEqual(args.customFieldsData);
+    expect(result.customFieldsData.length).toEqual(0);
     expect(result.ownerId).toBe(_user._id);
     expect(result.parentCompanyId).toBe(parent._id);
     expect(result.mergedIds.length).toBe(1);

diff --git a/src/__tests__/customerDb.test.ts b/src/__tests__/customerDb.test.ts
@@ -184,7 +184,7 @@ describe('Customers model tests', () => {
       await Customers.createCustomer({
         primaryEmail: 'email',
         emails: ['dombo@yahoo.com'],
-        customFieldsData: { [field._id]: 'invalid number' },
+        customFieldsData: [{ field: field._id, value: 'invalid number' }],
       });
     } catch (e) {
       expect(e.message).toBe(`${field.text}: Invalid number`);
@@ -257,7 +257,7 @@ describe('Customers model tests', () => {
       await Customers.updateCustomer(_customer._id, {
         primaryEmail: 'email',
         emails: ['dombo@yahoo.com'],
-        customFieldsData: { [field._id]: 'invalid number' },
+        customFieldsData: [{ field: field._id, value: 'invalid number' }],
       });
     } catch (e) {
       expect(e.message).toBe(`${field.text}: Invalid number`);

diff --git a/src/__tests__/customerMutations.test.ts b/src/__tests__/customerMutations.test.ts
@@ -30,7 +30,6 @@ const args = {
     github: 'github',
     website: 'website',
   },
-  customFieldsData: {},
 };
 
 const checkCustomer = src => {
@@ -142,7 +141,7 @@ describe('Customers mutations', () => {
     const customer = await graphqlRequest(mutation, 'customersAdd', args, context);
 
     checkCustomer(customer);
-    expect(customer.customFieldsData).toEqual(null);
+    expect(customer.customFieldsData.length).toEqual(0);
   });
 
   test('Edit customer', async () => {
@@ -181,7 +180,7 @@ describe('Customers mutations', () => {
     expect(customer._id).toBe(_customer._id);
 
     checkCustomer(customer);
-    expect(customer.customFieldsData).toEqual({});
+    expect(customer.customFieldsData.length).toEqual(0);
   });
 
   test('Remove customer', async () => {

diff --git a/src/__tests__/customerQueries.test.ts b/src/__tests__/customerQueries.test.ts
@@ -229,7 +229,7 @@ describe('customerQueries', () => {
           location
           visitorContactInfo
           customFieldsData
-          getTrackedData
+          trackedData
           ownerId
           position
           department

diff --git a/src/__tests__/dealQueries.test.ts b/src/__tests__/dealQueries.test.ts
@@ -19,7 +19,7 @@ import { BOARD_STATUSES } from '../db/models/definitions/constants';
 import './setup.ts';
 
 describe('dealQueries', () => {
-  const commonDealTypes = `
+  const commonDealFields = `
     _id
     name
     stageId
@@ -79,7 +79,7 @@ describe('dealQueries', () => {
         initialStageId: $initialStageId
         userIds: $userIds
       ) {
-        ${commonDealTypes}
+        ${commonDealFields}
       }
     }
   `;
@@ -173,7 +173,7 @@ describe('dealQueries', () => {
       throw new Error('Field not found');
     }
 
-    const customFieldsData = { [field1._id]: 'text' };
+    const customFieldsData = [{ field: field1._id, value: 'text' }];
 
     const product = await productFactory({ customFieldsData });
     const productNoCustomData = await productFactory();
@@ -334,7 +334,7 @@ describe('dealQueries', () => {
     const qry = `
       query deals($stageId: String!, $pipelineId: String, $sortField: String, $sortDirection: Int) {
         deals(stageId: $stageId, pipelineId: $pipelineId, sortField: $sortField, sortDirection: $sortDirection) {
-          ${commonDealTypes}
+          ${commonDealFields}
         }
       }
     `;
@@ -374,7 +374,7 @@ describe('dealQueries', () => {
     const qry = `
       query dealDetail($_id: String!) {
         dealDetail(_id: $_id) {
-          ${commonDealTypes}
+          ${commonDealFields}
         }
       }
     `;

diff --git a/src/__tests__/fieldQueries.test.ts b/src/__tests__/fieldQueries.test.ts
@@ -75,16 +75,33 @@ describe('fieldQueries', () => {
   });
 
   test('Fields combined by content type', async () => {
-    const mock = sinon.stub(elk, 'getMappings').callsFake(() => {
+    const mock = sinon.stub(elk, 'fetchElk').callsFake(() => {
       return Promise.resolve({
-        [`${elk.getIndexPrefix()}customers`]: {
-          mappings: {
-            properties: {
-              trackedData: {
-                properties: {
-                  name: 'value',
+        aggregations: {
+          trackedDataKeys: {
+            fieldKeys: {
+              buckets: [
+                {
+                  key: 'pageView',
+                  hits: {
+                    hits: {
+                      hits: [
+                        {
+                          _source: {
+                            name: 'pageView',
+                            attributes: [
+                              {
+                                field: 'url',
+                                value: '/test',
+                              },
+                            ],
+                          },
+                        },
+                      ],
+                    },
+                  },
                 },
-              },
+              ],
             },
           },
         },

diff --git a/src/__tests__/productDb.test.ts b/src/__tests__/productDb.test.ts
@@ -82,14 +82,19 @@ describe('Test products model', () => {
     expect(productObj.sku).toEqual(`${product.sku}-update`);
 
     // testing custom field data
-    const field = await fieldFactory({ contentType: 'product', contentTypeId: product._id });
-    args.customFieldsData = {
-      [field._id]: 10,
-    };
+    const field1 = await fieldFactory({ contentType: 'product', contentTypeId: product._id });
+    const field2 = await fieldFactory({ contentType: 'product', contentTypeId: product._id, validation: 'date' });
+
+    args.customFieldsData = [
+      { field: field1._id, value: 10 },
+      { field: field2._id, value: '2011-01-01' },
+    ];
 
     productObj = await Products.updateProduct(product._id, args);
 
-    expect(productObj.customFieldsData[field._id]).toBe(10);
+    if (productObj.customFieldsData) {
+      expect(productObj.customFieldsData[0].value).toBe('10');
+    }
   });
 
   test('Can not remove products', async () => {

diff --git a/src/__tests__/segmentQueries.test.ts b/src/__tests__/segmentQueries.test.ts
@@ -105,7 +105,12 @@ describe('segmentQueries', () => {
                       {
                         _source: {
                           name: 'pageView',
-                          attributes: [],
+                          attributes: [
+                            {
+                              field: 'url',
+                              value: '/test',
+                            },
+                          ],
                         },
                       },
                     ],