WIP: add dataloader to graphql

package.json

@@ -57,6 +57,7 @@
     "body-parser": "^1.17.1",
     "cookie-parser": "^1.4.3",
     "cors": "^2.8.1",
+    "dataloader": "^2.0.0",
     "debug": "^4.1.1",
     "dotenv": "^4.0.0",
     "elasticsearch": "^16.6.0",
@@ -73,6 +74,7 @@
     "handlebars": "^4.7.3",
     "ioredis": "^3.2.2",
     "jsonwebtoken": "^8.1.0",
+    "lodash": "^4.17.15",
     "meteor-random": "^0.0.3",
     "moment": "^2.18.1",
     "mongo-uri": "^0.1.2",

src/apolloClient.ts

@@ -1,5 +1,15 @@
 import { ApolloServer, PlaygroundConfig } from 'apollo-server-express';
 import * as dotenv from 'dotenv';
+import {
+  assignedUsersLoader,
+  boardIdLoader,
+  companiesLoader,
+  customersLoader,
+  pipelineLabelsLoader,
+  pipelineLoader,
+  stageLoader,
+  userLoader,
+} from './data/dataLoaders/deal';
 import { EngagesAPI, IntegrationsAPI } from './data/dataSources';
 import resolvers from './data/resolvers';
 import typeDefs from './data/schema';
@@ -35,6 +45,21 @@ const generateDataSources = () => {
   };
 };
 
+const dataLoaders = () => {
+  return {
+    dealLoaders: {
+      assignedUsersLoader: assignedUsersLoader(),
+      userLoader: userLoader(),
+      customersLoader: customersLoader(),
+      companiesLoader: companiesLoader(),
+      pipelineLabelsLoader: pipelineLabelsLoader(),
+      pipelineLoader: pipelineLoader(),
+      stageLoader: stageLoader(),
+      boardIdLoader: boardIdLoader(),
+    },
+  };
+};
+
 const apolloServer = new ApolloServer({
   typeDefs,
   resolvers,
@@ -44,6 +69,7 @@ const apolloServer = new ApolloServer({
   context: ({ req, res }) => {
     if (!req || NODE_ENV === 'test') {
       return {
+        loaders: dataLoaders(),
         dataSources: generateDataSources(),
       };
     }
@@ -56,6 +82,7 @@ const apolloServer = new ApolloServer({
 
     if (USE_BRAND_RESTRICTIONS !== 'true') {
       return {
+        loaders: dataLoaders(),
         brandIdSelector: {},
         userBrandIdsSelector: {},
         docModifier: doc => doc,
@@ -89,6 +116,7 @@ const apolloServer = new ApolloServer({
     }
 
     return {
+      loaders: dataLoaders(),
       brandIdSelector,
       docModifier: doc => ({ ...doc, scopeBrandIds }),
       commonQuerySelector,

src/data/dataLoaders/deal.ts

@@ -0,0 +1,178 @@
+import * as DataLoader from 'DataLoader';
+import * as _ from 'lodash';
+import { Companies, Conformities, Customers, PipelineLabels, Pipelines, Stages, Users } from '../../db/models';
+import { IPipeline, IStage } from '../../db/models/definitions/boards';
+import { ICompany } from '../../db/models/definitions/companies';
+import { ICustomer } from '../../db/models/definitions/customers';
+import { IPipelineLabel } from '../../db/models/definitions/pipelineLabels';
+import { IUser } from '../../db/models/definitions/users';
+
+const groupByIds = (listObject, ids: string[]) => {
+  return listObject.filter(obj => ids.includes(obj._id));
+};
+
+const batchLabels = async labelIds => {
+  const pipelines: IPipeline[][] = [];
+
+  const flattenIds = _.flattenDeep(labelIds);
+
+  const labels = await PipelineLabels.find({ _id: { $in: flattenIds } });
+
+  for (const ids of labelIds) {
+    pipelines.push(groupByIds(labels, ids));
+  }
+
+  return pipelines;
+};
+
+const batchAssignedUsers = async userIds => {
+  const assignedUsers: IUser[][] = [];
+
+  const flattenIds = _.flattenDeep(userIds);
+
+  const users = await Users.find({ _id: { $in: flattenIds } });
+
+  for (const ids of userIds) {
+    assignedUsers.push(groupByIds(users, ids));
+  }
+
+  return assignedUsers;
+};
+
+const batchCompanies = async dealIds => {
+  const companies: ICompany[][] = [];
+  const allCompanyIds: string[][] = [];
+
+  for (const dealId of dealIds) {
+    allCompanyIds.push(
+      await Conformities.savedConformity({
+        mainType: 'deal',
+        mainTypeId: dealId,
+        relTypes: ['company'],
+      }),
+    );
+  }
+
+  const flattenIds = _.flattenDeep(allCompanyIds);
+
+  const allCompanies = await Companies.find({ _id: { $in: flattenIds } });
+
+  for (const comapnyIds of allCompanyIds) {
+    companies.push(groupByIds(allCompanies, comapnyIds));
+  }
+
+  return dealIds.map((_, idx) => companies[idx]);
+};
+
+const batchCustomers = async dealIds => {
+  const customers: ICustomer[][] = [];
+  const allCustomerIds: string[][] = [];
+
+  for (const dealId of dealIds) {
+    allCustomerIds.push(
+      await Conformities.savedConformity({
+        mainType: 'deal',
+        mainTypeId: dealId,
+        relTypes: ['customer'],
+      }),
+    );
+  }
+
+  const flattenIds = _.flattenDeep(allCustomerIds);
+
+  const allCustomers = await Customers.find({ _id: { $in: flattenIds } });
+
+  for (const customerIds of allCustomerIds) {
+    customers.push(groupByIds(allCustomers, customerIds));
+  }
+
+  return dealIds.map((_, idx) => customers[idx]);
+};
+
+const batchPipelines = async stageIds => {
+  const stages = await Stages.find({ _id: { $in: stageIds } });
+
+  const pipelineIds = stages.map(stage => stage.pipelineId);
+  const pipelines = await Pipelines.find({ _id: { $in: pipelineIds } });
+
+  return stageIds.map((_, idx) => pipelines[idx]);
+};
+
+const batchUsers = async userIds => {
+  const users = await Users.find({ _id: { $in: userIds } });
+
+  const userMap = {};
+
+  users.map(user => {
+    userMap[user._id] = user;
+  });
+
+  return userIds.map(id => userMap[id]);
+};
+
+const batchStage = async stageIds => {
+  const stages = await Stages.find({ _id: { $in: stageIds } });
+
+  const stageMap = {};
+
+  stages.map(stage => {
+    stageMap[stage._id] = stage;
+  });
+
+  return stageIds.map(id => stageMap[id]);
+};
+
+const batchBoardId = async stageIds => {
+  const boards = await Stages.aggregate([
+    {
+      $match: { _id: { $in: stageIds } },
+    },
+    {
+      $lookup: {
+        from: 'pipelines',
+        localField: 'pipelineId',
+        foreignField: '_id',
+        as: 'pipelines',
+      },
+    },
+    {
+      $unwind: '$pipelines',
+    },
+    {
+      $project: {
+        _id: '$pipelines.boardId',
+      },
+    },
+  ]);
+
+  const boardIds = boards.map(board => board._id);
+
+  return boardIds;
+};
+
+const batchSchedule = {
+  batchScheduleFn: callback => setTimeout(callback, 90),
+};
+
+const pipelineLabelsLoader = () => new DataLoader<string[], IPipelineLabel[]>(batchLabels as any, batchSchedule);
+const pipelineLoader = () => new DataLoader<string, IPipeline[]>(batchPipelines, batchSchedule);
+
+const assignedUsersLoader = () => new DataLoader<string[], IUser[]>(batchAssignedUsers, batchSchedule);
+const userLoader = () => new DataLoader<string, IPipeline[]>(batchUsers, batchSchedule);
+
+const customersLoader = () => new DataLoader<string, ICustomer[]>(batchCustomers, batchSchedule);
+const companiesLoader = () => new DataLoader<string, ICustomer[]>(batchCompanies, batchSchedule);
+
+const stageLoader = () => new DataLoader<string, IStage>(batchStage, batchSchedule);
+const boardIdLoader = () => new DataLoader<string, string>(batchBoardId, batchSchedule);
+
+export {
+  pipelineLabelsLoader,
+  pipelineLoader,
+  userLoader,
+  assignedUsersLoader,
+  customersLoader,
+  companiesLoader,
+  stageLoader,
+  boardIdLoader,
+};

src/data/resolvers/deals.ts

@@ -1,38 +1,18 @@
-import {
-  Companies,
-  Conformities,
-  Customers,
-  Fields,
-  Notifications,
-  PipelineLabels,
-  Pipelines,
-  Products,
-  Stages,
-  Users,
-} from '../../db/models';
+import { Fields, Notifications, Products } from '../../db/models';
 import { IDealDocument } from '../../db/models/definitions/deals';
 import { IContext } from '../types';
-import { boardId } from './boardUtils';
 
 export default {
-  async companies(deal: IDealDocument) {
-    const companyIds = await Conformities.savedConformity({
-      mainType: 'deal',
-      mainTypeId: deal._id,
-      relTypes: ['company'],
-    });
+  async companies(deal: IDealDocument, _args, { loaders }: IContext) {
+    const { dealLoaders } = loaders;
 
-    return Companies.find({ _id: { $in: companyIds } });
+    return dealLoaders.companiesLoader.load(deal._id);
   },
 
-  async customers(deal: IDealDocument) {
-    const customerIds = await Conformities.savedConformity({
-      mainType: 'deal',
-      mainTypeId: deal._id,
-      relTypes: ['customer'],
-    });
+  async customers(deal: IDealDocument, _args, { loaders }: IContext) {
+    const { dealLoaders } = loaders;
 
-    return Customers.find({ _id: { $in: customerIds } });
+    return dealLoaders.customersLoader.load(deal._id);
   },
 
   async products(deal: IDealDocument) {
@@ -96,22 +76,28 @@ export default {
     return amountsMap;
   },
 
-  assignedUsers(deal: IDealDocument) {
-    return Users.find({ _id: { $in: deal.assignedUserIds } });
+  assignedUsers(deal: IDealDocument, _args, { loaders }: IContext) {
+    const { dealLoaders } = loaders;
+
+    return dealLoaders.assignedUsersLoader.load(deal.assignedUserIds || []);
   },
 
-  async pipeline(deal: IDealDocument) {
-    const stage = await Stages.getStage(deal.stageId);
+  async pipeline(deal: IDealDocument, _args, { loaders }: IContext) {
+    const { dealLoaders } = loaders;
 
-    return Pipelines.findOne({ _id: stage.pipelineId });
+    return dealLoaders.pipelineLoader.load(deal.stageId);
   },
 
-  boardId(deal: IDealDocument) {
-    return boardId(deal);
+  boardId(deal: IDealDocument, _args, { loaders }: IContext) {
+    const { dealLoaders } = loaders;
+
+    return dealLoaders.boardIdLoader.load(deal.stageId);
   },
 
-  stage(deal: IDealDocument) {
-    return Stages.getStage(deal.stageId);
+  stage(deal: IDealDocument, _args, { loaders }: IContext) {
+    const { dealLoaders } = loaders;
+
+    return dealLoaders.stageLoader.load(deal.stageId);
   },
 
   isWatched(deal: IDealDocument, _args, { user }: IContext) {
@@ -128,11 +114,15 @@ export default {
     return Notifications.checkIfRead(user._id, deal._id);
   },
 
-  labels(deal: IDealDocument) {
-    return PipelineLabels.find({ _id: { $in: deal.labelIds } });
+  labels(deal: IDealDocument, _args, { loaders }: IContext) {
+    const { dealLoaders } = loaders;
+
+    return dealLoaders.pipelineLabelsLoader.load(deal.labelIds || []);
   },
 
-  createdUser(deal: IDealDocument) {
-    return Users.findOne({ _id: deal.userId });
+  createdUser(deal: IDealDocument, _args, { loaders }: IContext) {
+    const { dealLoaders } = loaders;
+
+    return dealLoaders.userLoader.load(deal.userId || '');
   },
 };

src/data/resolvers/queries/boards.ts

@@ -150,7 +150,7 @@ const boardQueries = {
    *  Pipeline detail
    */
   pipelineDetail(_root, { _id }: { _id: string }) {
-    return Pipelines.findOne({ _id });
+    return Pipelines.findOne({ _id }).lean();
   },
 
   /**