feat: Improve Engagement Dashboard's "Channels" tab performance

.changeset/proud-waves-bathe.md

@@ -0,0 +1,6 @@
+---
+"@rocket.chat/meteor": minor
+"@rocket.chat/model-typings": minor
+---
+
+Fixed Engagement Dashboard's "Channels" tab not loading when there are many rooms in the workspace

apps/meteor/ee/server/lib/engagementDashboard/channels.ts

@@ -43,15 +43,7 @@ export const findAllChannelsWithNumberOfMessages = async ({
  options,
  }).toArray();
 
- const total =
- (
- await Rooms.countChannelsWithNumberOfMessagesBetweenDate({
- start: convertDateToInt(start),
- end: convertDateToInt(end),
- startOfLastWeek: convertDateToInt(startOfLastWeek),
- endOfLastWeek: convertDateToInt(endOfLastWeek),
- }).toArray()
- )[0]?.total ?? 0;
+ const total = await Rooms.countDocuments({});
 
  return {
  channels,

apps/meteor/server/models/raw/Rooms.ts

@@ -494,24 +494,25 @@ export class RoomsRaw extends BaseRaw<IRoom> implements IRoomsModel {
  diffFromLastWeek: { $subtract: ['$messages', '$lastWeekMessages'] },
  },
  };
- const firstParams = [
- lookup,
- messagesProject,
- messagesUnwind,
- messagesGroup,
- lastWeekMessagesUnwind,
- lastWeekMessagesGroup,
- presentationProject,
- ];
+ const firstParams = [lookup, messagesProject, messagesUnwind, messagesGroup];
+ const lastParams = [lastWeekMessagesUnwind, lastWeekMessagesGroup, presentationProject];
+
  const sort = { $sort: options?.sort || { messages: -1 } };
- const params: Exclude<Parameters<Collection<IRoom>['aggregate']>[0], undefined> = [...firstParams, sort];
+ const sortAndPaginationParams: Exclude<Parameters<Collection<IRoom>['aggregate']>[0], undefined> = [sort];
 
  if (options?.offset) {
- params.push({ $skip: options.offset });
+ sortAndPaginationParams.push({ $skip: options.offset });
  }
 
  if (options?.count) {
- params.push({ $limit: options.count });
+ sortAndPaginationParams.push({ $limit: options.count });
+ }
+ const params: Exclude<Parameters<Collection<IRoom>['aggregate']>[0], undefined> = [...firstParams];
+
+ if (options?.sort) {
+ params.push(...lastParams, ...sortAndPaginationParams);
+ } else {
+ params.push(...sortAndPaginationParams, ...lastParams, sort);
  }
 
  return params;
@@ -531,22 +532,6 @@ export class RoomsRaw extends BaseRaw<IRoom> implements IRoomsModel {
  });
  }
 
- countChannelsWithNumberOfMessagesBetweenDate(params: {
- start: number;
- end: number;
- startOfLastWeek: number;
- endOfLastWeek: number;
- options?: any;
- }): AggregationCursor<{ total: number }> {
- const aggregationParams = this.getChannelsWithNumberOfMessagesBetweenDateQuery(params);
- aggregationParams.push({ $count: 'total' });
-
- return this.col.aggregate<{ total: number }>(aggregationParams, {
- allowDiskUse: true,
- readPreference: readSecondaryPreferred(),
- });
- }
-
  findOneByNameOrFname(name: NonNullable<IRoom['name'] | IRoom['fname']>, options: FindOptions<IRoom> = {}): Promise<IRoom | null> {
  const query = {
  $or: [

apps/meteor/tests/data/chat.helper.js

@@ -1,6 +1,6 @@
 import { api, credentials, request } from './api-data';
 
-export const sendSimpleMessage = ({ roomId, text = 'test message', tmid }) => {
+export const sendSimpleMessage = ({ roomId, text = 'test message', tmid = undefined }) => {
  if (!roomId) {
  throw new Error('"roomId" is required in "sendSimpleMessage" test helper');
  }

apps/meteor/tests/data/rooms.helper.js

@@ -2,14 +2,14 @@ import { resolve } from 'path';
 import { api, credentials, request } from './api-data';
 
 export const createRoom = ({
- name,
+ name = '',
  type,
- username,
- token,
- agentId,
- members,
- credentials: customCredentials,
- extraData,
+ username = '',
+ token = '',
+ agentId = '',
+ members = [],
+ credentials: customCredentials = undefined,
+ extraData = undefined,
  voipCallDirection = 'inbound',
 }) => {
  if (!type) {
@@ -41,7 +41,7 @@ export const createRoom = ({
  .set(customCredentials || credentials)
  .send({
  ...params,
- ...(members && { members }),
+ ...(members && members.length && { members }),
  ...(extraData && { extraData }),
  });
 };

apps/meteor/tests/data/uploads.helper.ts

@@ -42,10 +42,6 @@ export async function testFileUploads(filesEndpoint: 'channels.files' | 'groups.
  type: 'v',
  agentId: testUser._id,
  credentials: testUserCredentials,
- name: null,
- username: null,
- members: null,
- extraData: null,
  });
 
  return roomResponse.body.room;

apps/meteor/tests/end-to-end/api/34-engagement-dashboard.ts

@@ -0,0 +1,255 @@
+import type { IRoom } from '@rocket.chat/core-typings';
+import { expect } from 'chai';
+import { after, before, describe, it } from 'mocha';
+import type { Response } from 'supertest';
+
+import { getCredentials, api, request, credentials } from '../../data/api-data.js';
+import { sendSimpleMessage } from '../../data/chat.helper.js';
+import { updatePermission } from '../../data/permissions.helper';
+import { createRoom, deleteRoom } from '../../data/rooms.helper';
+
+describe('[Engagement Dashboard]', function () {
+ this.retries(0);
+
+ const isEnterprise = Boolean(process.env.IS_EE);
+
+ before((done) => getCredentials(done));
+
+ before(() => updatePermission('view-engagement-dashboard', ['admin']));
+
+ before(() => updatePermission('view-engagement-dashboard', ['admin']));
+
+ (isEnterprise ? describe : describe.skip)('[/engagement-dashboard/channels/list]', () => {
+ let testRoom: IRoom;
+
+ before(async () => {
+ testRoom = (await createRoom({ type: 'c', name: `channel.test.engagement.${Date.now()}-${Math.random()}` })).body.channel;
+ });
+
+ after(async () => {
+ await deleteRoom({ type: 'c', roomId: testRoom._id });
+ });
+
+ it('should fail if user does not have the view-engagement-dashboard permission', async () => {
+ await updatePermission('view-engagement-dashboard', []);
+ await request
+ .get(api('engagement-dashboard/channels/list'))
+ .set(credentials)
+ .query({
+ end: new Date().toISOString(),
+ start: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString(),
+ offset: 0,
+ count: 25,
+ })
+ .expect('Content-Type', 'application/json')
+ .expect(403)
+ .expect((res: Response) => {
+ expect(res.body).to.have.property('success', false);
+ expect(res.body.error).to.be.equal('User does not have the permissions required for this action [error-unauthorized]');
+ });
+ });
+
+ it('should fail if start param is not a valid date', async () => {
+ await updatePermission('view-engagement-dashboard', ['admin']);
+ await request
+ .get(api('engagement-dashboard/channels/list'))
+ .set(credentials)
+ .query({
+ start: 'invalid-date',
+ end: new Date().toISOString(),
+ offset: 0,
+ count: 25,
+ })
+ .expect('Content-Type', 'application/json')
+ .expect(400)
+ .expect((res: Response) => {
+ expect(res.body).to.have.property('success', false);
+ expect(res.body.error).to.be.equal('Match error: Failed Match.Where validation in field start');
+ });
+ });
+
+ it('should fail if end param is not a valid date', async () => {
+ await request
+ .get(api('engagement-dashboard/channels/list'))
+ .set(credentials)
+ .query({
+ start: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString(),
+ end: 'invalid-date',
+ offset: 0,
+ count: 25,
+ })
+ .expect('Content-Type', 'application/json')
+ .expect(400)
+ .expect((res: Response) => {
+ expect(res.body).to.have.property('success', false);
+ expect(res.body.error).to.be.equal('Match error: Failed Match.Where validation in field end');
+ });
+ });
+
+ it('should fail if start param is not provided', async () => {
+ await request
+ .get(api('engagement-dashboard/channels/list'))
+ .set(credentials)
+ .query({
+ end: new Date(),
+ })
+ .expect('Content-Type', 'application/json')
+ .expect(400)
+ .expect((res: Response) => {
+ expect(res.body).to.have.property('success', false);
+ expect(res.body.error).to.be.equal("Match error: Missing key 'start'");
+ });
+ });
+
+ it('should fail if end param is not provided', async () => {
+ await request
+ .get(api('engagement-dashboard/channels/list'))
+ .set(credentials)
+ .query({
+ start: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString(),
+ })
+ .expect('Content-Type', 'application/json')
+ .expect(400)
+ .expect((res: Response) => {
+ expect(res.body).to.have.property('success', false);
+ expect(res.body.error).to.be.equal("Match error: Missing key 'end'");
+ });
+ });
+
+ it('should succesfuly return results', async () => {
+ await request
+ .get(api('engagement-dashboard/channels/list'))
+ .set(credentials)
+ .query({
+ end: new Date().toISOString(),
+ start: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString(),
+ })
+ .expect('Content-Type', 'application/json')
+ .expect(200)
+ .expect((res: Response) => {
+ expect(res.body).to.have.property('success', true);
+ expect(res.body).to.have.property('offset', 0);
+ expect(res.body).to.have.property('count');
+ expect(res.body).to.have.property('total');
+ expect(res.body).to.have.property('channels');
+ expect(res.body.channels).to.be.an('array').that.is.not.empty;
+
+ expect(res.body.channels[0]).to.be.an('object').that.is.not.empty;
+ expect(res.body.channels[0]).to.have.property('messages').that.is.a('number');
+ expect(res.body.channels[0]).to.have.property('lastWeekMessages').that.is.a('number');
+ expect(res.body.channels[0]).to.have.property('diffFromLastWeek').that.is.a('number');
+ expect(res.body.channels[0].room).to.be.an('object').that.is.not.empty;
+
+ expect(res.body.channels[0].room).to.have.property('_id').that.is.a('string');
+ expect(res.body.channels[0].room).to.have.property('name').that.is.a('string');
+ expect(res.body.channels[0].room).to.have.property('ts').that.is.a('string');
+ expect(res.body.channels[0].room).to.have.property('t').that.is.a('string');
+ expect(res.body.channels[0].room).to.have.property('_updatedAt').that.is.a('string');
+ });
+ });
+
+ it('should correctly count messages in an empty room', async () => {
+ await request
+ .get(api('engagement-dashboard/channels/list'))
+ .set(credentials)
+ .query({
+ end: new Date().toISOString(),
+ start: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString(),
+ })
+ .expect('Content-Type', 'application/json')
+ .expect(200)
+ .expect((res: Response) => {
+ expect(res.body).to.have.property('success', true);
+ expect(res.body).to.have.property('offset', 0);
+ expect(res.body).to.have.property('count');
+ expect(res.body).to.have.property('total');
+ expect(res.body).to.have.property('channels');
+ expect(res.body.channels).to.be.an('array').that.is.not.empty;
+
+ const channelRecord = res.body.channels.find(({ room }: { room: { _id: string } }) => room._id === testRoom._id);
+ expect(channelRecord).not.to.be.undefined;
+
+ expect(channelRecord).to.be.an('object').that.is.not.empty;
+ expect(channelRecord).to.have.property('messages', 0);
+ expect(channelRecord).to.have.property('lastWeekMessages', 0);
+ expect(channelRecord).to.have.property('diffFromLastWeek', 0);
+ expect(channelRecord.room).to.be.an('object').that.is.not.empty;
+
+ expect(channelRecord.room).to.have.property('_id', testRoom._id);
+ expect(channelRecord.room).to.have.property('name', testRoom.name);
+ expect(channelRecord.room).to.have.property('ts', testRoom.ts);
+ expect(channelRecord.room).to.have.property('t', testRoom.t);
+ expect(channelRecord.room).to.have.property('_updatedAt', testRoom._updatedAt);
+ });
+ });
+
+ it('should correctly count messages diff compared to last week when there are messages in a room', async () => {
+ await sendSimpleMessage({ roomId: testRoom._id });
+ await request
+ .get(api('engagement-dashboard/channels/list'))
+ .set(credentials)
+ .query({
+ end: new Date().toISOString(),
+ start: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString(),
+ })
+ .expect('Content-Type', 'application/json')
+ .expect(200)
+ .expect((res: Response) => {
+ expect(res.body).to.have.property('success', true);
+ expect(res.body).to.have.property('offset', 0);
+ expect(res.body).to.have.property('count');
+ expect(res.body).to.have.property('total');
+ expect(res.body).to.have.property('channels');
+ expect(res.body.channels).to.be.an('array').that.is.not.empty;
+
+ const channelRecord = res.body.channels.find(({ room }: { room: { _id: string } }) => room._id === testRoom._id);
+ expect(channelRecord).not.to.be.undefined;
+
+ expect(channelRecord).to.be.an('object').that.is.not.empty;
+ expect(channelRecord).to.have.property('messages', 1);
+ expect(channelRecord).to.have.property('lastWeekMessages', 0);
+ expect(channelRecord).to.have.property('diffFromLastWeek', 1);
+ expect(channelRecord.room).to.be.an('object').that.is.not.empty;
+
+ expect(channelRecord.room).to.have.property('_id', testRoom._id);
+ expect(channelRecord.room).to.have.property('name', testRoom.name);
+ expect(channelRecord.room).to.have.property('ts', testRoom.ts);
+ expect(channelRecord.room).to.have.property('t', testRoom.t);
+ });
+ });
+
+ it('should correctly count messages from last week and diff when moving to the next week', async () => {
+ await request
+ .get(api('engagement-dashboard/channels/list'))
+ .set(credentials)
+ .query({
+ end: new Date(Date.now() + 8 * 24 * 60 * 60 * 1000).toISOString(),
+ start: new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(),
+ })
+ .expect('Content-Type', 'application/json')
+ .expect(200)
+ .expect((res: Response) => {
+ expect(res.body).to.have.property('success', true);
+ expect(res.body).to.have.property('offset', 0);
+ expect(res.body).to.have.property('count');
+ expect(res.body).to.have.property('total');
+ expect(res.body).to.have.property('channels');
+ expect(res.body.channels).to.be.an('array').that.is.not.empty;
+
+ const channelRecord = res.body.channels.find(({ room }: { room: { _id: string } }) => room._id === testRoom._id);
+ expect(channelRecord).not.to.be.undefined;
+
+ expect(channelRecord).to.be.an('object').that.is.not.empty;
+ expect(channelRecord).to.have.property('messages', 0);
+ expect(channelRecord).to.have.property('lastWeekMessages', 1);
+ expect(channelRecord).to.have.property('diffFromLastWeek', -1);
+ expect(channelRecord.room).to.be.an('object').that.is.not.empty;
+
+ expect(channelRecord.room).to.have.property('_id', testRoom._id);
+ expect(channelRecord.room).to.have.property('name', testRoom.name);
+ expect(channelRecord.room).to.have.property('ts', testRoom.ts);
+ expect(channelRecord.room).to.have.property('t', testRoom.t);
+ });
+ });
+ });
+});