Skip to content

Commit

Permalink
Chore: Importer rest types, meteor methods to TS and API unit tests (#…
Browse files Browse the repository at this point in the history
…26284)

Co-authored-by: Pierre Lehnen <55164754+pierre-lehnen-rc@users.noreply.github.com>
  • Loading branch information
albuquerquefabio and pierre-lehnen-rc authored Aug 24, 2022
1 parent 76ed463 commit c3eb54a
Show file tree
Hide file tree
Showing 31 changed files with 565 additions and 338 deletions.
37 changes: 22 additions & 15 deletions apps/meteor/app/api/server/v1/import.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,14 @@ import {
import { API } from '../api';
import { Imports } from '../../../models/server';
import { Importers } from '../../../importer/server';
import { executeUploadImportFile } from '../../../importer/server/methods/uploadImportFile';
import {
executeUploadImportFile,
executeDownloadPublicImportFile,
executeGetImportProgress,
executeGetImportFileData,
executeStartImport,
executeGetLatestImportOperations,
} from '../../../importer/server/methods';

API.v1.addRoute(
'uploadImportFile',
Expand All @@ -36,12 +43,12 @@ API.v1.addRoute(
{
authRequired: true,
validateParams: isDownloadPublicImportFileParamsPOST,
permissionsRequired: ['run-import'],
},
{
post() {
const { fileUrl, importerKey } = this.bodyParams;

Meteor.call('downloadPublicImportFile', fileUrl, importerKey);
executeDownloadPublicImportFile(this.userId, fileUrl, importerKey);

return API.v1.success();
},
Expand All @@ -53,12 +60,13 @@ API.v1.addRoute(
{
authRequired: true,
validateParams: isStartImportParamsPOST,
permissionsRequired: ['run-import'],
},
{
post() {
const { input } = this.bodyParams;

Meteor.call('startImport', input);
executeStartImport({ input });

return API.v1.success();
},
Expand All @@ -70,10 +78,12 @@ API.v1.addRoute(
{
authRequired: true,
validateParams: isGetImportFileDataParamsGET,
permissionsRequired: ['run-import'],
},
{
get() {
const result = Meteor.call('getImportFileData');
async get() {
const result = await executeGetImportFileData();

return API.v1.success(result);
},
},
Expand All @@ -84,10 +94,11 @@ API.v1.addRoute(
{
authRequired: true,
validateParams: isGetImportProgressParamsGET,
permissionsRequired: ['run-import'],
},
{
get() {
const result = Meteor.call('getImportProgress');
const result = executeGetImportProgress();
return API.v1.success(result);
},
},
Expand All @@ -98,10 +109,11 @@ API.v1.addRoute(
{
authRequired: true,
validateParams: isGetLatestImportOperationsParamsGET,
permissionsRequired: ['view-import-operations'],
},
{
get() {
const result = Meteor.call('getLatestImportOperations');
const result = executeGetLatestImportOperations();
return API.v1.success(result);
},
},
Expand All @@ -118,9 +130,7 @@ API.v1.addRoute(
post() {
const importer = Importers.get('pending-files');
if (!importer) {
throw new Meteor.Error('error-importer-not-defined', 'The Pending File Importer was not found.', {
method: 'downloadPendingFiles',
});
throw new Meteor.Error('error-importer-not-defined', 'The Pending File Importer was not found.', 'downloadPendingFiles');
}

importer.instance = new importer.importer(importer); // eslint-disable-line new-cap
Expand All @@ -144,9 +154,7 @@ API.v1.addRoute(
post() {
const importer = Importers.get('pending-avatars');
if (!importer) {
throw new Meteor.Error('error-importer-not-defined', 'The Pending File Importer was not found.', {
method: 'downloadPendingAvatars',
});
throw new Meteor.Error('error-importer-not-defined', 'The Pending File Importer was not found.', 'downloadPendingAvatars');
}

importer.instance = new importer.importer(importer); // eslint-disable-line new-cap
Expand All @@ -170,7 +178,6 @@ API.v1.addRoute(
get() {
const operation = Imports.findLastImport();
return API.v1.success({
success: true,
operation,
});
},
Expand Down
2 changes: 1 addition & 1 deletion apps/meteor/app/importer-slack/server/importer.js
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ export class SlackImporter extends Base {
}

this.converter.addUser(newUser);
Promise.all(Settings.incrementValueById('Slack_Importer_Count'));
Promise.await(Settings.incrementValueById('Slack_Importer_Count'));
}
}

Expand Down
4 changes: 2 additions & 2 deletions apps/meteor/app/importer/server/classes/ImporterBase.js
Original file line number Diff line number Diff line change
Expand Up @@ -291,8 +291,8 @@ export class Base {
this.progress.count.completed = 0;
}

this.progress.count.total = this.importRecord.count.total || 0;
this.progress.count.completed = this.importRecord.count.completed || 0;
this.progress.count.total = this.importRecord.count?.total || 0;
this.progress.count.completed = this.importRecord.count?.completed || 0;
}

/**
Expand Down
7 changes: 1 addition & 6 deletions apps/meteor/app/importer/server/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,7 @@ import { SelectionUser } from './classes/ImporterSelectionUser';
import { ProgressStep } from '../lib/ImporterProgressStep';
import { ImporterInfo } from '../lib/ImporterInfo';
import { Importers } from '../lib/Importers';
import './methods/getImportProgress';
import './methods/startImport';
import './methods/uploadImportFile';
import './methods/getImportFileData';
import './methods/downloadPublicImportFile';
import './methods/getLatestImportOperations';
import './methods';
import './startup/setImportsToInvalid';
import './startup/store';

Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
import http from 'http';
import https from 'https';
import fs from 'fs';

import { Meteor } from 'meteor/meteor';
import type { IUser } from '@rocket.chat/core-typings';

import { RocketChatImportFileInstance } from '../startup/store';
import { ProgressStep } from '../../lib/ImporterProgressStep';
import { hasPermission } from '../../../authorization/server';
import { Importers } from '..';

function downloadHttpFile(fileUrl: string, writeStream: fs.WriteStream): void {
const protocol = fileUrl.startsWith('https') ? https : http;
protocol.get(fileUrl, function (response) {
response.pipe(writeStream);
});
}

function copyLocalFile(filePath: fs.PathLike, writeStream: fs.WriteStream): void {
const readStream = fs.createReadStream(filePath);
readStream.pipe(writeStream);
}

export const executeDownloadPublicImportFile = (userId: IUser['_id'], fileUrl: string, importerKey: string): void => {
const importer = Importers.get(importerKey);
const isUrl = fileUrl.startsWith('http');
if (!importer) {
throw new Meteor.Error(
'error-importer-not-defined',
`The importer (${importerKey}) has no import class defined.`,
'downloadImportFile',
);
}
// Check if it's a valid url or path before creating a new import record
if (!isUrl) {
if (!fs.existsSync(fileUrl)) {
throw new Meteor.Error('error-import-file-missing', fileUrl, 'downloadPublicImportFile');
}
}

importer.instance = new importer.importer(importer); // eslint-disable-line new-cap

const oldFileName = fileUrl.substring(fileUrl.lastIndexOf('/') + 1).split('?')[0];
const date = new Date();
const dateStr = `${date.getUTCFullYear()}${date.getUTCMonth()}${date.getUTCDate()}${date.getUTCHours()}${date.getUTCMinutes()}${date.getUTCSeconds()}`;
const newFileName = `${dateStr}_${userId}_${oldFileName}`;

// Store the file name on the imports collection
importer.instance.startFileUpload(newFileName);
importer.instance.updateProgress(ProgressStep.DOWNLOADING_FILE);

const writeStream = RocketChatImportFileInstance.createWriteStream(newFileName);

writeStream.on(
'error',
Meteor.bindEnvironment(() => {
importer.instance.updateProgress(ProgressStep.ERROR);
}),
);

writeStream.on(
'end',
Meteor.bindEnvironment(() => {
importer.instance.updateProgress(ProgressStep.FILE_LOADED);
}),
);

if (isUrl) {
downloadHttpFile(fileUrl, writeStream);
} else {
// If the url is actually a folder path on the current machine, skip moving it to the file store
if (fs.statSync(fileUrl).isDirectory()) {
importer.instance.updateRecord({ file: fileUrl });
importer.instance.updateProgress(ProgressStep.FILE_LOADED);
return;
}

copyLocalFile(fileUrl, writeStream);
}
};

Meteor.methods({
downloadPublicImportFile(fileUrl: string, importerKey: string) {
const userId = Meteor.userId();

if (!userId) {
throw new Meteor.Error('error-invalid-user', 'Invalid user', 'downloadPublicImportFile');
}

if (!hasPermission(userId, 'run-import')) {
throw new Meteor.Error('error-action-not-allowed', 'Importing is not allowed', 'downloadPublicImportFile');
}

executeDownloadPublicImportFile(userId, fileUrl, importerKey);
},
});
Loading

0 comments on commit c3eb54a

Please sign in to comment.