diff --git a/redisinsight/api/package.json b/redisinsight/api/package.json index 26656f0f4a..867b3706c7 100644 --- a/redisinsight/api/package.json +++ b/redisinsight/api/package.json @@ -21,7 +21,7 @@ "format": "prettier --write \"src/**/*.ts\"", "lint": "eslint --ext .ts .", "start": "nest start", - "start:dev": "cross-env NODE_ENV=development SERVER_STATIC_CONTENT=1 nest start --watch --preserveWatchOutput", + "start:dev": "cross-env NODE_ENV=development SERVER_STATIC_CONTENT=1 nest start --watch", "start:debug": "nest start --debug --watch", "start:stage": "cross-env NODE_ENV=staging SERVER_STATIC_CONTENT=true node dist/src/main", "start:prod": "cross-env NODE_ENV=production node dist/src/main", @@ -84,6 +84,7 @@ "@nestjs/cli": "^9.1.2", "@nestjs/schematics": "^9.0.3", "@nestjs/testing": "^9.0.11", + "@types/adm-zip": "^0.5.0", "@types/axios": "^0.14.0", "@types/express": "^4.17.3", "@types/jest": "^26.0.15", diff --git a/redisinsight/api/src/__mocks__/custom-tutorial.ts b/redisinsight/api/src/__mocks__/custom-tutorial.ts index 558854a783..197cfcf2a8 100644 --- a/redisinsight/api/src/__mocks__/custom-tutorial.ts +++ b/redisinsight/api/src/__mocks__/custom-tutorial.ts @@ -3,6 +3,7 @@ import { CustomTutorialEntity } from 'src/modules/custom-tutorial/entities/custo import { CustomTutorialManifestType } from 'src/modules/custom-tutorial/models/custom-tutorial.manifest'; import { MemoryStoredFile } from 'nestjs-form-data'; import { UploadCustomTutorialDto } from 'src/modules/custom-tutorial/dto/upload.custom-tutorial.dto'; +import AdmZip from 'adm-zip'; export const mockCustomTutorialId = 'a77b23c1-7816-4ea4-b61f-d37795a0f805-ct-id'; @@ -10,6 +11,8 @@ export const mockCustomTutorialId2 = 'a77b23c1-7816-4ea4-b61f-d37795a0f805-ct-id export const mockCustomTutorialTmpPath = '/tmp/path'; +export const mockCustomTutorialsHttpLink = 'https://somesime.com/archive.zip'; + export const mockCustomTutorial = Object.assign(new CustomTutorial(), { id: mockCustomTutorialId, name: 'custom tutorial', @@ -23,6 +26,7 @@ export const mockCustomTutorialEntity = Object.assign(new CustomTutorialEntity() export const mockCustomTutorial2 = Object.assign(new CustomTutorial(), { id: mockCustomTutorialId2, name: 'custom tutorial 2', + link: mockCustomTutorialsHttpLink, createdAt: new Date(), }); @@ -31,13 +35,30 @@ export const mockCustomTutorialZipFile = Object.assign(new MemoryStoredFile(), { buffer: Buffer.from('zip-content', 'utf8'), }); +export const mockCustomTutorialZipFileAxiosResponse = { + data: mockCustomTutorialZipFile.buffer, +}; + +export const mockCustomTutorialAdmZipEntry = { + entryName: 'somefolder/info.md', +} as AdmZip.IZipEntry; + +export const mockCustomTutorialMacosxAdmZipEntry = { + entryName: '__MACOSX/info.md', +} as AdmZip.IZipEntry; + export const mockUploadCustomTutorialDto = Object.assign(new UploadCustomTutorialDto(), { name: mockCustomTutorial.name, file: mockCustomTutorialZipFile, }); -export const mockCustomTutorialManifestManifestJson = { - 'ct-folder-1': { +export const mockUploadCustomTutorialExternalLinkDto = Object.assign(new UploadCustomTutorialDto(), { + name: mockCustomTutorial.name, + link: mockCustomTutorialsHttpLink, +}); + +export const mockCustomTutorialManifestManifestJson = [ + { type: 'group', id: 'ct-folder-1', label: 'ct-folder-1', @@ -45,16 +66,16 @@ export const mockCustomTutorialManifestManifestJson = { // withBorder: true, // initialIsOpen: true, // }, - children: { - 'ct-sub-folder-1': { + children: [ + { type: CustomTutorialManifestType.Group, id: 'ct-sub-folder-1', label: 'ct-sub-folder-1', // args: { // initialIsOpen: false, // }, - children: { - introduction: { + children: [ + { type: CustomTutorialManifestType.InternalLink, id: 'introduction', label: 'introduction', @@ -62,7 +83,7 @@ export const mockCustomTutorialManifestManifestJson = { path: '/ct-folder-1/ct-sub-folder-1/introduction.md', }, }, - 'working-with-hashes': { + { type: CustomTutorialManifestType.InternalLink, id: 'working-with-hashes', label: 'working-with-hashes', @@ -70,9 +91,9 @@ export const mockCustomTutorialManifestManifestJson = { path: '/ct-folder-1/ct-sub-folder-1/working-with-hashes.md', }, }, - }, + ], }, - 'ct-sub-folder-2': { + { type: CustomTutorialManifestType.Group, id: 'ct-sub-folder-2', label: 'ct-sub-folder-2', @@ -80,8 +101,8 @@ export const mockCustomTutorialManifestManifestJson = { // withBorder: true, // initialIsOpen: false, // }, - children: { - introduction: { + children: [ + { type: CustomTutorialManifestType.InternalLink, id: 'introduction', label: 'introduction', @@ -89,7 +110,7 @@ export const mockCustomTutorialManifestManifestJson = { path: '/ct-folder-1/ct-sub-folder-2/introduction.md', }, }, - 'working-with-graphs': { + { type: CustomTutorialManifestType.InternalLink, id: 'working-with-graphs', label: 'working-with-graphs', @@ -97,11 +118,11 @@ export const mockCustomTutorialManifestManifestJson = { path: '/ct-folder-1/ct-sub-folder-2/working-with-graphs.md', }, }, - }, + ], }, - }, + ], }, -}; +]; export const mockCustomTutorialManifestManifest = { type: CustomTutorialManifestType.Group, @@ -121,24 +142,26 @@ export const mockCustomTutorialManifestManifest2 = { children: mockCustomTutorialManifestManifestJson, }; -export const globalCustomTutorialManifest = { - 'custom-tutorials': { +export const globalCustomTutorialManifest = [ + { type: CustomTutorialManifestType.Group, id: 'custom-tutorials', - label: 'My Tutorials', + label: 'MY TUTORIALS', _actions: [CustomTutorialActions.CREATE], args: { withBorder: true, initialIsOpen: true, }, - children: { - [mockCustomTutorialManifestManifest.id]: mockCustomTutorialManifestManifest, - [mockCustomTutorialManifestManifest2.id]: mockCustomTutorialManifestManifest2, - }, + children: [ + mockCustomTutorialManifestManifest, + mockCustomTutorialManifestManifest2, + ], }, -}; +]; export const mockCustomTutorialFsProvider = jest.fn(() => ({ + unzipFromMemoryStoredFile: jest.fn().mockResolvedValue(mockCustomTutorialTmpPath), + unzipFromExternalLink: jest.fn().mockResolvedValue(mockCustomTutorialTmpPath), unzipToTmpFolder: jest.fn().mockResolvedValue(mockCustomTutorialTmpPath), moveFolder: jest.fn(), removeFolder: jest.fn(), diff --git a/redisinsight/api/src/common/utils/errors.util.ts b/redisinsight/api/src/common/utils/errors.util.ts index 906198d2c8..e9f7413d3c 100644 --- a/redisinsight/api/src/common/utils/errors.util.ts +++ b/redisinsight/api/src/common/utils/errors.util.ts @@ -1,9 +1,9 @@ import { HttpException, InternalServerErrorException } from '@nestjs/common'; -export const wrapHttpError = (error: Error) => { +export const wrapHttpError = (error: Error, message?: string) => { if (error instanceof HttpException) { return error; } - return new InternalServerErrorException(error.message); + return new InternalServerErrorException(error.message || message); }; diff --git a/redisinsight/api/src/constants/error-messages.ts b/redisinsight/api/src/constants/error-messages.ts index f1e1cb7296..aa968c6ece 100644 --- a/redisinsight/api/src/constants/error-messages.ts +++ b/redisinsight/api/src/constants/error-messages.ts @@ -8,6 +8,7 @@ export default { CONSUMER_GROUP_NOT_FOUND: 'Consumer Group with such name was not found.', PLUGIN_STATE_NOT_FOUND: 'Plugin state was not found.', CUSTOM_TUTORIAL_NOT_FOUND: 'Custom Tutorial was not found.', + CUSTOM_TUTORIAL_UNABLE_TO_FETCH_FROM_EXTERNAL: 'Unable fetch zip file from external source.', UNDEFINED_INSTANCE_ID: 'Undefined redis database instance id.', NO_CONNECTION_TO_REDIS_DB: 'No connection to the Redis Database.', WRONG_DATABASE_TYPE: 'Wrong database type.', diff --git a/redisinsight/api/src/modules/custom-tutorial/custom-tutorial.service.spec.ts b/redisinsight/api/src/modules/custom-tutorial/custom-tutorial.service.spec.ts index fa4ac63257..14d87f4450 100644 --- a/redisinsight/api/src/modules/custom-tutorial/custom-tutorial.service.spec.ts +++ b/redisinsight/api/src/modules/custom-tutorial/custom-tutorial.service.spec.ts @@ -7,11 +7,11 @@ import { mockCustomTutorialManifestManifest, mockCustomTutorialManifestManifest2, mockCustomTutorialManifestProvider, mockCustomTutorialRepository, - MockType, mockUploadCustomTutorialDto, + MockType, mockUploadCustomTutorialDto, mockUploadCustomTutorialExternalLinkDto } from 'src/__mocks__'; import * as fs from 'fs-extra'; import { CustomTutorialFsProvider } from 'src/modules/custom-tutorial/providers/custom-tutorial.fs.provider'; -import { InternalServerErrorException, NotFoundException } from '@nestjs/common'; +import { BadRequestException, InternalServerErrorException, NotFoundException } from '@nestjs/common'; import { CustomTutorialService } from 'src/modules/custom-tutorial/custom-tutorial.service'; import { CustomTutorialRepository } from 'src/modules/custom-tutorial/repositories/custom-tutorial.repository'; import { @@ -63,12 +63,27 @@ describe('CustomTutorialService', () => { }); describe('create', () => { - it('Should create custom tutorial', async () => { + it('Should create custom tutorial from file', async () => { const result = await service.create(mockUploadCustomTutorialDto); expect(result).toEqual(mockCustomTutorialManifestManifest); }); + it('Should create custom tutorial from external url', async () => { + const result = await service.create(mockUploadCustomTutorialExternalLinkDto); + + expect(result).toEqual(mockCustomTutorialManifestManifest); + }); + + it('Should throw BadRequestException in case when either link or file was not provided', async () => { + try { + await service.create({} as any); + } catch (e) { + expect(e).toBeInstanceOf(BadRequestException); + expect(e.message).toEqual('File or external link should be provided'); + } + }); + it('Should throw InternalServerError in case of any non-HttpException error', async () => { customTutorialRepository.create.mockRejectedValueOnce(new Error('Unable to create')); @@ -98,14 +113,14 @@ describe('CustomTutorialService', () => { const result = await service.getGlobalManifest(); - expect(result).toEqual({ - 'custom-tutorials': { - ...globalCustomTutorialManifest['custom-tutorials'], - children: { - [mockCustomTutorialManifestManifest.id]: mockCustomTutorialManifestManifest, - }, + expect(result).toEqual([ + { + ...globalCustomTutorialManifest[0], + children: [ + mockCustomTutorialManifestManifest, + ], }, - }); + ]); }); it('Should return global manifest without children in case of any error', async () => { @@ -113,12 +128,12 @@ describe('CustomTutorialService', () => { const result = await service.getGlobalManifest(); - expect(result).toEqual({ - 'custom-tutorials': { - ...globalCustomTutorialManifest['custom-tutorials'], - children: {}, + expect(result).toEqual([ + { + ...globalCustomTutorialManifest[0], + children: [], }, - }); + ]); }); }); diff --git a/redisinsight/api/src/modules/custom-tutorial/providers/custom-tutorial.fs.provider.spec.ts b/redisinsight/api/src/modules/custom-tutorial/providers/custom-tutorial.fs.provider.spec.ts index 138748a6c7..c273b0c154 100644 --- a/redisinsight/api/src/modules/custom-tutorial/providers/custom-tutorial.fs.provider.spec.ts +++ b/redisinsight/api/src/modules/custom-tutorial/providers/custom-tutorial.fs.provider.spec.ts @@ -1,17 +1,33 @@ import { Test, TestingModule } from '@nestjs/testing'; import { - mockCustomTutorial, mockCustomTutorialTmpPath, mockCustomTutorialZipFile, + mockCustomTutorial, + mockCustomTutorialAdmZipEntry, + mockCustomTutorialMacosxAdmZipEntry, mockCustomTutorialsHttpLink, + mockCustomTutorialTmpPath, + mockCustomTutorialZipFile, mockCustomTutorialZipFileAxiosResponse, } from 'src/__mocks__'; import * as fs from 'fs-extra'; +import axios from 'axios'; import { CustomTutorialFsProvider } from 'src/modules/custom-tutorial/providers/custom-tutorial.fs.provider'; import { InternalServerErrorException } from '@nestjs/common'; +import AdmZip from 'adm-zip'; +import ERROR_MESSAGES from 'src/constants/error-messages'; +import config from 'src/utils/config'; + +const PATH_CONFIG = config.get('dir_path'); jest.mock('fs-extra'); const mockedFs = fs as jest.Mocked; +jest.mock('axios'); +const mockedAxios = axios as jest.Mocked; + const mockedAdmZip = { extractAllTo: jest.fn(), -}; + getEntries: jest.fn(), + extractEntryTo: jest.fn(), +} as unknown as jest.Mocked; + jest.mock('adm-zip', () => jest.fn().mockImplementation(() => mockedAdmZip)); describe('CustomTutorialFsProvider', () => { @@ -21,6 +37,7 @@ describe('CustomTutorialFsProvider', () => { jest.clearAllMocks(); jest.mock('fs-extra', () => mockedFs); jest.mock('adm-zip', () => jest.fn().mockImplementation(() => mockedAdmZip)); + jest.mock('axios', () => mockedAxios); const module: TestingModule = await Test.createTestingModule({ providers: [ @@ -35,27 +52,81 @@ describe('CustomTutorialFsProvider', () => { let prepareTmpFolderSpy; beforeEach(() => { + mockedAxios.get.mockResolvedValueOnce(mockCustomTutorialZipFileAxiosResponse); + mockedAdmZip.getEntries.mockReturnValue([]); mockedFs.ensureDir.mockImplementationOnce(() => Promise.resolve()); mockedFs.remove.mockImplementationOnce(() => Promise.resolve()); prepareTmpFolderSpy = jest.spyOn(CustomTutorialFsProvider, 'prepareTmpFolder'); + prepareTmpFolderSpy.mockResolvedValueOnce(mockCustomTutorialTmpPath); + }); + + describe('unzipFromMemoryStoredFile', () => { + it('should unzip data', async () => { + const result = await service.unzipFromMemoryStoredFile(mockCustomTutorialZipFile); + expect(result).toEqual(mockCustomTutorialTmpPath); + }); + it('should unzip data into just generated tmp folder', async () => { + prepareTmpFolderSpy.mockRestore(); + const result = await service.unzipFromMemoryStoredFile(mockCustomTutorialZipFile); + expect(result).toContain(`${PATH_CONFIG.tmpDir}/RedisInsight-v2/custom-tutorials`); + }); }); - it('should unzip data', async () => { - await service.unzipToTmpFolder(mockCustomTutorialZipFile); + describe('unzipFromExternalLink', () => { + it('should unzip data from external link', async () => { + const result = await service.unzipFromExternalLink(mockCustomTutorialsHttpLink); + expect(result).toEqual(mockCustomTutorialTmpPath); + }); + + it('should throw InternalServerError when 4incorrect external link provided', async () => { + const responsePayload = { + response: { + status: 404, + data: { message: 'resource not found' }, + }, + }; + + mockedAxios.get.mockReset().mockRejectedValueOnce(responsePayload); + + try { + await service.unzipFromExternalLink(mockCustomTutorialsHttpLink); + fail(); + } catch (e) { + expect(e).toBeInstanceOf(InternalServerErrorException); + expect(e.message).toEqual(ERROR_MESSAGES.CUSTOM_TUTORIAL_UNABLE_TO_FETCH_FROM_EXTERNAL); + } + }); }); + it('should unzip data to particular tmp folder', async () => { - prepareTmpFolderSpy.mockResolvedValueOnce(mockCustomTutorialTmpPath); + mockedAdmZip.getEntries.mockReturnValueOnce([ + mockCustomTutorialAdmZipEntry, + mockCustomTutorialMacosxAdmZipEntry, + ]); - await service.unzipToTmpFolder(mockCustomTutorialZipFile); + const result = await service.unzipToTmpFolder(mockedAdmZip); - expect(mockedAdmZip.extractAllTo).toHaveBeenCalledWith(mockCustomTutorialTmpPath, true); + expect(result).toEqual(mockCustomTutorialTmpPath); + expect(mockedAdmZip.extractEntryTo).toHaveBeenCalledTimes(1); + expect(mockedAdmZip.extractEntryTo).toHaveBeenCalledWith( + mockCustomTutorialAdmZipEntry, + mockCustomTutorialTmpPath, + true, + true, + false, + ); }); it('should throw InternalServerError', async () => { - mockedAdmZip.extractAllTo.mockRejectedValueOnce(new Error('Unable to extract file')); + mockedAdmZip.getEntries.mockReturnValueOnce([ + mockCustomTutorialAdmZipEntry, + mockCustomTutorialMacosxAdmZipEntry, + ]); + mockedAdmZip.extractEntryTo.mockImplementationOnce(() => { throw new Error('Unable to extract file'); }); try { - await service.unzipToTmpFolder(mockCustomTutorialZipFile); + await service.unzipToTmpFolder(mockedAdmZip); + fail(); } catch (e) { expect(e).toBeInstanceOf(InternalServerErrorException); expect(e.message).toEqual('Unable to extract file'); @@ -145,7 +216,7 @@ describe('CustomTutorialFsProvider', () => { }); it('should not fail in case of any error', async () => { - mockedFs.remove.mockRejectedValueOnce(new Error('No file')); + mockedFs.remove.mockReset().mockRejectedValueOnce(new Error('No file')); await service.removeFolder(mockCustomTutorial.absolutePath); diff --git a/redisinsight/api/src/modules/custom-tutorial/providers/custom-tutorial.fs.provider.ts b/redisinsight/api/src/modules/custom-tutorial/providers/custom-tutorial.fs.provider.ts index fbc4d7362e..e0e01914aa 100644 --- a/redisinsight/api/src/modules/custom-tutorial/providers/custom-tutorial.fs.provider.ts +++ b/redisinsight/api/src/modules/custom-tutorial/providers/custom-tutorial.fs.provider.ts @@ -7,6 +7,7 @@ import config from 'src/utils/config'; import * as AdmZip from 'adm-zip'; import axios from 'axios'; import { wrapHttpError } from 'src/common/utils'; +import ERROR_MESSAGES from 'src/constants/error-messages'; const PATH_CONFIG = config.get('dir_path'); @@ -16,6 +17,30 @@ const TMP_FOLDER = `${PATH_CONFIG.tmpDir}/RedisInsight-v2/custom-tutorials`; export class CustomTutorialFsProvider { private logger = new Logger('CustomTutorialFsProvider'); + /** + * Custom implementation of AdmZip.extractAllTo to ignore __MACOSX folder in the root of archive + * In some cases when we try to delete __MACOSX folder Electron app might crash + * As workaround we will never extract this folder to user's FS + * @param zip + * @param targetPath + * @param overwrite + * @param keepOriginalPermission + * @private + */ + private async extractAll(zip: AdmZip, targetPath, overwrite = true, keepOriginalPermission = false) { + zip.getEntries().forEach((entry) => { + if (!entry.entryName.startsWith('__MACOSX')) { + zip.extractEntryTo( + entry, + targetPath, + true, + overwrite, + keepOriginalPermission, + ); + } + }); + } + /** * Unzip custom tutorials archive to temporary folder * @param zip @@ -25,7 +50,8 @@ export class CustomTutorialFsProvider { const path = await CustomTutorialFsProvider.prepareTmpFolder(); await fs.remove(path); - await zip.extractAllTo(path, true); + await this.extractAll(zip, path, true); + // await zip.extractAllTo(path, true); return path; } catch (e) { @@ -55,7 +81,7 @@ export class CustomTutorialFsProvider { return this.unzipToTmpFolder(new AdmZip(data)); } catch (e) { this.logger.error('Unable fetch zip file from external source', e); - throw wrapHttpError(e); + throw wrapHttpError(e, ERROR_MESSAGES.CUSTOM_TUTORIAL_UNABLE_TO_FETCH_FROM_EXTERNAL); } } diff --git a/redisinsight/api/src/modules/custom-tutorial/providers/custom-tutorial.manifest.provider.spec.ts b/redisinsight/api/src/modules/custom-tutorial/providers/custom-tutorial.manifest.provider.spec.ts index 70e217aa34..507add9b84 100644 --- a/redisinsight/api/src/modules/custom-tutorial/providers/custom-tutorial.manifest.provider.spec.ts +++ b/redisinsight/api/src/modules/custom-tutorial/providers/custom-tutorial.manifest.provider.spec.ts @@ -1,10 +1,12 @@ import { Test, TestingModule } from '@nestjs/testing'; +import { CustomTutorialManifestProvider } from 'src/modules/custom-tutorial/providers/custom-tutorial.manifest.provider'; +import * as fs from 'fs-extra'; +import { Dirent, Stats } from 'fs'; +import { join } from 'path'; import { mockCustomTutorial, mockCustomTutorialManifestManifest, mockCustomTutorialManifestManifestJson, } from 'src/__mocks__'; -import { CustomTutorialManifestProvider } from 'src/modules/custom-tutorial/providers/custom-tutorial.manifest.provider'; -import * as fs from 'fs-extra'; jest.mock('fs-extra'); const mockedFs = fs as jest.Mocked; @@ -25,6 +27,148 @@ describe('CustomTutorialManifestProvider', () => { service = await module.get(CustomTutorialManifestProvider); }); + describe('generateManifestFile', () => { + it('should return empty array for empty folder', async () => { + mockedFs.readdir.mockResolvedValueOnce([]); + mockedFs.writeFile.mockImplementationOnce(() => Promise.resolve()); + + await service['generateManifestFile'](mockCustomTutorial.absolutePath); + + expect(mockedFs.writeFile).toHaveBeenCalledWith( + join(mockCustomTutorial.absolutePath, '_manifest.json'), + JSON.stringify([]), + 'utf8', + ); + }); + }); + + describe('getManifestJson', () => { + it('should return null in case of an error', async () => { + jest.spyOn(service as any, 'getManifestJsonFile').mockRejectedValueOnce(new Error('any error')); + + const result = await service.getManifestJson(mockCustomTutorial.absolutePath); + + expect(result).toEqual(null); + }); + }); + + describe('generateManifestEntry', () => { + it('should return empty array for empty folder', async () => { + mockedFs.readdir.mockResolvedValueOnce([]); + + const result = await service['generateManifestEntry'](mockCustomTutorial.absolutePath); + + expect(result).toEqual([]); + }); + it('should return empty array for empty folder', async () => { + // root level entries + const mockRootLevelEntries = [ + 'intro.md', + '.idea', // should be ignored since starts with . + 'subfolder', + 'manifest.json', // should be ignored since not md file + '_manifest.json', // should be ignored since starts with _ + '_some.md', // should be ignored since starts with _ + ] as unknown as Dirent[]; + + // subfolder entries + const mockSubFolderEntries = [ + 'file.md', + 'file2.md', + 'subsubfolder', + '.idea', // should be ignored since starts with . + '_some.md', // should be ignored since starts with _ + ] as unknown as Dirent[]; + + const mockSubSubFolderEntries = [ + 'file.md', + 'file2.md', + '.idea', // should be ignored since starts with . + '_some.md', // should be ignored since starts with _ + ] as unknown as Dirent[]; + + mockedFs.readdir + .mockResolvedValueOnce(mockRootLevelEntries) + .mockResolvedValueOnce(mockSubFolderEntries) + .mockResolvedValueOnce(mockSubSubFolderEntries); + + mockedFs.lstat + .mockResolvedValueOnce(({ isDirectory: () => false }) as Stats) // intro.md + .mockResolvedValueOnce(({ isDirectory: () => true }) as Stats) // subfolder/ + .mockResolvedValueOnce(({ isDirectory: () => false }) as Stats) // subfolder/file.md + .mockResolvedValueOnce(({ isDirectory: () => false }) as Stats) // subfolder/file2.md + .mockResolvedValueOnce(({ isDirectory: () => true }) as Stats) // subfolder/subsubfolder/ + .mockResolvedValueOnce(({ isDirectory: () => false }) as Stats) // subfolder/subsubfolder/file.md + .mockResolvedValueOnce(({ isDirectory: () => false }) as Stats) // subfolder/subsubfolder/file2.md + .mockResolvedValueOnce(({ isDirectory: () => false }) as Stats); // manifest.json + + const result = await service['generateManifestEntry'](mockCustomTutorial.absolutePath); + + expect(result).toEqual([ + { + args: { + path: '/intro.md', + }, + id: 'intro.md', + label: 'intro', + type: 'internal-link', + }, + { + args: { + initialIsOpen: true, + }, + children: [ + { + args: { + path: '/subfolder/file.md', + }, + id: 'file.md', + label: 'file', + type: 'internal-link', + }, + { + args: { + path: '/subfolder/file2.md', + }, + id: 'file2.md', + label: 'file2', + type: 'internal-link', + }, + { + args: { + initialIsOpen: true, + }, + children: [ + { + args: { + path: '/subfolder/subsubfolder/file.md', + }, + id: 'file.md', + label: 'file', + type: 'internal-link', + }, + { + args: { + path: '/subfolder/subsubfolder/file2.md', + }, + id: 'file2.md', + label: 'file2', + type: 'internal-link', + }, + ], + id: 'subsubfolder', + label: 'subsubfolder', + type: 'group', + }, + ], + id: 'subfolder', + label: 'subfolder', + type: 'group', + }, + ]); + }); + }); + describe('getManifest', () => { it('should successfully get manifest', async () => { mockedFs.readFile.mockResolvedValueOnce(Buffer.from(JSON.stringify(mockCustomTutorialManifestManifestJson))); @@ -34,12 +178,12 @@ describe('CustomTutorialManifestProvider', () => { expect(result).toEqual(mockCustomTutorialManifestManifestJson); }); - it('should return null when no manifest found', async () => { + it('should return [] when no manifest found', async () => { mockedFs.readFile.mockRejectedValueOnce(new Error('No file')); const result = await service.getManifestJson(mockCustomTutorial.absolutePath); - expect(result).toEqual(null); + expect(result).toEqual([]); }); }); @@ -59,7 +203,7 @@ describe('CustomTutorialManifestProvider', () => { expect(result).toEqual({ ...mockCustomTutorialManifestManifest, - children: null, + children: [], }); }); diff --git a/redisinsight/api/src/modules/custom-tutorial/repositories/local.custom-tutorial.repository.spec.ts b/redisinsight/api/src/modules/custom-tutorial/repositories/local.custom-tutorial.repository.spec.ts index 32f187425b..926f6c5857 100644 --- a/redisinsight/api/src/modules/custom-tutorial/repositories/local.custom-tutorial.repository.spec.ts +++ b/redisinsight/api/src/modules/custom-tutorial/repositories/local.custom-tutorial.repository.spec.ts @@ -69,7 +69,10 @@ describe('LocalCustomTutorialRepository', () => { const result = await service.create(mockCustomTutorial); expect(result).toEqual(mockCustomTutorial); - expect(repository.save).toHaveBeenCalledWith(mockCustomTutorialEntity); + expect(repository.save).toHaveBeenCalledWith({ + ...mockCustomTutorialEntity, + createdAt: jasmine.anything(), + }); }); }); diff --git a/redisinsight/api/src/modules/custom-tutorial/repositories/local.custom-tutorial.repository.ts b/redisinsight/api/src/modules/custom-tutorial/repositories/local.custom-tutorial.repository.ts index 1ec6b03aa2..3864d49b49 100644 --- a/redisinsight/api/src/modules/custom-tutorial/repositories/local.custom-tutorial.repository.ts +++ b/redisinsight/api/src/modules/custom-tutorial/repositories/local.custom-tutorial.repository.ts @@ -18,9 +18,10 @@ export class LocalCustomTutorialRepository extends CustomTutorialRepository { */ public async create(model: CustomTutorial): Promise { const entity = classToClass(CustomTutorialEntity, model); - await this.repository.save(entity); - return classToClass(CustomTutorial, entity); + entity.createdAt = new Date(); + + return classToClass(CustomTutorial, await this.repository.save(entity)); } /** diff --git a/redisinsight/api/src/modules/statics-management/providers/auto-updated-statics.provider.ts b/redisinsight/api/src/modules/statics-management/providers/auto-updated-statics.provider.ts index 427d8a5b02..9ac6cbbaa5 100644 --- a/redisinsight/api/src/modules/statics-management/providers/auto-updated-statics.provider.ts +++ b/redisinsight/api/src/modules/statics-management/providers/auto-updated-statics.provider.ts @@ -65,7 +65,7 @@ export class AutoUpdatedStaticsProvider implements OnModuleInit { const latestArchive = await this.getLatestArchive(); if (latestArchive) { - const zip = new AdmZip(latestArchive); + const zip = new AdmZip(latestArchive as Buffer); await fs.remove(this.options.destinationPath); await zip.extractAllTo(this.options.destinationPath, true); await fs.writeFile( diff --git a/redisinsight/api/test/api/custom-tutorials/POST-custom-tutorials.test.ts b/redisinsight/api/test/api/custom-tutorials/POST-custom-tutorials.test.ts new file mode 100644 index 0000000000..9e5908d1bd --- /dev/null +++ b/redisinsight/api/test/api/custom-tutorials/POST-custom-tutorials.test.ts @@ -0,0 +1,297 @@ +import { + expect, + describe, + it, + deps, + validateApiCall, + AdmZip, + fsExtra, + path, + serverConfig, requirements, + before, +} from '../deps'; +import { getBaseURL } from '../../helpers/server'; +const { server, request } = deps; + +// create endpoint +const creatEndpoint = () => request(server).post(`/custom-tutorials`); +const manifestEndpoint = () => request(server).get(`/custom-tutorials/manifest`); +const deleteEndpoint = (id: string) => () => request(server).delete(`/custom-tutorials/${id}`); + +const customTutorialsFolder = serverConfig.get('dir_path').customTutorials; +const staticsFolder = serverConfig.get('dir_path').staticDir; + + +const getZipArchive = () => { + const zipArchive = new AdmZip(); + + zipArchive.addFile('info.md', Buffer.from('# info.md', 'utf8')); + zipArchive.addFile('info.json', Buffer.from('# info.json', 'utf8')); + zipArchive.addFile('info.tar', Buffer.from('# info.tar', 'utf8')); + zipArchive.addFile('_info.tar', Buffer.from('# info.tar', 'utf8')); + zipArchive.addFile('folder/file.md', Buffer.from('# folder/file.md', 'utf8')); + zipArchive.addFile('.folder/file.md', Buffer.from('# .folder/file.md', 'utf8')); + zipArchive.addFile('.folder/file2.md', Buffer.from('# .folder/file2.md', 'utf8')); + zipArchive.addFile('_folder/file.md', Buffer.from('# _folder/file.md', 'utf8')); + zipArchive.addFile('__MACOSX/file.md', Buffer.from('# __MACOSX/file.md', 'utf8')); + + return zipArchive; +} + +const checkFilesUnarchivedFiles = (zip: AdmZip, tutorialFolder = '/') => { + zip.getEntries().forEach((entry) => { + expect(fsExtra.existsSync(path.join( + customTutorialsFolder, + tutorialFolder, + entry.entryName, + ))).eq(!entry.entryName.startsWith('__MACOSX')); + }); +} + +const autoGeneratedManifest = [ + { + id: 'folder', + type: 'group', + label: 'folder', + args: { initialIsOpen: true }, + children: [ + { + id: 'file.md', + type: 'internal-link', + label: 'file', + args: { path: '/folder/file.md' } + } + ] + }, + { + id: 'info.md', + type: 'internal-link', + label: 'info', + args: { path: '/info.md' } + } +] +const testManifest = [ + { + id: 'main-page', + type: 'internal-link', + label: 'INFO', + args: { path: '/info.md' } + }, + { + id: 'some-file', + type: 'internal-link', + label: 'FILE', + args: { path: '/folder/file.md' } + } +] + +const globalManifest = { + id: 'custom-tutorials', + label: 'MY TUTORIALS', + type: 'group', + _actions: [ + 'create', + ], + args: { + initialIsOpen: true, + withBorder: true, + }, + children: [], +}; + +describe('POST /custom-tutorials', () => { + requirements('rte.serverType=local'); + + describe('Common', () => { + + before(async () => { + await fsExtra.remove(customTutorialsFolder); + }); + + it('should import tutorial from file and generate _manifest.json', async () => { + const zip = getZipArchive(); + zip.writeZip(path.join(staticsFolder, 'test_no_manifest.zip')); + + // create tutorial + await validateApiCall({ + endpoint: creatEndpoint, + attach: ['file', zip.toBuffer(), 'a.zip'], + fields: [ + ['name', 'Tutorial without manifest'], + ], + statusCode: 201, + checkFn: async ({ body }) => { + const tutorialRootManifest = { + type: 'group', + id: body.id, + label: 'Tutorial without manifest', + _actions: [ 'delete' ], + _path: `/${body.id}`, + children: autoGeneratedManifest, + }; + + globalManifest.children = [tutorialRootManifest].concat(globalManifest.children); + + expect(body).deep.eq(tutorialRootManifest); + checkFilesUnarchivedFiles(zip, body?._path); + expect(JSON.parse(await fsExtra.readFile(path.join(customTutorialsFolder, body._path, '_manifest.json'), 'utf8'))) + .deep.eq(body.children); + expect((await fsExtra.readdir(customTutorialsFolder)).length).eq(1); + }, + }); + + // global manifest + await validateApiCall({ + endpoint: manifestEndpoint, + checkFn: async ({ body }) => { + expect(body).deep.eq([globalManifest]); + }, + }); + }); + + it('should import tutorial from file with manifest', async () => { + const zip = getZipArchive(); + zip.addFile('manifest.json', Buffer.from(JSON.stringify(testManifest), 'utf8')); + zip.writeZip(path.join(staticsFolder, 'test.zip')); + + await validateApiCall({ + endpoint: creatEndpoint, + attach: ['file', zip.toBuffer(), 'a.zip'], + fields: [ + ['name', 'Tutorial with manifest'], + ], + statusCode: 201, + checkFn: async ({ body }) => { + const tutorialRootManifest = { + type: 'group', + id: body.id, + label: 'Tutorial with manifest', + _actions: [ 'delete' ], + _path: `/${body.id}`, + children: testManifest, + }; + + globalManifest.children = [tutorialRootManifest].concat(globalManifest.children); + + expect(body).deep.eq(tutorialRootManifest); + checkFilesUnarchivedFiles(zip, body?._path); + expect(JSON.parse(await fsExtra.readFile(path.join(customTutorialsFolder, body._path, 'manifest.json'), 'utf8'))) + .deep.eq(body.children); + expect((await fsExtra.readdir(customTutorialsFolder)).length).eq(2); + }, + }); + + // global manifest + await validateApiCall({ + endpoint: manifestEndpoint, + checkFn: async ({ body }) => { + expect(body).deep.eq([globalManifest]); + }, + }); + }); + + it('should import tutorial from the external link with manifest', async () => { + const zip = new AdmZip(path.join(staticsFolder, 'test.zip')); + const link = `${getBaseURL()}/static/test.zip`; + + await validateApiCall({ + endpoint: creatEndpoint, + fields: [ + ['name', 'Tutorial with manifest'], + ['link', link], + ], + statusCode: 201, + checkFn: async ({ body }) => { + const tutorialRootManifest = { + type: 'group', + id: body.id, + label: 'Tutorial with manifest', + _actions: [ 'delete', 'sync' ], + _path: `/${body.id}`, + children: testManifest, + }; + + globalManifest.children = [tutorialRootManifest].concat(globalManifest.children); + + expect(body).deep.eq(tutorialRootManifest); + checkFilesUnarchivedFiles(zip, body?._path); + expect(JSON.parse(await fsExtra.readFile(path.join(customTutorialsFolder, body._path, 'manifest.json'), 'utf8'))) + .deep.eq(body.children); + expect((await fsExtra.readdir(customTutorialsFolder)).length).eq(3); + }, + }); + + // global manifest + await validateApiCall({ + endpoint: manifestEndpoint, + checkFn: async ({ body }) => { + expect(body).deep.eq([globalManifest]); + }, + }); + }); + + it('should delete tutorial', async () => { + expect((await fsExtra.readdir(customTutorialsFolder)).length).eq(3); + await validateApiCall({ + endpoint: manifestEndpoint, + checkFn: async ({ body }) => { + expect(body[0].children.length).eq(3); + }, + }); + + const toDelete = globalManifest.children.shift(); + await validateApiCall({ + endpoint: deleteEndpoint(toDelete.id), + }); + + expect((await fsExtra.readdir(customTutorialsFolder)).length).eq(2); + await validateApiCall({ + endpoint: manifestEndpoint, + checkFn: async ({ body }) => { + expect(body[0].children.length).eq(2); + expect(body).deep.eq([globalManifest]); + }, + }); + }); + + it('should delete tutorial and not fail even if folder does not exist', async () => { + expect((await fsExtra.readdir(customTutorialsFolder)).length).eq(2); + await validateApiCall({ + endpoint: manifestEndpoint, + checkFn: async ({ body }) => { + expect(body[0].children.length).eq(2); + }, + }); + + const toDelete = globalManifest.children.shift(); + + await fsExtra.remove(path.join(customTutorialsFolder, toDelete.id)); + expect((await fsExtra.readdir(customTutorialsFolder)).length).eq(1); + + await validateApiCall({ + endpoint: deleteEndpoint(toDelete.id), + }); + + expect((await fsExtra.readdir(customTutorialsFolder)).length).eq(1); + await validateApiCall({ + endpoint: manifestEndpoint, + checkFn: async ({ body }) => { + expect(body[0].children.length).eq(1); + expect(body).deep.eq([globalManifest]); + }, + }); + }); + + it('should fail when trying to delete not existing tutorial', async () => { + await validateApiCall({ + endpoint: deleteEndpoint('not existing'), + statusCode: 404, + responseBody: { + statusCode: 404, + message: 'Custom Tutorial was not found.', + error: 'Not Found', + } + }); + }); + }); +}); diff --git a/redisinsight/api/test/api/deps.ts b/redisinsight/api/test/api/deps.ts index e756baf48b..5241ec9946 100644 --- a/redisinsight/api/test/api/deps.ts +++ b/redisinsight/api/test/api/deps.ts @@ -21,7 +21,14 @@ export async function depsInit () { // initializing Redis Test Environment deps.rte = await redis.initRTE(); - testEnv.rte = deps.rte.env; + + testEnv.rte = deps.rte.env; + + if (typeof deps.server === 'string') { + testEnv.rte.serverType = 'docker'; + } else { + testEnv.rte.serverType = 'local'; + } // initializing local database await localDb.initLocalDb(deps.rte, deps.server); diff --git a/redisinsight/api/test/helpers/local-db.ts b/redisinsight/api/test/helpers/local-db.ts index 0c040b35ea..43872887b4 100644 --- a/redisinsight/api/test/helpers/local-db.ts +++ b/redisinsight/api/test/helpers/local-db.ts @@ -20,6 +20,7 @@ export const repositories = { NOTIFICATION: 'NotificationEntity', DATABASE_ANALYSIS: 'DatabaseAnalysisEntity', BROWSER_HISTORY: 'BrowserHistoryEntity', + CUSTOM_TUTORIAL: 'CustomTutorialEntity', } let localDbConnection; @@ -519,6 +520,7 @@ const truncateAll = async () => { await (await getRepository(repositories.DATABASE)).clear(); await (await getRepository(repositories.CA_CERT_REPOSITORY)).clear(); await (await getRepository(repositories.CLIENT_CERT_REPOSITORY)).clear(); + await (await getRepository(repositories.CUSTOM_TUTORIAL)).clear(); await (await resetSettings()); } diff --git a/redisinsight/api/test/helpers/server.ts b/redisinsight/api/test/helpers/server.ts index 28e4a1e327..eade20c49d 100644 --- a/redisinsight/api/test/helpers/server.ts +++ b/redisinsight/api/test/helpers/server.ts @@ -3,6 +3,8 @@ import { AppModule } from 'src/app.module'; import * as bodyParser from 'body-parser'; import { constants } from './constants'; import { connect, Socket } from "socket.io-client"; +import * as express from 'express'; +import { serverConfig } from './test'; /** * TEST_BE_SERVER - url to already running API that we want to test @@ -38,6 +40,7 @@ export const getServer = async () => { const app = moduleFixture.createNestApplication(); app.use(bodyParser.json({ limit: '512mb' })); app.use(bodyParser.urlencoded({ limit: '512mb', extended: true })); + app.use('/static', express.static(serverConfig.get('dir_path').staticDir)) await app.init(); server = await app.getHttpServer(); @@ -49,6 +52,8 @@ export const getServer = async () => { return server; } +export const getBaseURL = (): string => baseUrl; + export const getSocket = async (namespace: string, options = {}): Promise => { return new Promise((resolve, reject) => { const base = new URL(baseUrl); diff --git a/redisinsight/api/test/helpers/test.ts b/redisinsight/api/test/helpers/test.ts index 7b73ca38e9..e1cd1c31c6 100644 --- a/redisinsight/api/test/helpers/test.ts +++ b/redisinsight/api/test/helpers/test.ts @@ -1,14 +1,18 @@ import { describe, it, before, after, beforeEach } from 'mocha'; import * as util from 'util'; import * as _ from 'lodash'; +import * as path from 'path'; import * as fs from 'fs'; +import * as fsExtra from 'fs-extra'; import * as chai from 'chai'; import * as Joi from 'joi'; +import * as AdmZip from 'adm-zip'; import * as diff from 'object-diff'; import { cloneDeep, isMatch, isObject, set, isArray } from 'lodash'; import { generateInvalidDataArray } from './test/dataGenerator'; +import serverConfig from 'src/utils/config'; -export { _, fs } +export { _, path, fs, fsExtra, AdmZip, serverConfig } export const expect = chai.expect; export const testEnv: Record = {}; export { Joi, describe, it, before, after, beforeEach }; @@ -20,6 +24,7 @@ interface ITestCaseInput { endpoint: Function; // function that returns prepared supertest with url data?: any; attach?: any[]; + fields?: [string, string][]; query?: any; statusCode?: number; responseSchema?: Joi.AnySchema; @@ -37,6 +42,7 @@ export const validateApiCall = async function ({ endpoint, data, attach, + fields, query, statusCode = 200, responseSchema, @@ -54,6 +60,12 @@ export const validateApiCall = async function ({ request.attach(...attach); } + if (fields?.length) { + fields.forEach((field) => { + request.field(...field); + }) + } + // data to send with url query string if (query) { request.query(query); diff --git a/redisinsight/api/yarn.lock b/redisinsight/api/yarn.lock index 4a65435c3b..26a44d04f9 100644 --- a/redisinsight/api/yarn.lock +++ b/redisinsight/api/yarn.lock @@ -921,6 +921,13 @@ resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== +"@types/adm-zip@^0.5.0": + version "0.5.0" + resolved "https://registry.yarnpkg.com/@types/adm-zip/-/adm-zip-0.5.0.tgz#94c90a837ce02e256c7c665a6a1eb295906333c1" + integrity sha512-FCJBJq9ODsQZUNURo5ILAQueuA8WJhRvuihS3ke2iI25mJlfV2LK8jG2Qj2z2AWg8U0FtWWqBHVRetceLskSaw== + dependencies: + "@types/node" "*" + "@types/axios@^0.14.0": version "0.14.0" resolved "https://registry.yarnpkg.com/@types/axios/-/axios-0.14.0.tgz#ec2300fbe7d7dddd7eb9d3abf87999964cafce46"