diff --git a/app/jest.config.js b/app/jest.config.js index 2c3b7794..2e4995d2 100644 --- a/app/jest.config.js +++ b/app/jest.config.js @@ -1,5 +1,15 @@ module.exports = { preset: '@vue/cli-plugin-unit-jest', + collectCoverage: true, + collectCoverageFrom: ['src/**/*.{js,vue}', '!src/router/**'], + coverageThreshold: { + // global: { + // branches: 13, + // functions: 30, + // lines: 30, + // statements: 30 + // } + }, transform: { '^.+\\.vue$': 'vue-jest', '\\.(gif)$': '/tests/jest/__mocks__/fileMock.js', diff --git a/app/src/assets/css/modules/_utility.scss b/app/src/assets/css/modules/_utility.scss index 18fb00ed..6ea64d03 100644 --- a/app/src/assets/css/modules/_utility.scss +++ b/app/src/assets/css/modules/_utility.scss @@ -294,7 +294,7 @@ font-size: 3rem; } @include respond(phone){ - top: -10px; + top: -10px !important; font-size: 2.3rem; } &_subtitle { @@ -477,6 +477,14 @@ background-color: rgba($primary-black, 0.7) !important; border: 1px solid $primary !important; } + &_icon_mobile { + &_lg { + @include respond(phone){ + height: 72px; + font-size: 72px!important; + } + } + } } .dialog-box { diff --git a/app/src/components/TextEditor.vue b/app/src/components/TextEditor.vue index a8687fc2..f782dcea 100644 --- a/app/src/components/TextEditor.vue +++ b/app/src/components/TextEditor.vue @@ -43,6 +43,7 @@ export default { document.execCommand('italic') }, applyHeading () { + if (document.queryCommandValue('formatBlock') === 'h1') { return document.execCommand('formatBlock', false, 'div') } document.execCommand('formatBlock', false, '

') }, applyUl () { diff --git a/app/src/components/explorer/SearchHeader.vue b/app/src/components/explorer/SearchHeader.vue index f8c0f515..1cf39724 100644 --- a/app/src/components/explorer/SearchHeader.vue +++ b/app/src/components/explorer/SearchHeader.vue @@ -1,5 +1,5 @@ @@ -142,6 +144,17 @@ export default { computed: { imageSearch () { return this.$store.getters['explorer/getSelectedFacetFilterMaterialsValue'] + }, + // This is a WIP TODO (@Tolu) Update later + searchImagesEmpty () { + return this.searchImages.length === 0 || this.searchImages?.totalItems === 0 + }, + imagesEmpty () { + if (!Object.keys(this.images)?.length || this.images.totalItems === 0) return true + return false + }, + isEmpty () { + return (this.imagesEmpty && !this.searchEnabled) || (this.searchImagesEmpty && this.searchEnabled) } }, watch: { @@ -224,7 +237,7 @@ export default { skip () { if (this.searchEnabled) return this.skipQuery }, - fetchPolicy: 'cache-and-network', + fetchPolicy: 'cache-first', error (error) { if (error.networkError) { const err = error.networkError @@ -248,7 +261,7 @@ export default { skip () { if (!this.searchEnabled) return this.skipQuery }, - fetchPolicy: 'cache-and-network', + fetchPolicy: 'network-only', error (error) { if (error.networkError) { const err = error.networkError diff --git a/app/src/pages/explorer/xml/XmlLoader.vue b/app/src/pages/explorer/xml/XmlLoader.vue index 1ef242de..9af53d2c 100644 --- a/app/src/pages/explorer/xml/XmlLoader.vue +++ b/app/src/pages/explorer/xml/XmlLoader.vue @@ -9,27 +9,32 @@ -
+

{{ optionalChaining(() => xmlViewer.title) }}

- 
-            
-              {{ optionalChaining(() => xmlViewer.xmlString) }}
-            
-
+
- - Approve - check - - - Comment - comment - +
+ + Go Back + arrow_back + + + + Comment + comment + + + + Approve + check + + +
@@ -49,6 +54,7 @@ import 'prismjs/themes/prism-coy.min.css' import optionalChainingUtil from '@/mixins/optional-chaining-util' import Comment from '@/components/explorer/Comment' import spinner from '@/components/Spinner' +import XmlView from '@/components/explorer/XmlView' import { XML_VIEWER } from '@/modules/gql/xml-gql' import { mapGetters } from 'vuex' export default { @@ -56,7 +62,8 @@ export default { mixins: [optionalChainingUtil], components: { Comment, - spinner + spinner, + XmlView }, data () { return { @@ -69,7 +76,10 @@ export default { ...mapGetters({ isAuth: 'auth/isAuthenticated', isAdmin: 'auth/isAdmin' - }) + }), + isSmallTabView () { + return screen.width < 760 + } }, methods: { approveCuration () { @@ -77,6 +87,9 @@ export default { message: 'Something went wrong', action: () => this.approveCuration() }) + }, + navBack () { + this.$router.back() } }, mounted () { diff --git a/app/src/pages/portal/curation/ViewCuration.vue b/app/src/pages/portal/curation/ViewCuration.vue index 626acd65..aa2d36bd 100644 --- a/app/src/pages/portal/curation/ViewCuration.vue +++ b/app/src/pages/portal/curation/ViewCuration.vue @@ -27,8 +27,7 @@
- Last updated -

{{ new Date().toLocaleDateString() }}

+

Last updated: {{ new Date().toLocaleDateString() }}

@@ -37,12 +36,12 @@
diff --git a/app/src/router/module/explorer.js b/app/src/router/module/explorer.js index b39c8f94..742d4ec0 100644 --- a/app/src/router/module/explorer.js +++ b/app/src/router/module/explorer.js @@ -58,6 +58,12 @@ const explorerRoutes = [ props: true, component: () => import('@/pages/explorer/curate/spreadsheet/SpreadsheetUpload.vue'), meta: { requiresAuth: true } + }, + { + path: 'bulk', + name: 'CurateBulk', + component: () => import('@/pages/explorer/curate/spreadsheet/SpreadsheetUploadBulk.vue'), + meta: { requiresAuth: true } } ] }, @@ -125,7 +131,8 @@ const explorerRoutes = [ { path: 'curate/sdd', name: 'CurateSDD', - component: () => import('@/pages/explorer/curate/sdd/SddForm.vue') + component: () => import('@/pages/explorer/curate/sdd/SddForm.vue'), + meta: { requiresAuth: true } }, { path: 'chart', diff --git a/app/src/store/modules/explorer/curation/actions.js b/app/src/store/modules/explorer/curation/actions.js index b6dedbf0..309327ea 100644 --- a/app/src/store/modules/explorer/curation/actions.js +++ b/app/src/store/modules/explorer/curation/actions.js @@ -137,5 +137,26 @@ export default { } const responseData = await response.json() return commit('setDoiData', responseData) + }, + + async submitBulkXml ({ commit, rootGetters }, files) { + const token = rootGetters['auth/token'] + const url = `${window.location.origin}/api/curate/bulk` + const formData = new FormData() + files.forEach((file) => formData.append('uploadfile', file)) + const response = await fetch(url, { + method: 'POST', + body: formData, + redirect: 'follow', + headers: { + Authorization: 'Bearer ' + token + } + }) + if (response?.statusText !== 'OK') { + throw new Error(response.message || 'Something went wrong while submitting XMLs') + } + const result = await response.json() + commit('setXmlBulkResponse', result) + return response } } diff --git a/app/src/store/modules/explorer/curation/getters.js b/app/src/store/modules/explorer/curation/getters.js index c0346151..507aa568 100644 --- a/app/src/store/modules/explorer/curation/getters.js +++ b/app/src/store/modules/explorer/curation/getters.js @@ -13,5 +13,8 @@ export default { }, getOrcidData (state) { return state.orcidData + }, + getXmlBulkResponse (state) { + return state.xmlBulkResponse } } diff --git a/app/src/store/modules/explorer/curation/index.js b/app/src/store/modules/explorer/curation/index.js index b19b175e..a9312ca0 100644 --- a/app/src/store/modules/explorer/curation/index.js +++ b/app/src/store/modules/explorer/curation/index.js @@ -10,7 +10,8 @@ export default { fieldNameSelected: '', newChartExist: false, doiData: null, - orcidData: null + orcidData: null, + xmlBulkResponse: null } }, mutations, diff --git a/app/src/store/modules/explorer/curation/mutations.js b/app/src/store/modules/explorer/curation/mutations.js index 48e895b4..fd039dc0 100644 --- a/app/src/store/modules/explorer/curation/mutations.js +++ b/app/src/store/modules/explorer/curation/mutations.js @@ -14,5 +14,8 @@ export default { }, setDoiData (state, payload) { state.doiData = payload + }, + setXmlBulkResponse (state, payload) { + state.xmlBulkResponse = payload } } diff --git a/app/src/store/modules/explorer/results/actions.js b/app/src/store/modules/explorer/results/actions.js index d9faf92f..5ee2d720 100644 --- a/app/src/store/modules/explorer/results/actions.js +++ b/app/src/store/modules/explorer/results/actions.js @@ -182,7 +182,8 @@ export default { getArticles: articlesLength, getSamples: samplesLength, getCharts: chartsLength, - getMaterials: 0 + getMaterials: 0, + getImages: 0 }) }, diff --git a/app/tests/unit/components/portal/sideNav.spec.js b/app/tests/unit/components/portal/sideNav.spec.js index 13fc11db..fef82524 100644 --- a/app/tests/unit/components/portal/sideNav.spec.js +++ b/app/tests/unit/components/portal/sideNav.spec.js @@ -57,9 +57,9 @@ describe('SideNav.vue', () => { for (let i = 0; i < linksContainer.length; i++) { expect(linksContainer.at(i).find('div.u--font-emph-l.u_margin-top-small > span.md-body-2').text()).toBe(data[i].name) expect(linksContainer.at(i).findComponent('md-divider-stub').exists()).toBeTruthy() - expect(linksContainer.at(i).findAll('.md-list-item').length).toBe(data[i].children.length) + // expect(linksContainer.at(i).findAll('.md-list-item').length).toBe(data[i].children.length) expect(linksContainer.at(i).find('.md-list-item-link > div.md-list-item-content').exists()).toBeTruthy() - expect(linksContainer.at(i).findAll('.md-list-item-content > i.md-icon').length).toBe(data[i].children.length) + // expect(linksContainer.at(i).findAll('.md-list-item-content > i.md-icon').length).toBe(data[i].children.length) expect(linksContainer.at(i).find('.md-list-item-content > span.md-body-1.u--color-black').exists()).toBeTruthy() expect(linksContainer.at(i).find('.md-list-item-content.md-list-item-content-reduce.u--layout-flex-justify-fs.md-ripple').exists()).toBeTruthy() expect(linksContainer.at(i).find('.md-icon.md-icon-font.u--default-size.md-theme-default').exists()).toBeTruthy() diff --git a/app/tests/unit/pages/explorer/Image.spec.js b/app/tests/unit/pages/explorer/Image.spec.js index 1562b6f8..abf1f456 100644 --- a/app/tests/unit/pages/explorer/Image.spec.js +++ b/app/tests/unit/pages/explorer/Image.spec.js @@ -66,8 +66,8 @@ describe('Image.vue', () => { it('shows number of results', async () => { expect.assertions(2) expect(await wrapper.find('.utility-roverflow').exists()).toBe(true) - expect(await wrapper.find('.u_content__result').text()).toMatch( - /[1-9]\d* result/ + expect(await wrapper.find('#css-adjust-navfont > span').text()).toMatch( + /[1-9]\d*|No result/ ) }) @@ -117,4 +117,15 @@ describe('Image.vue', () => { const searchBtn = wrapper.findAll('button').at(0) expect(searchBtn.text()).toBe('Search Images') }) + + it('renders the right text when response empty', async () => { + var expectedString = 'Sorry! No Image Found' + const emptyData = apollo.images + emptyData.images = [] + emptyData.totalItems = 0 + await wrapper.setData({ images: emptyData }) + expect(wrapper.find('.gallery-item').exists()).toBe(false) + expect(wrapper.find('.utility-roverflow.u_centralize_text.u_margin-top-med').exists()).toBe(true) + expect(wrapper.find('h1.visualize_header-h1.u_margin-top-med').text()).toBe(expectedString) + }) }) diff --git a/app/tests/unit/pages/explorer/SddForm.spec.js b/app/tests/unit/pages/explorer/SddForm.spec.js new file mode 100644 index 00000000..2e66267d --- /dev/null +++ b/app/tests/unit/pages/explorer/SddForm.spec.js @@ -0,0 +1,33 @@ +import createWrapper from '../../../jest/script/wrapper' +import { enableAutoDestroy } from '@vue/test-utils' +import SddForm from '@/pages/explorer/curate/sdd/SddForm.vue' + +describe('SddForm.vue', () => { + let wrapper + beforeEach(async () => { + wrapper = await createWrapper(SddForm, { + stubs: { + MdPortal: { template: '
' } + } + }, true) + }) + + enableAutoDestroy(afterEach) + + it('renders steppers', () => { + expect.assertions(1) + const steppers = wrapper.findAll('.md-stepper') + expect(steppers.length).toBe(3) + }) + + it('provides field input for doi', () => { + const fields = wrapper.findAll('.md-field') + expect(fields.at(0).text()).toContain('DOI') + }) + + it('contains input areas for spreadsheet and supplementary files', () => { + expect.assertions(1) + const fileDrop = wrapper.findAll('.form__file-input') + expect(fileDrop.length).toBe(2) + }) +}) diff --git a/app/tests/unit/pages/explorer/XMLBulkUpload.spec.js b/app/tests/unit/pages/explorer/XMLBulkUpload.spec.js new file mode 100644 index 00000000..d0f8fb67 --- /dev/null +++ b/app/tests/unit/pages/explorer/XMLBulkUpload.spec.js @@ -0,0 +1,147 @@ +import createWrapper from '../../../jest/script/wrapper' +import { enableAutoDestroy } from '@vue/test-utils' +import SpreadsheetUploadBulk from '@/pages/explorer/curate/spreadsheet/SpreadsheetUploadBulk.vue' + +const apollo = { + verifyUser: { + isAuth: true, + user: { + username: 'Test User' + } + } +} + +const testFiles = [{ + file: { name: 'FakeFile.zip' }, + id: 'MultipleSamples 2.zip-6541598-1688622380706-application/zip', + status: 'incomplete' +}] + +const mockValues = { + bulkCurations: [ + { + xml: '', + user: { + _id: '63feb2a02e34b87a5c278ab8', + displayName: 'Test User' + }, + sampleID: 'L1_S23', + groupId: '123456', + isApproved: false, + status: 'Editing' + }, + { + xml: '', + user: { + _id: '63feb2a02e34b87a5c278ab8', + displayName: 'Test User' + }, + sampleID: 'L2_S34', + groupId: '123456', + isApproved: false, + status: 'Editing' + } + ], + bulkErrors: [ + { + filename: 'bulkzip_029485858/L340_5ty_2/permitivity_master_template.xlsx', + errors: 'This had been curated already' + }, + { + filename: 'bulkzip_029485858/L340_5ty_2/master_template.xlsx', + errors: { + 'real_permittivity.csv': 'file not uploaded', + 'loss_permittivity.csv': 'file not uploaded' + } + } + ] +} + +global.fetch = jest.fn(() => + Promise.resolve({ + json: () => Promise.resolve(mockValues), + statusText: 'OK', + status: 200 + }) +) + +describe('SpreadsheetUploadBulk.vue', () => { + let wrapper + beforeEach(async () => { + wrapper = await createWrapper(SpreadsheetUploadBulk, { + mocks: { + $apollo: { + loading: false + } + }, + stubs: { + MdPortal: { template: '
' } + } + }, true) + await wrapper.setData({ verifyUser: apollo.verifyUser }) + }) + + enableAutoDestroy(afterEach) + + it('renders steppers', () => { + expect.assertions(1) + const steppers = wrapper.findAll('.md-stepper') + expect(steppers.length).toBe(3) + }) + + it('provides link to download template', () => { + expect.assertions(3) + const steppers = wrapper.findAll('.md-stepper') + expect(steppers.at(0).text()).toContain('Click here to download the template spreadsheet, and fill it out with your data.') + const downloadLinks = steppers.at(0).findAll('a') + expect(downloadLinks.at(0).exists()).toBe(true) + expect(downloadLinks.at(0).html()).toContain('href') + }) + + it('contains drop area for zip file', () => { + expect.assertions(1) + const steppers = wrapper.findAll('.form__drop-area') + expect(steppers.length).toBe(1) + }) + + it('verifies provided information', async () => { + expect.assertions(1) + await wrapper.setData({ spreadsheetFiles: testFiles }) + const verificationStep = wrapper.findAll('.md-stepper').at(2) + expect(verificationStep.text()).toContain(testFiles[0].file.name) + }) + + it('renders a submit button', () => { + expect.assertions(1) + const submitButton = wrapper.find('#submit') + expect(submitButton.exists()).toBe(true) + }) + + it('calls submit functions', async () => { + expect.assertions(2) + const submitFiles = jest.spyOn(wrapper.vm, 'submitFiles') + + const submitButton = wrapper.find('#submit') + await submitButton.trigger('click') + + expect(submitFiles).toHaveBeenCalledTimes(1) + expect(wrapper.text()).toContain('Upload in progress') + }) + + it('renders results', async () => { + expect.assertions(6) + await wrapper.setData({ submitted: true, uploadInProgress: false, uploadResponse: mockValues }) + + // Successful curations + const cards = wrapper.findAll('.md-card') + expect(cards.length).toBe(2) + expect(cards.at(0).text()).toContain(mockValues.bulkCurations[0].sampleID) + + // Errors + const list = wrapper.find('.md-list') + expect(list.exists()).toBe(true) + expect(list.text()).toContain('master_template.xlsx') + expect(list.text()).toContain(mockValues.bulkErrors[0].errors) + expect(wrapper.findAll('.md-list-item').length).toBe(3) + }) +}) diff --git a/app/tests/unit/pages/explorer/XMLSpreadsheetUpload.spec.js b/app/tests/unit/pages/explorer/XMLSpreadsheetUpload.spec.js index a7b99ecb..00fd507c 100644 --- a/app/tests/unit/pages/explorer/XMLSpreadsheetUpload.spec.js +++ b/app/tests/unit/pages/explorer/XMLSpreadsheetUpload.spec.js @@ -10,6 +10,41 @@ const apollo = { } } } +const testSpreadsheet = [{ + file: { name: 'master_template.xlsx' }, + id: 'master_template.xlsx', + status: 'incomplete' +}] +const testFiles = [{ + file: { name: 'fakedata.csv' }, + id: 'fakedata.csv', + status: 'incomplete' +}, { + file: { name: 'fakeimage.jpeg' }, + id: 'fakeimage.jpeg', + status: 'incomplete' +}] + +const mockValues = +{ + xml: '', + user: { + _id: '63feb2a02e34b87a5c278ab8', + displayName: 'Anya Wallace' + }, + sampleID: 'L1_S23', + groupId: '123456', + isApproved: false, + status: 'Editing' +} + +global.fetch = jest.fn(() => + Promise.resolve({ + json: () => Promise.resolve(mockValues), + statusText: 'OK', + status: 200 + }) +) describe('SpreadsheetUpload.vue', () => { const defaultProps = { @@ -44,25 +79,37 @@ describe('SpreadsheetUpload.vue', () => { expect(steppers.length).toBe(6) }) + it('provides link to download template', () => { + expect.assertions(3) + const steppers = wrapper.findAll('.md-stepper') + expect(steppers.at(0).text()).toContain('Click here to download the template spreadsheet, and fill it out with your data.') + const downloadLinks = steppers.at(0).findAll('a') + expect(downloadLinks.at(0).exists()).toBe(true) + expect(downloadLinks.at(0).html()).toContain('href') + }) + it('contains drop areas for spreadsheet and supplementary files', () => { expect.assertions(1) - const steppers = wrapper.findAll('.form__drop-area') - expect(steppers.length).toBe(2) + const dropArea = wrapper.findAll('.form__drop-area') + expect(dropArea.length).toBe(2) }) - it.skip('provides field input for doi', () => { + it('provides field input for doi', () => { expect.assertions(2) - const steppers = wrapper.findAll('.md-field') - expect(steppers.length).toBe(1) - expect(steppers.at(0).text()).toContain('DOI') + const fields = wrapper.findAll('.md-field') + expect(fields.length).toBe(1) + expect(fields.at(0).text()).toContain('DOI') }) it('verifies provided information', async () => { - expect.assertions(1) - await wrapper.setData({ doi: '10.000' }) + expect.assertions(4) + await wrapper.setData({ doi: '10.000', spreadsheetFiles: testSpreadsheet, suppFiles: testFiles }) const verificationStep = wrapper.findAll('.md-stepper').at(4) - // TODO: test for files expect(verificationStep.html()).toContain('10.000') + expect(verificationStep.text()).toContain(testSpreadsheet[0].file.name) + for (const index in testFiles) { + expect(verificationStep.text()).toContain(testFiles[index].file.name) + } }) it('provides a button for changing dataset ID', async () => { @@ -76,4 +123,20 @@ describe('SpreadsheetUpload.vue', () => { const submitButton = wrapper.find('#submit') expect(submitButton.exists()).toBe(true) }) + + it('calls submit functions', async () => { + expect.assertions(3) + const submitFiles = jest.spyOn(wrapper.vm, 'submitFiles') + const createSample = jest.spyOn(wrapper.vm, 'createSample') + + const submitButton = wrapper.find('#submit') + await submitButton.trigger('click') + + const confirmButton = wrapper.find('#confirmSubmit') + expect(confirmButton.exists()).toBe(true) + await confirmButton.trigger('click') + + expect(submitFiles).toHaveBeenCalledTimes(1) + expect(createSample).toHaveBeenCalledTimes(1) + }) }) diff --git a/docker-compose.yml b/docker-compose.yml index 28f9e402..b6f62a45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,14 +38,14 @@ services: - ./mockDB/es:/usr/share/elasticsearch/data minio: container_name: minio - image: quay.io/minio/minio:RELEASE.2022-03-17T06-34-49Z - command: server /data - hostname: minio + image: minio/minio + command: server --console-address ":9001" /data environment: - MINIO_ROOT_USER=${MINIO_ROOT_USER} - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} ports: - "9000:9000" + - "9001:9001" volumes: - ./mockDB/minio:/data proxy: @@ -63,6 +63,7 @@ services: depends_on: - es - mongo + - minio container_name: restful restart: always build: @@ -82,6 +83,10 @@ services: - ESADDRESS=${ESADDRESS} - PORT=${PORT} - MINIO_PORT=${MINIO_PORT} + - MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT} + - MINIO_BUCKET=${MINIO_BUCKET} + - MINIO_ROOT_USER=${MINIO_ROOT_USER} + - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} - ROUTER=${HOST_PORT} - MM_RUNTIME_ENV=${MM_RUNTIME_ENV} - MM_USER=${MM_USER} diff --git a/nginx/default.conf b/nginx/default.conf index 1c5b3255..c0ebd096 100644 --- a/nginx/default.conf +++ b/nginx/default.conf @@ -8,8 +8,8 @@ upstream api { server api:3001; } -upstream minio { - server minio:9000; +upstream console { + server minio:9001; } server { @@ -20,6 +20,7 @@ server { } location /sockjs-node { + rewrite /sockjs-node/(.*) /$1 break; proxy_pass http://client; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; @@ -38,33 +39,54 @@ server { location /api { rewrite /api/(.*) /$1 break; + client_max_body_size 1G; proxy_pass http://api; proxy_read_timeout 1500; proxy_connect_timeout 1500; proxy_send_timeout 1500; send_timeout 1500; + + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + chunked_transfer_encoding off; } +} - location /storage { - sendfile on; - keepalive_timeout 60; - default_type application/octet-stream; +server { + listen 9001; + listen [::]:9001; + server_name localhost; - client_max_body_size 0; - # To disable buffering - proxy_buffering off; - proxy_request_buffering off; + # To allow special characters in headers + # ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + # client_max_body_size 0; + # To disable buffering + # proxy_buffering off; + location / { proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + + # This is necessary to pass the correct IP to be hashed + real_ip_header X-Real-IP; proxy_connect_timeout 300; + + # To support websocket proxy_http_version 1.1; - proxy_set_header Connection ""; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + chunked_transfer_encoding off; - proxy_pass http://minio; + proxy_pass http://console; } } diff --git a/resfulservice/config/constant.js b/resfulservice/config/constant.js index c6aabc66..bd286e1a 100644 --- a/resfulservice/config/constant.js +++ b/resfulservice/config/constant.js @@ -26,5 +26,17 @@ module.exports = { 'CHARACTERIZATION METHODS': 'CHARACTERIZATION', MICROSTRUCTURE: 'MICROSTRUCTURE' }, - ContactPagePurposeOpt: ['QUESTION', 'TICKET', 'SUGGESTION', 'COMMENT'] + ContactPagePurposeOpt: ['QUESTION', 'TICKET', 'SUGGESTION', 'COMMENT'], + SupportedFileTypes: ['png', 'jpg', 'jpeg', 'tiff', 'tif', 'csv', 'zip', 'xls', 'xlsx'], + SupportedFileResponseHeaders: { + '.csv': 'text/csv', + '.png': 'image/png', + '.jpg': 'image/jpg', + '.jpeg': 'image/jpeg', + '.tiff': 'image/tiff', + '.tif': 'image/tif', + '.xls': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + }, + MinioBucket: 'mgi' }; diff --git a/resfulservice/package.json b/resfulservice/package.json index 6e06cfec..1290f3c2 100644 --- a/resfulservice/package.json +++ b/resfulservice/package.json @@ -17,12 +17,14 @@ "axios": "^0.26.1", "bcryptjs": "^2.4.3", "csv-parser": "^3.0.0", + "decompress": "^4.2.1", "express": "^4.17.1", "express-validator": "^6.14.0", "graphql": "^15.3.0", "graphql-ws": "^5.8.2", "helmet": "^4.6.0", "jsonwebtoken": "^8.5.1", + "minio": "^7.1.1", "mongodb": "^4.2.1", "mongoose": "^6.1.4", "morgan": "^1.10.0", diff --git a/resfulservice/spec/controllers/adminController.spec.js b/resfulservice/spec/controllers/adminController.spec.js index 5ecbcf44..37fbbcb3 100644 --- a/resfulservice/spec/controllers/adminController.spec.js +++ b/resfulservice/spec/controllers/adminController.spec.js @@ -2,7 +2,7 @@ const chai = require('chai'); const sinon = require('sinon'); const DatasetProperty = require('../../src/models/datasetProperty'); const { getDatasetProperties } = require('../../src/controllers/adminController'); - +const { next } = require('../mocks'); const { expect } = chai; const mockDatasetProperties = [ @@ -53,9 +53,6 @@ describe('Admin Controllers Unit Tests:', function() { context('getDatasetProperties', () => { it('should return a 400 error if no search query params', async function() { - const next = function (fn) { - return fn; - }; sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returnsThis(); const result = await getDatasetProperties(req, res, next); @@ -66,9 +63,6 @@ describe('Admin Controllers Unit Tests:', function() { it('should return a list of filtered dataset properties', async function() { req.query = { search: 'Loss' } - const next = function (fn) { - return fn; - }; sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returns({data: mockDatasetProperties}); sinon.stub(DatasetProperty, 'find').returns(fetchedDatasetProperties) @@ -79,9 +73,7 @@ describe('Admin Controllers Unit Tests:', function() { it.skip('should return a 500 server error', async function() { req.query = { search: 'Loss' } - const next = function (fn) { - return fn; - }; + sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returns({message: 'Server Error'}); sinon.stub(DatasetProperty, 'find').throws('Error connecting to database'); diff --git a/resfulservice/spec/controllers/curationController.spec.js b/resfulservice/spec/controllers/curationController.spec.js index 848f603a..c97b735f 100644 --- a/resfulservice/spec/controllers/curationController.spec.js +++ b/resfulservice/spec/controllers/curationController.spec.js @@ -5,9 +5,11 @@ const Xmljs = require('xml-js'); const { user, correctXlsxFile, + mockBulkCurationZipFile, wrongXlsxFile, mockCurationList, mockCuratedXlsxObject, + mockCurateObject, fetchedCuratedXlsxObject, mockSheetData, mockSheetData2, @@ -25,16 +27,24 @@ const { mockDatasetId, mockXmlData, mockCSVData, - mockCurateObject + mockUnzippedFolder, + mockCurationError, + mockBulkCuration1, + mockBulkCuration2, + mockReadFolder, + next } = require('../mocks') const XlsxObject = require('../../src/models/curatedSamples'); const XlsxCurationList = require('../../src/models/xlsxCurationList'); const DatasetId = require('../../src/models/datasetId'); const XmlData = require('../../src/models/xmlData'); const XlsxFileManager = require('../../src/utils/curation-utility'); +const FileManager = require('../../src/utils/fileManager'); const XlsxController = require('../../src/controllers/curationController'); const { createMaterialObject } = require('../../src/controllers/curationController') -const { logger } = require('../common/utils') +const { logger } = require('../common/utils'); +const latency = require('../../src/middlewares/latencyTimer'); +const FileStorage = require('../../src/middlewares/fileStorage'); const { expect } = chai; @@ -55,15 +65,8 @@ describe('Curation Controller', function() { send: () => {} }; - const next = function (fn) { - return fn; - }; - context('curateXlsxSpreadsheet', () => { it('should return a 400 error if no file is uploaded', async function() { - const next = function (fn) { - return fn; - }; sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returnsThis(); const result = await XlsxController.curateXlsxSpreadsheet(req, res, next); @@ -74,9 +77,6 @@ describe('Curation Controller', function() { it('should return a 400 error if master_template.xlsx file is not uploaded', async function() { req.files.uploadfile = wrongXlsxFile - const next = function (fn) { - return fn; - }; sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returnsThis(); const result = await XlsxController.curateXlsxSpreadsheet(req, res, next); @@ -85,43 +85,23 @@ describe('Curation Controller', function() { expect(result.message).to.equal('Master template xlsx file not uploaded', 'createXlsxObject'); }); - it('should return a 400 error if dataset query is not added', async function() { - req.files.uploadfile = correctXlsxFile; - req.query = { dataset: "" } - const next = function (fn) { - return fn; - }; - sinon.stub(res, 'status').returnsThis(); - sinon.stub(res, 'json').returnsThis(); - const result = await XlsxController.curateXlsxSpreadsheet(req, res, next); - - expect(result).to.have.property('message'); - expect(result.message).to.equal('Missing dataset ID in query', 'createXlsxObject'); - }); - - it('should return a 400 error if provided dataset ID is not found in the database', async function() { + it('should return a 404 error if provided dataset ID is not found in the database', async function() { req.files.uploadfile = correctXlsxFile; req.query = { dataset: '583e3d6ae74a1d205f4e3fd3' } - const next = function (fn) { - return fn; - }; sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returnsThis(); sinon.stub(XlsxObject, 'find').returns([]); sinon.stub(XlsxCurationList, 'find').returns(mockCurationList); + sinon.stub(XlsxController, 'createMaterialObject').returns(mockCuratedXlsxObject); sinon.stub(DatasetId, 'findOne').returns(null); const result = await XlsxController.curateXlsxSpreadsheet(req, res, next); expect(result).to.have.property('message'); expect(result.message).to.equal(`A sample must belong to a dataset. Dataset ID: ${req.query.dataset ?? null} not found`, 'createXlsxObject'); }); - it('should return a 400 error if error is found while processing the parsing spreadsheet', async function() { req.files.uploadfile = correctXlsxFile; req.query = { dataset: "583e3d6ae74a1d205f4e3fd3" } - const next = function (fn) { - return fn; - }; sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returns({ errors: { Origin: 'invalid value' } }); sinon.stub(XlsxObject, 'find').returns([]); @@ -133,11 +113,21 @@ describe('Curation Controller', function() { expect(result).to.have.property('errors'); }); + it('should return a 400 error if error is found while processing the parsing spreadsheet', async function() { + req.files.uploadfile = correctXlsxFile; + req.query = { dataset: "583e3d6ae74a1d205f4e3fd3" } + sinon.stub(XlsxObject, 'find').returns([]); + sinon.stub(XlsxCurationList, 'find').returns(mockCurationList); + sinon.stub(DatasetId, 'findOne').returns(mockDatasetId); + sinon.stub(XlsxController, 'createMaterialObject').returns( { count: 1, errors: { Origin: 'invalid value' }}); + + const result = await XlsxController.curateXlsxSpreadsheet({ ...req, isParentFunction: true }, res, next); + expect(result).to.have.property('errors'); + }); + + it('should return a 409 conflict error if curated sheet has same title and publication year', async function() { req.files.uploadfile = correctXlsxFile; - const next = function (fn) { - return fn; - }; sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returnsThis(); sinon.stub(DatasetId, 'findOne').returns(mockDatasetId); @@ -151,16 +141,16 @@ describe('Curation Controller', function() { it('should curate master template', async function() { req.files.uploadfile = correctXlsxFile; - const next = function (fn) { - return fn; - }; + req.query = { dataset: null } sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returns(mockCurateObject); sinon.stub(XlsxObject, 'find').returns([]); sinon.stub(XlsxCurationList, 'find').returns(mockCurationList); - sinon.stub(DatasetId, 'findOne').returns({...mockDatasetId, updateOne: sinon.stub().returns(true)}); + sinon.stub(DatasetId, 'findOne').returns(null); + sinon.stub(DatasetId, 'create').returns({...mockDatasetId, updateOne: sinon.stub().returns(true)}) sinon.stub(XlsxController, 'createMaterialObject').returns(mockCuratedXlsxObject); sinon.stub(XlsxObject.prototype, 'save').callsFake(() => (fetchedCuratedXlsxObject)) + sinon.stub(latency, 'latencyCalculator').returns(true) sinon.stub(Xmljs, 'json2xml').returns(fetchedCuratedXlsxObject) const result = await XlsxController.curateXlsxSpreadsheet(req, res, next); @@ -169,6 +159,23 @@ describe('Curation Controller', function() { expect(result).to.have.property('user'); }); + it('should curate master template when called by bulk controller', async function() { + req.files.uploadfile = correctXlsxFile; + req.isParentFunction = true; + sinon.stub(XlsxObject, 'find').returns([]); + sinon.stub(XlsxCurationList, 'find').returns(mockCurationList); + sinon.stub(DatasetId, 'findOne').returns({...mockDatasetId, updateOne: sinon.stub().returns(true)}); + sinon.stub(XlsxController, 'createMaterialObject').returns(mockCuratedXlsxObject); + sinon.stub(XlsxObject.prototype, 'save').callsFake(() => (fetchedCuratedXlsxObject)) + sinon.stub(Xmljs, 'json2xml').returns(fetchedCuratedXlsxObject) + + const result = await XlsxController.curateXlsxSpreadsheet(req, res, next); + + expect(result).to.have.property('curatedSample'); + expect(result).to.have.property('processedFiles'); + }); + + it('should return a 500 server error when database throws an error', async function() { req.files.uploadfile = correctXlsxFile; const nextSpy = sinon.spy(); @@ -181,23 +188,117 @@ describe('Curation Controller', function() { }); }) + context('bulkXlsxCurations', () => { + it('should return a 400 error if zip file is not uploaded', async function() { + req.files.uploadfile = wrongXlsxFile + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returnsThis(); + const result = await XlsxController.bulkXlsxCurations(req, res, fn => fn); + + expect(result).to.have.property('message'); + expect(result.message).to.equal('bulk curation zip file not uploaded'); + }); + + it('should bulk curate folders when curation returns errors', async function() { + req.files.uploadfile = mockBulkCurationZipFile; + req.query = { dataset: mockDatasetId._id } + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returns(mockBulkCuration1); + sinon.stub(DatasetId, 'findOne').returns(mockDatasetId); + sinon.stub(XlsxFileManager, 'unZipFolder').returns(mockUnzippedFolder); + sinon.stub(XlsxController, 'curateXlsxSpreadsheet').returns(mockCurationError); + sinon.stub(XlsxFileManager, 'readFolder').returns({ ...mockReadFolder, folders: [] }); + sinon.stub(FileStorage, 'minioPutObject').returns(true); + sinon.stub(FileManager, 'deleteFile').returns(true); + sinon.stub(FileManager, 'deleteFolder').returns(true); + sinon.stub(latency, 'latencyCalculator').returns(true) + + const result = await XlsxController.bulkXlsxCurations(req, res, fn => fn); + + expect(result).to.have.property('bulkCurations'); + expect(result).to.have.property('bulkErrors'); + expect(result.bulkCurations).to.be.an('Array'); + expect(result.bulkErrors).to.be.an('Array'); + }); + + it('should return 404 not found error when datasetId provided is not present in the database', async function() { + req.files.uploadfile = mockBulkCurationZipFile; + req.query = { dataset: mockDatasetId._id } + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returns(mockBulkCuration1); + sinon.stub(DatasetId, 'findOne').returns(null); + // sinon.stub(XlsxFileManager, 'unZipFolder').returns(mockReadFolder); + // sinon.stub(XlsxController, 'curateXlsxSpreadsheet').returns(mockCurationError); + // sinon.stub(XlsxFileManager, 'readFolder').returns(mockUnzippedFolder); + // sinon.stub(latency, 'latencyCalculator').returns(true) + + const result = await XlsxController.bulkXlsxCurations(req, res, fn => fn); + expect(result).to.have.property('message'); + expect(result.message).to.equal(`Dataset ID: ${req.query.dataset ?? null} not found`, 'bulkXlsxCurations'); + }); + + it('should bulk curate folders when successful curation when files and folders in root folder', async function() { + req.files.uploadfile = mockBulkCurationZipFile; + req.query = { dataset: null } + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returns(mockBulkCuration2); + sinon.stub(DatasetId, 'findOne').returns(mockDatasetId); + sinon.stub(XlsxFileManager, 'unZipFolder').returns(mockUnzippedFolder); + sinon.stub(XlsxController, 'curateXlsxSpreadsheet').returns({curatedSample: mockCurateObject, processedFiles: mockReadFolder.curationFiles }); + const readFolderStub = sinon.stub(XlsxFileManager, 'readFolder'); + readFolderStub.onFirstCall().returns(mockReadFolder); + readFolderStub.onSecondCall().returns({ ...mockReadFolder, folders: [] }); + sinon.stub(FileManager, 'deleteFolder').returns(true); + sinon.stub(FileManager, 'deleteFile').returns(true); + sinon.stub(latency, 'latencyCalculator').returns(true) + sinon.stub(FileStorage, 'minioPutObject').returns(true); + + + const result = await XlsxController.bulkXlsxCurations(req, res, fn => fn); + + expect(result).to.have.property('bulkCurations'); + expect(result).to.have.property('bulkErrors'); + expect(result.bulkCurations).to.be.an('Array'); + expect(result.bulkErrors).to.be.an('Array'); + }); + + it('should return a 500 server error when database throws an error', async function() { + req.files.uploadfile = mockBulkCurationZipFile; + const nextSpy = sinon.spy(); + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returnsThis(); + sinon.stub(XlsxFileManager, 'unZipFolder').returns(mockUnzippedFolder); + sinon.stub(XlsxController, 'curateXlsxSpreadsheet').throws(); + sinon.stub(latency, 'latencyCalculator').returns(true) + + await XlsxController.bulkXlsxCurations(req, res, nextSpy); + sinon.assert.calledOnce(nextSpy); + }); + }) + context('Retrieve curations', () => { - it('should return 404 not found error if req.param ID is invalid', async () => { - req.params = { xmlId: 'null', xlsxObjectId: 'a90w49a40ao4094k4aed'} + it('should return 404 not found error if req.params ID is invalid', async () => { + req.query = { xlsxObjectId: 'a90w49a40ao4094k4aed'} sinon.stub(XlsxObject, 'findOne').returns(null); const result = await XlsxController.getXlsxCurations(req, res, next); expect(result).to.have.property('message'); expect(result.message).to.equal('Curation sample not found'); }) + it('should return 404 not found error if req.param ID is invalid', async () => { + req.query = { xmlId: 'a90w49a40ao4094k4aed'} + sinon.stub(XmlData, 'findOne').returns(null); + const result = await XlsxController.getXlsxCurations(req, res, next); + expect(result).to.have.property('message'); + expect(result.message).to.equal('Sample xml not found'); + }); + it('returns a curation data when a valid req.param ID is provided', async () => { - req.params = { xlsxObjectId: 'a90w49a40ao4094k4aed'} - const next = function (fn) { - return fn; - }; + req.query = { xlsxObjectId: 'a90w49a40ao4094k4aed'} sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returns(fetchedCuratedXlsxObject); sinon.stub(XlsxObject, 'findOne').returns(fetchedCuratedXlsxObject); + sinon.stub(latency, 'latencyCalculator').returns(true) const result = await XlsxController.getXlsxCurations(req, res, next); @@ -207,14 +308,12 @@ describe('Curation Controller', function() { }); it('returns curated object when an ID is provided and user is admin', async () => { - req.params = { xlsxObjectId: 'a90w49a40ao4094k4aed'}; + req.query = { xlsxObjectId: 'a90w49a40ao4094k4aed'}; req.user.roles = 'admin' - const next = function (fn) { - return fn; - }; sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returns(fetchedCuratedXlsxObject); sinon.stub(XlsxObject, 'findOne').returns(fetchedCuratedXlsxObject); + sinon.stub(latency, 'latencyCalculator').returns(true) const result = await XlsxController.getXlsxCurations(req, res, next); @@ -222,28 +321,22 @@ describe('Curation Controller', function() { expect(result).to.have.property('object'); expect(result).to.have.property('user'); }) - it('should return curation object when an xmlId is provided', async () => { - req.params = { xmlId: 'a90w49a40ao4094k4aed', xlsxObjectId: null } - const next = function (fn) { - return fn; - }; - sinon.stub(res, 'status').returnsThis(); - sinon.stub(res, 'json').returns(mockCuratedXlsxObject); - sinon.stub(XmlData, 'findOne').returns(mockXmlData); - sinon.stub(Xmljs, 'xml2json').returns(mockCuratedXlsxObject) - - const result = await XlsxController.getXlsxCurations(req, res, next); - - expect(result).to.be.an('Object'); - expect(result).to.have.property('DATA_SOURCE'); - }); + req.query = { xmlId: 'a90w49a40ao4094k4aed' } + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returns(mockCuratedXlsxObject); + sinon.stub(XmlData, 'findOne').returns(mockXmlData); + sinon.stub(Xmljs, 'xml2json').returns(mockCuratedXlsxObject) + sinon.stub(latency, 'latencyCalculator').returns(true) + + const result = await XlsxController.getXlsxCurations(req, res, next); + + expect(result).to.be.an('Object'); + expect(result).to.have.property('DATA_SOURCE'); + }); it('returns list of curations when an ID is not provided', async () => { - req.params = { xlsxObjectId: null, xmlId: null } - const next = function (fn) { - return fn; - }; + req.query = { } sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returns(Array(3).fill(fetchedCuratedXlsxObject)); sinon.stub(XlsxObject, 'find').returns({ object: Array(3).fill(fetchedCuratedXlsxObject), select: sinon.stub().returnsThis()}); @@ -255,7 +348,7 @@ describe('Curation Controller', function() { }) it('should return a 500 server error when database throws an error', async function() { - req.params = { xlsxObjectId: 'a90w49a40ao4094k4aed'} + req.query = { xlsxObjectId: 'a90w49a40ao4094k4aed'} const nextSpy = sinon.spy(); sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returnsThis(); @@ -273,20 +366,21 @@ describe('Curation Controller', function() { sinon.stub(XlsxObject, 'findOne').returns(null); const result = await XlsxController.updateXlsxCurations(req, res, next); expect(result).to.have.property('message'); - expect(result.message).to.equal(`Curated sample ID: ${req.params.xlsxObjectId} not found`); + expect(result.message).to.equal(`Curated sample ID: ${req.query.xlsxObjectId} not found`); }); - it.skip('Should return a message "No changes" if no changes occurred with submitted payload', async () => { + it('Should return a message "No changes" if no changes occurred with submitted payload', async () => { req.body = { payload: mockBaseObject } req.query = { xlsxObjectId: 'a90w49a40ao4094k4aed'} sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returns({ message: "No changes"}); + sinon.stub(util, 'isDeepStrictEqual').returns(true); sinon.stub(XlsxObject, 'findOne').returns(fetchedCuratedXlsxObject); const result = await XlsxController.updateXlsxCurations(req, res, next); expect(result).to.have.property('message'); - expect(result.message).to.equal('No changes'); + expect(result.message).to.equal("No changes"); }) it('should update curation object with submitted payload', async () => { @@ -337,9 +431,6 @@ describe('Curation Controller', function() { it('deletes a curation when a valid req.query curation ID is provided', async () => { req.query = { xlsxObjectId: 'a90w49a40ao4094k4aed'} - const next = function (fn) { - return fn; - }; sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returns({message: `Curated sample ID: ${req.query.xlsxObjectId} successfully deleted`}); sinon.stub(XlsxObject, 'findOneAndDelete').returns(fetchedCuratedXlsxObject); @@ -353,9 +444,6 @@ describe('Curation Controller', function() { it('deletes multiple curated objects and datasetID when a valid dataset ID query is provided', async () => { req.query = { dataset: 'a90w49a40ao4094k4aed' }; - const next = function (fn) { - return fn; - }; sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returns({ message: `Curated Samples with Dataset ID: ${req.query.dataset} successfully deleted`}); sinon.stub(DatasetId, 'findOneAndDelete').returns(mockDatasetId); @@ -368,7 +456,7 @@ describe('Curation Controller', function() { }) it('should return a 500 server error when database throws an error', async function() { - req.query = { xmlId: 'null', xlsxObjectId: 'a90w49a40ao4094k4aed'} + req.query = { xlsxObjectId: 'a90w49a40ao4094k4aed'} const nextSpy = sinon.spy(); sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returnsThis(); @@ -388,7 +476,6 @@ describe('Curation Controller', function() { expect(error).to.have.property('count'); expect(error).to.have.property('errors'); }); - it('should return parsed and filtered xlsx object 1', async () => { sinon.stub(XlsxFileManager, 'xlsxFileReader').returns(mockSheetData2); const result = await XlsxController.createMaterialObject(correctXlsxFile[0].path, mockJsonStructure, mockCurationListMap); @@ -396,7 +483,6 @@ describe('Curation Controller', function() { expect(result).to.be.an('Object') expect(result).to.have.property('Your Name'); }); - it('should return parsed and filtered xlsx object for varied_multiple types', async () => { sinon.stub(XlsxFileManager, 'xlsxFileReader').returns(mockSheetData5); const result = await XlsxController.createMaterialObject(correctXlsxFile[0].path, mockJsonStructure5, mockCurationListMap); @@ -404,27 +490,24 @@ describe('Curation Controller', function() { expect(result).to.be.an('Object') expect(result).to.have.property('MeltMixing'); }); - it('should return parsed and filtered xlsx object 2', async () => { sinon.stub(XlsxFileManager, 'xlsxFileReader').returns(mockSheetData3); sinon.stub(XlsxFileManager, 'parseCSV').returns(mockCSVData); - const result = await XlsxController.createMaterialObject(correctXlsxFile[0].path, mockJsonStructure2, mockCurationListMap, mockUploadedFiles); + const result = await XlsxController.createMaterialObject(correctXlsxFile[0].path, mockJsonStructure2, mockCurationListMap, mockUploadedFiles, []); expect(result).to.be.an('Object'); expect(result).to.have.property('DMA Datafile'); expect(result['DMA Datafile']).to.be.an('Array'); - // expect(result['DMA Datafile'][1]).to.have.property('data'); }); it('should return parsed and filtered xlsx object 3 handling default', async () => { sinon.stub(XlsxFileManager, 'xlsxFileReader').returns(mockSheetData4); - const result = await XlsxController.createMaterialObject(correctXlsxFile[0].path, mockJsonStructure4, mockCurationListMap, mockUploadedFiles); + const result = await XlsxController.createMaterialObject(correctXlsxFile[0].path, mockJsonStructure4, mockCurationListMap, mockUploadedFiles, []); expect(result).to.be.an('Object') expect(result).to.have.property('Microstructure'); expect(result.Microstructure).to.have.property('Imagefile'); expect(result.Microstructure.Imagefile).to.be.an('Array'); }); - it('should return error when file is not uploaded', async () => { sinon.stub(XlsxFileManager, 'xlsxFileReader').returns(mockSheetData3); @@ -439,9 +522,6 @@ describe('Curation Controller', function() { context('Get curation base schema object', () => { it('should return a partial json structure when query param is present', async () => { req.query = { sheetName: 'Data Origin'} - const next = function (fn) { - return fn; - }; sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returns(mockJsonStructure); @@ -453,9 +533,6 @@ describe('Curation Controller', function() { it('Returns the whole base schema object structure if no query param is provided', async () => { req.query = { sheetName: '' } - const next = function (fn) { - return fn; - }; sinon.stub(res, 'status').returnsThis(); sinon.stub(res, 'json').returns(mockJsonStructure2); diff --git a/resfulservice/spec/controllers/fileController.spec.js b/resfulservice/spec/controllers/fileController.spec.js new file mode 100644 index 00000000..fa9da4cf --- /dev/null +++ b/resfulservice/spec/controllers/fileController.spec.js @@ -0,0 +1,204 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const { logger } = require('../common/utils'); +const { next, mockFSFiles, mockBucket, mockDownloadStream, mockEmptyStream, mockFindObjectStream } = require('../mocks'); +const FileController = require('../../src/controllers/fileController'); +const FsFile = require('../../src/models/fsFiles'); +const FileManager = require('../../src/utils/fileManager') +const latency = require('../../src/middlewares/latencyTimer'); +const minioClient = require('../../src/utils/minio'); + + +describe('File Controller Unit Tests:', function() { + afterEach(() => sinon.restore()); + + const req = { + logger, + files: {}, + env: { FILES_DIRECTORY: 'file_directory' } + } + + const res = { + header: () => {}, + status: () => {}, + json: () => {}, + send: () => {}, + setHeader: () => {}, + sendStatus: sinon.spy() + }; + + context('imageMigration', () => { + it('should return success when files successful upload', async () => { + + req.params = { imageType: 'charts'}; + const files = [[{ path: '/images/cat.png'}, { path: '/images/dog.png'}]]; + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returns({ images: files }); + sinon.stub(FileController, 'connectToMongoBucket').returns(mockBucket); + sinon.stub(latency, 'latencyCalculator').returns(true) + const result = await FileController.imageMigration(req, res, next); + expect(result).to.have.property('images'); + }); + it('should return a 500 server error', async function() { + const nextSpy = sinon.spy(); + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returnsThis(); + sinon.stub(FileController, 'connectToMongoBucket').throws(); + await FileController.imageMigration(req, res, nextSpy); + sinon.assert.calledOnce(nextSpy); + }); + }); + + context('fileContent', () => { + req.params = { fileId: '638dd8e9af9d478e0136ffcb' }; + + it('should successfully stream files from mongo bucket', async () => { + req.query = { isDirectory: false, isBucket: false }; + const files = [[{ path: '/images/cat.png'}, { path: '/images/dog.png'}]]; + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returns({ images: files }); + sinon.stub(FsFile, 'findById').returns(mockFSFiles); + sinon.stub(FileController, 'connectToMongoBucket').returns(mockBucket); + sinon.stub(res, 'setHeader').returns(true); + sinon.stub(latency, 'latencyCalculator').returns(true) + + await FileController.fileContent(req, res, next); + sinon.assert.calledOnce(mockDownloadStream.pipe); + }); + + it('should empty stream files from mongo bucket', async () => { + req.query = { isDirectory: false, isBucket: false }; + const files = [[{ path: '/images/cat.png'}, { path: '/images/dog.png'}]]; + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returns({ images: files }); + sinon.stub(FsFile, 'findById').returns({ limit: sinon.stub().returns(null)}); + sinon.stub(FileController, 'connectToMongoBucket').returns(mockBucket); + sinon.stub(FileController, '_createEmptyStream').returns(mockEmptyStream); + sinon.stub(res, 'setHeader').returns(true); + sinon.stub(latency, 'latencyCalculator').returns(true) + + await FileController.fileContent(req, res, next); + sinon.assert.calledOnce(mockEmptyStream.pipe); + }); + + it.skip('should successfully stream file from the file system', async () => { + req.query = { isDirectory: true, isBucket: false }; + const files = [[{ path: '/images/cat.png'}, { path: '/images/dog.png'}]]; + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returns({ images: files }); + sinon.stub(FileManager, 'findFile').returns(mockDownloadStream); + sinon.stub(res, 'setHeader').returns(true); + sinon.stub(latency, 'latencyCalculator').returns(true) + + await FileController.fileContent(req, res, next); + sinon.assert.calledTwice(mockDownloadStream.pipe); + }); + + it.skip('should return empty stream file from the file system', async () => { + req.query = { isDirectory: true, isBucket: false }; + const files = [[{ path: '/images/cat.png'}, { path: '/images/dog.png'}]]; + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returns({ images: files }); + sinon.stub(FileManager, 'findFile').returns(null); + sinon.stub(FileController, '_createEmptyStream').returns(mockEmptyStream); + sinon.stub(res, 'setHeader').returns(true); + sinon.stub(latency, 'latencyCalculator').returns(true) + + await FileController.fileContent(req, res, next); + sinon.assert.calledTwice(mockEmptyStream.pipe); + }); + + it.skip('should successfully stream file from minio bucket', async () => { + req.query = { isDirectory: false, isBucket: true }; + const files = [[{ path: '/images/cat.png'}, { path: '/images/dog.png'}]]; + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returns({ images: files }); + sinon.stub(minioClient, 'getObject').returns(mockDownloadStream); + sinon.stub(res, 'setHeader').returns(true); + sinon.stub(latency, 'latencyCalculator').returns(true) + + await FileController.fileContent(req, res, next); + sinon.assert.calledThrice(mockDownloadStream.pipe); + }); + + it.skip('should return empty stream file from minio bucket', async () => { + req.query = { isDirectory: false, isBucket: true }; + const files = [[{ path: '/images/cat.png'}, { path: '/images/dog.png'}]]; + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returns({ images: files }); + sinon.stub(minioClient, 'getObject').returns(null); + sinon.stub(FileController, '_createEmptyStream').returns(mockEmptyStream); + sinon.stub(res, 'setHeader').returns(true); + sinon.stub(latency, 'latencyCalculator').returns(true) + + await FileController.fileContent(req, res, next); + sinon.assert.calledThrice(mockEmptyStream.pipe); + }); + + it('should return a 500 server error', async function() { + const nextSpy = sinon.spy(); + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returnsThis(); + sinon.stub(FileController, 'connectToMongoBucket').throws(); + await FileController.fileContent(req, res, nextSpy); + sinon.assert.calledOnce(nextSpy); + }); + }); + + + context('uploadFile', () => { + it('should return success when files successfully upload', async () => { + req.files = { uploadfile: [{ path: '/images/cat.png'}, { path: '/images/dog.png'}]}; + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returns({ files: req.files.uploadfile }); + sinon.stub(latency, 'latencyCalculator').returns(true) + + const result = await FileController.uploadFile(req, res, next); + + expect(result).to.have.property('files'); + }); + it('should return a 500 server error', async function() { + const nextSpy = sinon.spy(); + req.files = { uploadfile: [{ path: '/images/cat.png'}, { path: '/images/dog.png'}]}; + sinon.stub(res, 'status').throws(); + // sinon.stub(res, 'json').returns({ data: response.data.hits }); + + await FileController.uploadFile(req, res, nextSpy); + sinon.assert.calledOnce(nextSpy); + }); + }); + + context('deleteFile', () => { + req.params = { fileId: '638dd8e9af9d478e0136ffcb' }; + it('should delete a file from both file system and minio', async () => { + sinon.stub(minioClient, 'removeObject').returns(true); + sinon.stub(FileManager, 'deleteFile').returns(true); + sinon.stub(latency, 'latencyCalculator').returns(true) + + await FileController.deleteFile(req, res, next); + sinon.assert.calledOnce(res.sendStatus); + }) + + it('should return a 500 server error when deleting a file', async function() { + const nextSpy = sinon.spy(); + sinon.stub(res, 'status').returnsThis(); + sinon.stub(res, 'json').returnsThis(); + sinon.stub(minioClient, 'removeObject').throws(); + await FileController.deleteFile(req, res, nextSpy); + sinon.assert.calledOnce(nextSpy); + }); + }) + + context('findFiles', () => { + req.query = { filename: '001.tif'} + it('should find files based on filename query from minio', async () => { + sinon.stub(res, 'status').returnsThis() + sinon.stub(minioClient, 'listObjects').returns(mockFindObjectStream); + sinon.stub(latency, 'latencyCalculator').returns(true) + + FileController.findFiles(req, res, next); + // sinon.assert.calledThrice(mockFindObjectStream.on); + sinon.assert.called(mockFindObjectStream.on); + }) + }) +}) diff --git a/resfulservice/spec/graphql/resolver/dataset.spec.js b/resfulservice/spec/graphql/resolver/dataset.spec.js index f9e1b670..84d86f2b 100644 --- a/resfulservice/spec/graphql/resolver/dataset.spec.js +++ b/resfulservice/spec/graphql/resolver/dataset.spec.js @@ -91,7 +91,7 @@ describe('Dataset Resolver Unit Tests:', function () { expect(createDatasetId.name).to.equal('createDatasetId'); expect(createDatasetId.type.toString()).to.equal('Datasets!'); }); - + it("should throw a 401, not authenticated error", async () => { const result = await createDatasetId({}, { }, { user, req, isAuthenticated: false }); @@ -107,14 +107,16 @@ describe('Dataset Resolver Unit Tests:', function () { expect(datasetId).to.have.property('datasetGroupId'); }); + it("should throw a 409, when an unused datasetId exists", async () => { sinon.stub(DatasetId, 'findOne').returns({_id: '62d951cb6981a12d136a0a0d', status: 'WORK IN PROGRESS', samples: [] }) const result = await createDatasetId({}, { }, { user, req, isAuthenticated: true }); expect(result).to.have.property('extensions'); - expect(result.extensions.code).to.be.equal(409); + expect(result.extensions.code).to.be.equal(409) }); + it('should throw a 500 Internal server error when error is thrown', async () => { sinon.stub(DatasetId, 'findOne').throws(); diff --git a/resfulservice/spec/mocks/curationMock.js b/resfulservice/spec/mocks/curationMock.js index d9611b5d..c6c8a0ad 100644 --- a/resfulservice/spec/mocks/curationMock.js +++ b/resfulservice/spec/mocks/curationMock.js @@ -1026,8 +1026,8 @@ const mockCuratedXlsxObject = { DATA_SOURCE: { Citation: { CommonFields: { - YourName: 'Tolulomo Fateye', - YourEmail: 'tolulomo@toluconsulting.com', + YourName: 'John Doe', + YourEmail: 'john@doe.com', Origin: 'experiments', CitationType: 'lab-generated', Author: [ @@ -1330,6 +1330,19 @@ const correctXlsxFile = [ } ]; +const mockBulkCurationZipFile = [ + { + fieldname: 'uploadfile', + originalname: 'curations.zip', + encoding: '7bit', + mimetype: 'application/zip', + destination: 'mm_files', + filename: 'entitled_bobolink_emmaline-2023-06-15T12:02:53.834Z-curations.zip', + path: 'mm_files/entitled_bobolink_emmaline-2023-06-15T12:02:53.834Z-curations.zip', + size: 121836 + } +]; + const mockUploadedFiles = [ { fieldname: 'uploadfile', @@ -1394,11 +1407,23 @@ const mockUploadedFiles = [ ]; const mockDatasetId = { + _id: '583e3d6ae74a1d205f4e3fd3', user: '583e3d6ae44a1d205f4e3fd3', status: 'APPROVED', samples: ['583e3d6ae74a3d205f4e3fd3', '583e3d6ae74a1d205f4e3fd3'] }; +const mockCurateObject = { + xml: '\n \n S10\n S28\n \n \n \n John Doe\n john@doe.com\n experiments\n lab-generated\n Aditya Shanker Prasad\n John Doe\n https://search.proquest.com/openview/eb63d4d6b84b1252971b3e3eec53b97c/1?pq-origsite=gscholar&cbl=51922&diss=y\n Rensselaer Polytechnic Institute\n \n \n \n \n', + user: { + _id: '643931cc6f44b02f01380f7a', + displayName: 'Test' + }, + groupId: '583e3d6ae74a1d205f4e3fd3', + isApproved: 'false,', + status: 'Editing' +}; + const mockXmlData = { _id: '64394c8032bc6325505af6f9', title: 'L183_53_Portschke_2003.xml', @@ -1410,22 +1435,88 @@ const user = { displayName: 'test' }; -const mockCurateObject = { - xml: '\n \n S10\n S28\n \n \n \n Tolulomo Fateye\n tolulomo@toluconsulting.com\n experiments\n lab-generated\n Aditya Shanker Prasad\n Gbolahan Adeleke\n https://search.proquest.com/openview/eb63d4d6b84b1252971b3e3eec53b97c/1?pq-origsite=gscholar&cbl=51922&diss=y\n Rensselaer Polytechnic Institute\n \n \n \n \n', - user: { - _id: '643931cc6f44b02f01380f7a', - displayName: 'Test' +const mockUnzippedFolder = { + folderPath: 'mm_files/bulk-curation-1688982949940', + allfiles: [{ + mode: 33188, + mtime: '2023-05-10T11:43:10.000Z', + path: 'bulk/S10_L1/001.tif', + type: 'file', + data: '' + } + ] +}; + +const mockReadFolder = { + masterTemplates: [ + 'mm_files/bulk-curation-1686834726293/master_template.xlsx', + 'mm_files/bulk-curation-1688984487096/master_template (1).xlsx' + ], + curationFiles: [ + 'mm_files/bulk-curation-1688982949940/Ls-94k-askd/real_permittivity.csv', + 'mm_files/bulk-curation-1688982949940/Ls-94k-askd/loss_permittivity.csv', + 'mm_files/bulk-curation-1688982949940/Ls-94k-askd/tan_delta.csv', + 'mm_files/bulk-curation-1688982949940/Ls-94k-askd/weibull.csv', + 'mm_files/bulk-curation-1688982949940/Ls-94k-askd/001.tif' + ], + folders: ['mm_files/bulk-curation-1686834726293/Ls-94k-askd'] +}; + +const mockCurationError = { + errors: { + 'real_permittivity.csv': 'file not uploaded', + 'loss_permittivity.csv': 'file not uploaded', + 'tan_delta.csv': 'file not uploaded', + 'weibull.csv': 'file not uploaded', + '001.tif': 'file not uploaded' + } +}; + +const mockBulkCuration1 = { + bulkCurations: [], + bulkErrors: [ + { + filename: 'mm_files/bulk-curation-1686834726293/master_template.xlsx', + '001.tif': 'file not uploaded' + } + ] +}; + +const mockBulkCuration2 = { + bulkCurations: [ + mockCurateObject + ], + bulkErrors: [ + { + filename: 'mm_files/bulk-curation-1686834726293/master_template.xlsx', + '001.tif': 'file not uploaded' + } + ] +}; + +const mockRes = { + status: function (code) { + return this; + }, + json: function (object) { + return object; + } }; module.exports = { user, correctXlsxFile, + mockBulkCurationZipFile, wrongXlsxFile, mockCurationList, + mockCurateObject, mockCuratedXlsxObject, fetchedCuratedXlsxObject, mockSheetData, @@ -1445,5 +1536,10 @@ module.exports = { mockDatasetId, mockXmlData, mockCSVData, - mockCurateObject + mockUnzippedFolder, + mockCurationError, + mockBulkCuration1, + mockBulkCuration2, + mockReadFolder, + mockRes }; diff --git a/resfulservice/spec/mocks/fileMock.js b/resfulservice/spec/mocks/fileMock.js new file mode 100644 index 00000000..f59d0874 --- /dev/null +++ b/resfulservice/spec/mocks/fileMock.js @@ -0,0 +1,50 @@ +const sinon = require('sinon'); + +const mockFiles = [[{ path: '/images/cat.png' }, { path: '/images/dog.png' }]]; + +const mockDownloadStream = { + pipe: sinon.spy() +}; + +const mockEmptyStream = { + pipe: sinon.spy() +}; + +const mockFSFiles = { + files: mockFiles, + limit: sinon.stub().returnsThis() +}; + +const mockBucket = { + openDownloadStream: sinon.stub().returns(mockDownloadStream), + find: sinon.stub().returnsThis(), + limit: sinon.stub().returnsThis(), + toArray: sinon.stub().returns(mockFiles) +}; + +const mockFindObjectStream = { + on: sinon.stub().callsFake(function (filePath, cb) { + cb(new Error(), { name: 'code.png' }); + }) +}; + +const mockFileLstat = { + isDirectory: (path) => false, + isFile: (path) => true +}; + +const mockDirectoryLstat = { + isDirectory: (path) => true, + isFile: (path) => false +}; + +module.exports = { + mockBucket, + mockFiles, + mockFSFiles, + mockDownloadStream, + mockEmptyStream, + mockFindObjectStream, + mockFileLstat, + mockDirectoryLstat +}; diff --git a/resfulservice/spec/mocks/index.js b/resfulservice/spec/mocks/index.js index 8dda7568..9b1bf869 100644 --- a/resfulservice/spec/mocks/index.js +++ b/resfulservice/spec/mocks/index.js @@ -1,7 +1,15 @@ const curation = require('./curationMock'); const user = require('./userMock'); +const files = require('./fileMock'); + +// It is re-useable across all tests files +const next = function (fn) { + return fn; +}; module.exports = { ...curation, - ...user + ...user, + ...files, + next }; diff --git a/resfulservice/spec/utils/fileManager.spec.js b/resfulservice/spec/utils/fileManager.spec.js new file mode 100644 index 00000000..1dec5c97 --- /dev/null +++ b/resfulservice/spec/utils/fileManager.spec.js @@ -0,0 +1,104 @@ +const { expect } = require('chai'); +const fs = require('fs'); +const sinon = require('sinon'); +const { mockDownloadStream, mockFileLstat, mockDirectoryLstat } = require('../mocks'); +const { logger } = require('../common/utils'); +const { deleteFile, findFile, deleteFolder } = require('../../src/utils/fileManager'); + +describe.skip('FileManager Utils', function () { + afterEach(() => sinon.restore()); + + const req = { + logger, + env: { + FILES_DIRECTORY: 'files-directory' + }, + params: { + fileId: 'bulk-curation-1689354647298/001.tif' + } + } + + + context('deleteFile', () => { + it('creates logger', async function () { + const req = { logger: { info: sinon.spy() }} + const deleteStub = sinon.stub(fs, 'unlink').callsFake(function (filePath, cb) { + cb(null) + }) + deleteFile('/images/jpeg', req); + sinon.assert.called(deleteStub); + }); + }) + + context('deleteFolder', () => { + it('creates logger', async function () { + const req = { logger: { info: sinon.spy() }} + const deleteStub = sinon.stub(fs, 'rm').callsFake(function (filePath, { recursive, force }, cb) { + cb(null) + }) + deleteFolder('/images', req); + sinon.assert.called(deleteStub); + }); + }) + + context('findFile', () => { + it('should return null if fileId is in a nested directory but does not exist', async () => { + sinon.stub(fs, 'existsSync').returns(false); + const result = await findFile(req); + expect(result).to.equals(null); + }); + + it('should return a fileStream if fileId is in a nested directory and exists', async () => { + sinon.stub(fs, 'existsSync').returns(true); + sinon.stub(fs, 'createReadStream').returns(mockDownloadStream) + const result = await findFile(req); + expect(result).to.have.property('pipe'); + }); + + it('should return null if no files in the directory and fileId is not full path', async () => { + req.params.fileId = '001.tif'; + sinon.stub(fs.promises, 'readdir').returns([]); + const result = await findFile(req); + expect(result).to.equals(null); + }); + + it('should return file if files is found in the root directory and fileId is not full path', async () => { + req.params.fileId = '001.tif'; + sinon.stub(fs.promises, 'readdir').returns(['001.tif']); + sinon.stub(fs.promises, 'lstat').returns(mockFileLstat) + sinon.stub(fs, 'createReadStream').returns(mockDownloadStream) + const result = await findFile(req); + expect(result).to.have.property('pipe'); + }); + + it('should return file if files is found in the nested directory and fileId is not full path', async () => { + req.params.fileId = '001.tif'; + const readFolderStub = sinon.stub(fs.promises, 'readdir') + readFolderStub.onFirstCall().returns(['bulk-curation-1689354647298']); + readFolderStub.onSecondCall().returns(['001.tif']); + const lstatStub = sinon.stub(fs.promises, 'lstat'); + lstatStub.onFirstCall().returns(mockDirectoryLstat); + lstatStub.onSecondCall().returns(mockFileLstat); + sinon.stub(fs, 'createReadStream').returns(mockDownloadStream) + const result = await findFile(req); + expect(result).to.have.property('pipe'); + }); + + it('should return null if file is not found in nested directory and fileId is not full path', async () => { + req.params.fileId = '001.tif'; + const readFolderStub = sinon.stub(fs.promises, 'readdir') + readFolderStub.onFirstCall().returns(['bulk-curation-1689354647298']); + readFolderStub.onSecondCall().returns([]); + const lstatStub = sinon.stub(fs.promises, 'lstat'); + lstatStub.onFirstCall().returns(mockDirectoryLstat); + const result = await findFile(req); + expect(result).to.equals(null); + }); + + it('should return null if FILES_DIRECTORY env variable is not set', async function () { + req.env.FILES_DIRECTORY = undefined + const result = await findFile(req); + expect(result).to.equals(null); + }); + }) +}); diff --git a/resfulservice/src/api-docs/swagger-service.yaml b/resfulservice/src/api-docs/swagger-service.yaml index 871f1c7b..e0012368 100644 --- a/resfulservice/src/api-docs/swagger-service.yaml +++ b/resfulservice/src/api-docs/swagger-service.yaml @@ -274,6 +274,7 @@ paths: description: Missing required input content: application/json: + schema: $ref: '#/components/schemas/Missing-Input-Error' '403': @@ -367,6 +368,7 @@ paths: application/json: schema: $ref: '#/components/schemas/Successful-Curation' + '400': description: If user doesn't upload master template xlsx file content: @@ -405,7 +407,7 @@ paths: schema: type: object example: - message: 'This has already been curated' + message: 'This had been curated already' '500': description: Internal Server Error content: @@ -438,14 +440,14 @@ paths: type: object properties: payload: - $ref: '#/components/schemas/Successful-Curation' + $ref: '#/components/schemas/Sample-Curation' responses: '200': description: Returns the newly curated object content: application/json: schema: - $ref: '#/components/schemas/Successful-Curation' + $ref: '#/components/schemas/Sample-Curation' '304': description: No changes to the curation object to update @@ -485,7 +487,82 @@ paths: message: type: string example: An error occurred - /curate/get{xmlId}/{xlsxObjectId}: + delete: + + security: + - BearerAuth: [] + summary: Deletes a curation or list of curations + description: A route to delete a single curation or list of curations with similar datasetID + tags: + - Curation + parameters: + - in: query + name: xlsxObjectId + schema: + type: string + description: the id of the curation to delete + - in: query + name: dataset + schema: + type: string + description: Dataset ID linked to multiple curations + + responses: + '200': + description: Returns a success message for deleting curations + content: + application/json: + schema: + + type: object + properties: + message: + + type: string + enum: ['Curated sample ID: 6475c82ffbeb8571b77a5991 successfully deleted', 'Curated Samples with Dataset ID: 583e3d6ae74a1d205f4e3fd3 successfully deleted'] + + '400': + description: Bad user input + content: + application/json: + schema: + type: object + example: + success: false + message: validation error + data: + - value: '231543536' + msg: invalid xlsx object id + param: dataset + location: query + '401': + description: User not authorized for this service + content: + application/json: + schema: + $ref: '#/components/schemas/Unauthorized-Error' + '404': + description: Curation not found + content: + application/json: + schema: + type: object + properties: + message: + type: string + enum: ['Curation sample not found', '`Dataset ID: 583e3d6ae74a1d205f4e3fd3 not found`'] + '500': + description: Internal Server Error + content: + application/json: + schema: + type: object + properties: + message: + type: string + example: An error occurred + + /curate/get: get: security: - BearerAuth: [] @@ -495,22 +572,22 @@ paths: - Curation parameters: - name: xlsxObjectId - in: path + in: query + description: The ID of the curated data to return for editing schema: type: string - description: The ID of the curated data to return for editing - name: xmlId - in: path + in: query + description: Alternatively the ID of the xml to return for editing schema: type: string - description: Alternatively the ID of the xml to return for editing responses: '200': description: Returns the newly curated object content: application/json: schema: - $ref: '#/components/schemas/Successful-Curation' + $ref: '#/components/schemas/Sample-Curation' '401': description: User not authorized for this service @@ -538,6 +615,308 @@ paths: message: type: string example: An error occurred + + /curate/bulk: + post: + security: + - BearerAuth: [] + summary: Creates bulk curations + description: Creates multiple curations based on multiple master template uploaded in a zip file + tags: + - Curation + parameters: + - in: query + name: dataset + schema: + type: string + description: Dataset ID + requestBody: + description: A payload that contains zip file which holds all the files for separate curations in individual folders + required: true + content: + multipart/form-data: + schema: + type: object + properties: + uploadfile: + type: array + items: + type: string + format: binary + responses: + '201': + description: Returns the newly curated object + content: + application/json: + schema: + type: object + properties: + bulkCurations: + type: array + items: + $ref: '#/components/schemas/Successful-Curation' + bulkErrors: + example: + - "flename": "mm_files/bulk-curation-1688984487096/master_template (1).xlsx" + "real_permittivity.csv": "file not uploaded" + "loss_permittivity.csv": "file not uploaded" + "tan_delta.csv": "file not uploaded" + "weibull.csv": "file not uploaded" + + '400': + description: If user doesn't upload master template xlsx file + content: + application/json: + schema: + type: object + properties: + message: + type: string + enum: ['Missing dataset ID in query', 'bulk curation zip file not uploaded'] + + '401': + description: User not authorized for this service + content: + application/json: + schema: + $ref: '#/components/schemas/Unauthorized-Error' + + '404': + description: Dataset not found + content: + application/json: + schema: + type: object + example: + message: "Dataset ID: 583e3d6ae74a1d205f4e3fd3 not found" + + '500': + description: Internal Server Error + content: + application/json: + schema: + type: object + properties: + message: + type: string + example: An error occurred + + /files/{fileId}: + get: + summary: Download files from the server + description: A route to download files by specifying a boolean isFileStore query params for files saved in the directory on the server or in database + tags: + - Files + parameters: + - in: query + name: isFileStore + schema: + type: boolean + example: true + description: a boolean value to specify if the file is stored in a directory or database + - in: query + name: isStore + schema: + type: boolean + example: true + description: a boolean value to specify if the file is stored in a directory or database + - in: path + required: true + name: fileId + description: the id of the file to download + schema: + type: string + example: 'mobile (15).png' + responses: + '200': + description: Returns the downloaded file type + content: + application/octet-stream: + schema: + type: string + format: binary + + '400': + description: Bad user input + content: + application/json: + schema: + type: object + example: + success: false + message: validation error + data: + - value: true + msg: only boolean value allowed + param: isFileStore + location: query + '500': + description: Internal Server Error + content: + application/json: + schema: + type: object + properties: + message: + type: string + example: 'Error fetching file' + + delete: + security: + - BearerAuth: [] + summary: Deletes a file from the server + description: A route to delete a file by passing the fileId path params + tags: + - Files + parameters: + - in: path + required: true + name: fileId + description: the id of the file to delete + schema: + type: string + example: 'mobile (15).png' + responses: + '200': + description: OK + + '400': + description: Bad user input + content: + application/json: + schema: + type: object + example: + success: false + message: validation error + data: + - value: 'mobile (15).pnge' + msg: Unsupported filetype + param: fileId + location: params + + '401': + description: User not authorized for this service + content: + application/json: + schema: + $ref: '#/components/schemas/Unauthorized-Error' + + '500': + description: Internal Server Error + content: + application/json: + schema: + type: object + properties: + message: + type: string + example: 'Error deleting files' + + /files/image_migration/{imageType}: + get: + summary: Image migration routes + description: A route to fetch images of specific types only + tags: + - Files + parameters: + - in: path + required: true + name: imageType + description: the type of images to fetch + schema: + type: string + example: 'tiff' + responses: + '200': + description: Returns the fetched image information + content: + application/json: + schema: + type: object + example: + images: + - fieldname: 'uploadfile' + originalname: '001.tif' + encoding: '7bit' + mimetype: 'image/tiff' + destination: 'mm_files' + filename: 'exciting_spoonbill_ermentrude-2023-05-05T11:25:53.451Z-001.tif' + path: 'mm_files/exciting_spoonbill_ermentrude-2023-05-05T11:25:53.451Z-001.tif' + size: 101196 + + '400': + description: Bad user input + content: + application/json: + schema: + type: object + example: + success: false + message: validation error + data: + - value: png + msg: only supports tiff & tif migration + param: imageType + location: params + '500': + description: Internal Server Error + content: + application/json: + schema: + type: object + properties: + message: + type: string + example: 'Error fetching file' + + /files/upload: + post: + summary: upload files to the server + description: A route to upload files to the server + tags: + - Files + requestBody: + description: An array selection of files to upload + required: true + content: + multipart/form-data: + schema: + type: object + properties: + uploadfile: + type: array + items: + type: string + format: binary + responses: + '201': + description: Returns the downloaded file type + content: + application/json: + schema: + $ref: '#/components/schemas/Sample-Files' + + '400': + description: User uploads unsupported files + content: + application/json: + schema: + type: object + properties: + message: + type: string + example: 'Only .png, .jpg, .jpeg, .tiff, .tif, .csv, .zip, .xls and .xlsx format allowed!' + '500': + description: Internal Server Error + content: + application/json: + schema: + type: object + properties: + message: + type: string + example: Error uploading files components: securitySchemes: BearerAuth: @@ -563,13 +942,50 @@ components: message: type: string example: Category type or doc is missing + Successful-Curation: type: object example: - xml: "\n \n S10\n S28\n \n \n \n Tolulomo Fateye\n tolulomo@toluconsulting.com\n experiments\n lab-generated\n Aditya Shanker Prasad\n Gbolahan Adeleke\n https://search.proquest.com/openview/eb63d4d6b84b1252971b3e3eec53b97c/1?pq-origsite=gscholar&cbl=51922&diss=y\n Rensselaer Polytechnic Institute\n \n \n \n \n" + xml: "\n \n S10\n S28\n \n \n \n John Smith\n jonsmith@nomail.org\n experiments\n lab-generated\n Aditya Shanker Prasad\n John Doe\n https://search.proquest.com/openview/eb63d4d6b84b1252971b3e3eec53b97c/1?pq-origsite=gscholar&cbl=51922&diss=y\n Rensselaer Polytechnic Institute\n \n \n \n \n" user: _id: "643931cc6f44b02f01380f7a" displayName: "Test" groupId: "583e3d6ae74a1d205f4e3fd3" + sampleID: "64394c8032bc6325505af6f9" isApproved: false, status: "Editing" + + Sample-Curation: + type: object + example: + data origin: + YourName: Akash Prasad + YourEmail: akash@prasad.com + ID: S10 + Control_ID: S28 + Origin: experiments + Citation Type: lab-generated + Author: Aditya Shanker Prasad + URL: https://search.proquest.com/openview/eb63d4d6b84b1252971b3e3eec53b97c/1?pq-origsite=gscholar&cbl=51922&diss=y + Location: Rensselaer Polytechnic Institute + + Sample-Files: + type: object + example: + files: + - fieldname: 'uploadfile' + originalname: '001.tif' + encoding: '7bit' + mimetype: 'image/tiff' + destination: 'mm_files' + filename: 'exciting_spoonbill_ermentrude-2023-05-05T11:25:53.451Z-001.tif' + path: 'mm_files/exciting_spoonbill_ermentrude-2023-05-05T11:25:53.451Z-001.tif' + size: 101196 + - fieldname: 'uploadfile' + originalname: 'master-template.xlsx' + encoding: '7bit' + mimetype: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + destination: 'mm_files' + filename: 'educational_hornet_maisie-2023-04-18T16:01:27.609Z-master-template.xlsx' + path: 'mm_files/educational_hornet_maisie-2023-04-18T16:01:27.609Z-master-template.xlsx' + size: 104928 \ No newline at end of file diff --git a/resfulservice/src/controllers/adminController.js b/resfulservice/src/controllers/adminController.js index 9bbbfbcb..a8f96819 100644 --- a/resfulservice/src/controllers/adminController.js +++ b/resfulservice/src/controllers/adminController.js @@ -29,6 +29,20 @@ exports.initializeElasticSearch = async (req, res, next) => { } }; +/** + * Redirect admin user to object store + * @param {*} req + * @param {*} res + * @param {*} next + * @returns {*} redirect + */ +exports.loadObjectStore = async (req, res, next) => { + const log = req.logger; + log.info('loadObjectStore(): Function entry'); + + return res.redirect(`http://localhost:${req.env.MINIO_CONSOLE_PORT}`); +}; + /** * Bulk Load Elastic Search * @param {*} req @@ -210,6 +224,7 @@ exports.populateDatasetIds = async (req, res, next) => { // if (!connDB) return next(errorWriter(req, 'DB error', 'populateDatasetIds')); try { + // TODO: Fix iterator or remove // const db = await iterator.dbConnectAndOpen(connDB, req?.env?.MM_DB); // const Dataset = await db.collection('datasets'); // const datasets = await Dataset.find({}); diff --git a/resfulservice/src/controllers/curationController.js b/resfulservice/src/controllers/curationController.js index 78a95d54..d4e44851 100644 --- a/resfulservice/src/controllers/curationController.js +++ b/resfulservice/src/controllers/curationController.js @@ -2,11 +2,14 @@ const util = require('util'); const XlsxFileManager = require('../utils/curation-utility'); const BaseSchemaObject = require('../../config/xlsx.json'); const { errorWriter } = require('../utils/logWriter'); +const latency = require('../middlewares/latencyTimer'); const { BaseObjectSubstitutionMap, CurationEntityStateDefault } = require('../../config/constant'); const CuratedSamples = require('../models/curatedSamples'); const XlsxCurationList = require('../models/xlsxCurationList'); const XmlData = require('../models/xmlData'); const DatasetId = require('../models/datasetId'); +const FileStorage = require('../middlewares/fileStorage'); +const FileManager = require('../utils/fileManager'); exports.curateXlsxSpreadsheet = async (req, res, next) => { const { user, logger, query } = req; @@ -16,40 +19,42 @@ exports.curateXlsxSpreadsheet = async (req, res, next) => { if (!req.files?.uploadfile) { return next(errorWriter(req, 'Material template files not uploaded', 'curateXlsxSpreadsheet', 400)); } - - const regex = /master_template.xlsx$/gi; + const regex = /(?=.*?(master_template))(?=.*?(.xlsx)$)/gi; const xlsxFile = req.files.uploadfile.find((file) => regex.test(file?.path)); if (!xlsxFile) { return next(errorWriter(req, 'Master template xlsx file not uploaded', 'curateXlsxSpreadsheet', 400)); } - if (!query.dataset) { - return next(errorWriter(req, 'Missing dataset ID in query', 'curateXlsxSpreadsheet', 400)); - } - try { - const [validList, storedCurations, datasets] = await Promise.all([ + const [validList, storedCurations] = await Promise.all([ XlsxCurationList.find({}, null, { lean: true }), - CuratedSamples.find({ user: user._id }, { object: 1 }, { lean: true }), - DatasetId.findOne({ _id: query.dataset }) + CuratedSamples.find({ user: user._id }, { object: 1 }, { lean: true }) ]); - if (!datasets) { - return next(errorWriter(req, `A sample must belong to a dataset. Dataset ID: ${query.dataset ?? null} not found`, 'curateXlsxSpreadsheet', 404)); - } - const validListMap = generateCurationListMap(validList); - const result = await this.createMaterialObject(xlsxFile.path, BaseSchemaObject, validListMap, req.files.uploadfile); - + const processedFiles = []; + const result = await this.createMaterialObject(xlsxFile.path, BaseSchemaObject, validListMap, req.files.uploadfile, processedFiles); + if (result?.count && req?.isParentFunction) return { errors: result.errors }; if (result?.count) return res.status(400).json({ errors: result.errors }); const curatedAlready = storedCurations.find( - object => object?.['data origin']?.Title === result?.['data origin']?.Title && - object?.['data origin']?.PublicationType === result?.['data origin']?.PublicationType); + object => object?.DATA_SOURCE?.Citation?.CommonFields?.Title === result?.DATA_SOURCE?.Citation?.CommonFields?.Title && + object?.DATA_SOURCE?.Citation?.CommonFields?.PublicationType === result?.DATA_SOURCE?.Citation?.CommonFields?.PublicationType); + + if (curatedAlready) return next(errorWriter(req, 'This had been curated already', 'curateXlsxSpreadsheet', 409)); - if (curatedAlready) return next(errorWriter(req, 'This has already been curated', 'curateXlsxSpreadsheet', 409)); + let datasets; + if (query.dataset) { + datasets = await DatasetId.findOne({ _id: query.dataset }); + } else if (result?.Control_ID) { + const existingDataset = await DatasetId.findOne({ controlSampleID: result?.Control_ID }); + datasets = existingDataset ?? await DatasetId.create({ user, controlSampleID: result.Control_ID }); + } + if (!datasets) { + return next(errorWriter(req, `A sample must belong to a dataset. Dataset ID: ${query.dataset ?? null} not found`, 'curateXlsxSpreadsheet', 404)); + } const newCurationObject = new CuratedSamples({ object: result, user: user?._id, dataset: datasets._id }); const curatedObject = await (await newCurationObject.save()).populate('user', 'displayName'); @@ -58,29 +63,118 @@ exports.curateXlsxSpreadsheet = async (req, res, next) => { let xml = XlsxFileManager.xmlGenerator(JSON.stringify({ PolymerNanocomposite: curatedObject.object })); xml = `\n ${xml}`; - return res.status(201).json({ + const curatedSample = { + sampleID: curatedObject._id, xml, user: curatedObject.user, groupId: curatedObject.dataset, isApproved: curatedObject.entityState !== CurationEntityStateDefault, status: curatedObject.curationState - }); + }; + + if (req?.isParentFunction) return { curatedSample, processedFiles }; + latency.latencyCalculator(res); + return res.status(200).json({ ...curatedSample }); } catch (err) { next(errorWriter(req, err, 'curateXlsxSpreadsheet', 500)); } }; +exports.bulkXlsxCurations = async (req, res, next) => { + const { query, logger } = req; + + logger.info('bulkXlsxCurations Function Entry:'); + + const regex = /.zip$/gi; + const zipFile = req.files?.uploadfile?.find((file) => regex.test(file?.path)); + + if (!zipFile) { + return next(errorWriter(req, 'bulk curation zip file not uploaded', 'bulkXlsxCurations', 400)); + } + + if (query.dataset) { + const dataset = await DatasetId.findOne({ _id: query.dataset }); + if (!dataset) return next(errorWriter(req, `Dataset ID: ${query.dataset ?? null} not found`, 'bulkXlsxCurations', 404)); + } + const bulkErrors = []; + const bulkCurations = []; + try { + const { folderPath, allfiles } = await XlsxFileManager.unZipFolder(req, zipFile.path); + await processFolders(bulkCurations, bulkErrors, folderPath, req); + if (bulkErrors.length) { + const failedCuration = bulkErrors.map(curation => `mm_files/${curation.filename}`); + for (const file of allfiles) { + const filePath = `${folderPath}/${file?.path}`; + if (file.type === 'file' && !failedCuration.includes(filePath)) { + FileManager.deleteFile(filePath, req); + } + } + } else { + FileManager.deleteFolder(folderPath, req); + } + latency.latencyCalculator(res); + return res.status(200).json({ bulkCurations, bulkErrors }); + } catch (err) { + next(errorWriter(req, err, 'bulkXlsxCurations', 500)); + } +}; + +const processFolders = async (bulkCurations, bulkErrors, folder, req) => { + const { folders, masterTemplates, curationFiles } = XlsxFileManager.readFolder(folder); + await processSingleCuration(masterTemplates, curationFiles, bulkCurations, bulkErrors, req); + + if (folders.length) { + for (const folder of folders) { + await processFolders(bulkCurations, bulkErrors, folder, req); + } + } +}; + +const processSingleCuration = async (masterTemplates, curationFiles, bulkCurations, bulkErrors, req) => { + let imageBucketArray = []; + if (masterTemplates.length) { + for (const masterTemplate of masterTemplates) { + const newCurationFiles = [...curationFiles, masterTemplate]; + const newReq = { + ...req, + files: { uploadfile: newCurationFiles.map(file => ({ path: file })) }, + isParentFunction: true + }; + const nextFnCallBack = fn => fn; + const result = await this.curateXlsxSpreadsheet(newReq, {}, nextFnCallBack); + + if (result?.message || result?.errors) { + bulkErrors.push({ filename: masterTemplate.split('mm_files/').pop(), errors: result?.message ?? result?.errors }); + } else { + bulkCurations.push(result.curatedSample); + imageBucketArray = result.processedFiles.filter(file => /\.(jpe?g|tiff?|png)$/i.test(file)); + } + } + } + + if (imageBucketArray.length) { + for (const image of imageBucketArray) { + const file = { + filename: image, + mimetype: `image/${image.split('.').pop()}`, + path: image + }; + FileStorage.minioPutObject(file, req); + } + } +}; + exports.getXlsxCurations = async (req, res, next) => { - const { user, logger, params } = req; + const { user, logger, query } = req; logger.info('getXlsxCurations Function Entry:'); - const { xlsxObjectId, xmlId } = params; + const { xlsxObjectId, xmlId } = query; const filter = {}; if (user?.roles !== 'admin') filter.user = user._id; try { - if (!!xmlId || !!xlsxObjectId) { + if (xmlId || xlsxObjectId) { let fetchedObject; if (xlsxObjectId) { const xlsxObject = await CuratedSamples.findOne({ _id: xlsxObjectId, ...filter }, null, { lean: true, populate: { path: 'user', select: 'givenName surName' } }); @@ -92,7 +186,7 @@ exports.getXlsxCurations = async (req, res, next) => { if (!xmlData) return next(errorWriter(req, 'Sample xml not found', 'getXlsxCurations', 404)); fetchedObject = XlsxFileManager.jsonGenerator(xmlData.xml_str); } - + latency.latencyCalculator(res); return res.status(200).json(fetchedObject); } else { const xlsxObjects = await CuratedSamples.find(filter, { user: 1, createdAt: 1, updatedAt: 1, _v: 1 }, { lean: true, populate: { path: 'user', select: 'givenName surName' } }); @@ -154,7 +248,6 @@ exports.deleteXlsxCurations = async (req, res, next) => { await CuratedSamples.deleteMany({ _id: { $in: datasets.samples } }); return res.status(200).json({ message: `Dataset ID: ${query.dataset} successfully deleted` }); } - return next(errorWriter(req, 'Missing dataset ID or curation ID in query', 'curateXlsxSpreadsheet', 400)); } catch (err) { next(errorWriter(req, err, 'deleteXlsxCurations', 500)); } @@ -204,7 +297,7 @@ const appendUploadedFiles = (parsedCSVData) => { * @param {Object} errors - Object created to store errors that occur while parsing the spreadsheets * @returns {Object} - Newly curated object or errors that occur while proces */ -exports.createMaterialObject = async (path, BaseObject, validListMap, uploadedFiles, errors = {}) => { +exports.createMaterialObject = async (path, BaseObject, validListMap, uploadedFiles, processedFiles, errors = {}) => { const sheetsData = {}; const filteredObject = {}; @@ -215,7 +308,7 @@ exports.createMaterialObject = async (path, BaseObject, validListMap, uploadedFi const objArr = []; for (const prop of propertyValue.values) { - const newObj = await this.createMaterialObject(path, prop, validListMap, uploadedFiles, errors); + const newObj = await this.createMaterialObject(path, prop, validListMap, uploadedFiles, processedFiles, errors); const value = Object.values(newObj)[0]; if (value) { @@ -244,7 +337,7 @@ exports.createMaterialObject = async (path, BaseObject, validListMap, uploadedFi } const objArr = []; for (const prop of multiples) { - const newObj = await this.createMaterialObject(path, prop, validListMap, uploadedFiles, errors); + const newObj = await this.createMaterialObject(path, prop, validListMap, uploadedFiles, processedFiles, errors); if (Object.keys(newObj).length > 0) { objArr.push(newObj); @@ -258,7 +351,7 @@ exports.createMaterialObject = async (path, BaseObject, validListMap, uploadedFi const objArr = []; for (const prop of propertyValue) { - const newObj = await this.createMaterialObject(path, prop, validListMap, uploadedFiles, errors); + const newObj = await this.createMaterialObject(path, prop, validListMap, uploadedFiles, processedFiles, errors); if (Object.keys(newObj).length > 0) { objArr.push(newObj); @@ -300,6 +393,7 @@ exports.createMaterialObject = async (path, BaseObject, validListMap, uploadedFi filteredObject.data = data; } filteredObject[BaseObjectSubstitutionMap[property] ?? property] = file.filename; + processedFiles.push(file.path); } else { errors[cellValue] = 'file not uploaded'; } @@ -312,7 +406,7 @@ exports.createMaterialObject = async (path, BaseObject, validListMap, uploadedFi filteredObject[BaseObjectSubstitutionMap[property] ?? property] = propertyValue.default; } } else { - const nestedObj = await this.createMaterialObject(path, propertyValue, validListMap, uploadedFiles, errors); + const nestedObj = await this.createMaterialObject(path, propertyValue, validListMap, uploadedFiles, processedFiles, errors); const nestedObjectKeys = Object.keys(nestedObj); if (nestedObjectKeys.length > 0) { diff --git a/resfulservice/src/controllers/fileController.js b/resfulservice/src/controllers/fileController.js index 9f5b5f5a..c76f019b 100644 --- a/resfulservice/src/controllers/fileController.js +++ b/resfulservice/src/controllers/fileController.js @@ -1,24 +1,26 @@ const mongoose = require('mongoose'); const { PassThrough } = require('stream'); const fsFiles = require('../models/fsFiles'); +const latency = require('../middlewares/latencyTimer'); const { errorWriter, successWriter } = require('../utils/logWriter'); -const { deleteFile, findFile } = require('../utils/fileManager'); +const FileManager = require('../utils/fileManager'); +const { SupportedFileResponseHeaders } = require('../../config/constant'); +const minioClient = require('../utils/minio'); +const { MinioBucket } = require('../../config/constant'); -const _createEmptyStream = () => new PassThrough('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII').end(); +exports._createEmptyStream = () => new PassThrough('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII').end(); exports.imageMigration = async (req, res, next) => { const { imageType } = req.params; try { - const bucket = new mongoose.mongo.GridFSBucket(mongoose.connection.db, { - bucketName: 'fs' - }); - + const bucket = this.connectToMongoBucket(); const files = await bucket .find({ filename: { $regex: imageType } }) .limit(10) .toArray(); successWriter(req, { message: 'success' }, 'imageMigration'); + latency.latencyCalculator(res); return res.status(200).json({ images: files }); } catch (error) { next(errorWriter(req, 'Error fetching image', 'imageMigration', 500)); @@ -26,33 +28,52 @@ exports.imageMigration = async (req, res, next) => { }; exports.fileContent = async (req, res, next) => { + const { fileId } = req.params; try { - if (req.query.isDirectory) { - const fileStream = await findFile(req); + if (req.query.isFileStore) { + const { fileStream, ext } = await FileManager.findFile(req); if (!fileStream) { + // TODO (@TOLU): Refactor later as this is duplicated below Ln 67, also used in Ln 51 res.setHeader('Content-Type', 'image/png'); - return _createEmptyStream().pipe(res); + latency.latencyCalculator(res); + return this._createEmptyStream().pipe(res); } - + latency.latencyCalculator(res); + res.setHeader('Content-Type', SupportedFileResponseHeaders[ext]); return fileStream.pipe(res); } - const { fileId } = req.params; - const bucket = new mongoose.mongo.GridFSBucket(mongoose.connection.db, { - bucketName: 'fs' - }); + if (req.query.isStore) { + const bucketName = req.env.MINIO_BUCKET ?? MinioBucket; + const dataStream = await minioClient.getObject(bucketName, fileId); + + if (!dataStream) { + res.setHeader('Content-Type', 'image/png'); + latency.latencyCalculator(res); + return this._createEmptyStream().pipe(res); + } + + const { ext } = FileManager.getFileExtension(fileId); + res.setHeader('Content-Type', SupportedFileResponseHeaders[ext ?? 'image/png']); + latency.latencyCalculator(res); + return dataStream.pipe(res); + } + + const bucket = this.connectToMongoBucket(); const _id = new mongoose.Types.ObjectId(fileId); const exist = await fsFiles.findById(_id).limit(1); if (!exist) { res.setHeader('Content-Type', 'image/png'); - return _createEmptyStream().pipe(res); + latency.latencyCalculator(res); + return this._createEmptyStream().pipe(res); } const downloadStream = bucket.openDownloadStream(_id); + latency.latencyCalculator(res); downloadStream.pipe(res); } catch (error) { - next(errorWriter(req, 'Error fetching file', 'fileContent', 500)); + next(errorWriter(req, `${error.message ?? 'Error fetching file'}`, 'fileContent', 500)); } }; @@ -61,21 +82,53 @@ exports.uploadFile = async (req, res, next) => { req.logger.info('datasetIdUpload Function Entry:'); successWriter(req, { message: 'success' }, 'uploadFile'); + latency.latencyCalculator(res); return res.status(201).json({ files: req.files.uploadfile }); } catch (error) { - next(errorWriter(req, 'error uploading files', 'uploadFile', 500)); + next(errorWriter(req, 'Error uploading files', 'uploadFile', 500)); } }; -exports.deleteFile = (req, res, next) => { +exports.deleteFile = async (req, res, next) => { const filesDirectory = req.env?.FILES_DIRECTORY; const { fileId } = req.params; const filePath = `${filesDirectory}/${fileId}`; try { - deleteFile(filePath, req); + const bucketName = req.env.MINIO_BUCKET ?? MinioBucket; + await minioClient.removeObject(bucketName, fileId); + FileManager.deleteFile(filePath, req); + latency.latencyCalculator(res); return res.sendStatus(200); } catch (err) { - next(errorWriter(req, 'Error deleting files', 'deleteFile', 500)); + next(errorWriter(req, `${err.message ?? 'Error deleting files'}`, 'deleteFile', 500)); } }; + +exports.findFiles = (req, res) => { + const bucketName = req.env.MINIO_BUCKET ?? MinioBucket; + const query = req.query.filename; + + const objectsStream = minioClient.listObjects(bucketName, query, true); + const foundFiles = []; + objectsStream.on('data', (obj) => { + foundFiles.push(obj.name); + }); + + objectsStream.on('end', () => { + res.json({ files: foundFiles }); + }); + + objectsStream.on('error', (err) => { + req.logger.error(err); + res.status(500).json({ error: 'Error finding files in Minio' }); + }); +}; + +// TODO (@TOLU): Move to utils folder +exports.connectToMongoBucket = () => { + const bucket = new mongoose.mongo.GridFSBucket(mongoose.connection.db, { + bucketName: 'fs' + }); + return bucket; +}; diff --git a/resfulservice/src/middlewares/fileStorage.js b/resfulservice/src/middlewares/fileStorage.js index 202b3dd7..f4c6f61d 100644 --- a/resfulservice/src/middlewares/fileStorage.js +++ b/resfulservice/src/middlewares/fileStorage.js @@ -1,7 +1,8 @@ -const path = require('path'); -const express = require('express'); const multer = require('multer'); const { uniqueNamesGenerator, adjectives, names, animals } = require('unique-names-generator'); +const minioClient = require('../utils/minio'); +const { deleteFile } = require('../utils/fileManager'); +const { MinioBucket } = require('../../config/constant'); const shortName = uniqueNamesGenerator({ dictionaries: [adjectives, animals, names], @@ -11,7 +12,7 @@ const shortName = uniqueNamesGenerator({ const fileStorage = multer.diskStorage({ destination: (req, file, cb) => { - cb(null, 'mm_files'); + cb(null, req.env?.FILES_DIRECTORY ?? 'mm_files'); }, filename: (req, file, cb) => { cb(null, shortName + '-' + new Date().toISOString() + '-' + file.originalname); @@ -27,19 +28,47 @@ const fileFilter = (req, file, cb) => { file.mimetype === 'image/tiff' || file.mimetype === 'text/csv' || file.mimetype === 'application/vnd.ms-excel' || - file.mimetype === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + file.mimetype === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' || + file.mimetype === 'application/zip' || + file.mimetype === 'application/x-zip-compressed' ) { cb(null, true); } else { - cb(new Error('Only .png, .jpg, .jpeg, .tiff, .tif, .csv, .xls and .xlsx format allowed!'), false); + cb(new Error('Only .png, .jpg, .jpeg, .tiff, .tif, .csv, .zip, .xls and .xlsx format allowed!'), false); } }; const fileMgr = multer({ storage: fileStorage, fileFilter }).fields([{ name: 'uploadfile', maxCount: 20 }]); -const fileServer = express.static(path.join(__dirname, 'filestore')); +const minioUpload = (req, res, next) => { + const files = req.files?.uploadfile; + if (!files) { + return next(); + } + + files.forEach(file => { + minioPutObject(file, req); + }); + next(); +}; + +const minioPutObject = (file, req) => { + const bucketName = req.env.MINIO_BUCKET ?? MinioBucket; + const metaData = { + 'Content-Type': file.mimetype, + 'X-Amz-Meta-Testing': '1234' + }; + minioClient.fPutObject(bucketName, file.filename, file.path, metaData, (err, objInfo) => { + if (err) { + console.log(err); + } + + deleteFile(file.path, req); + }); +}; module.exports = { fileMgr, - fileServer + minioUpload, + minioPutObject }; diff --git a/resfulservice/src/middlewares/index.js b/resfulservice/src/middlewares/index.js index 084ac43e..260a4286 100644 --- a/resfulservice/src/middlewares/index.js +++ b/resfulservice/src/middlewares/index.js @@ -1,7 +1,7 @@ const express = require('express'); const acceptedHeaders = require('./accept'); const getEnv = require('./parseEnv'); -const { fileMgr, fileServer } = require('./fileStorage'); +const { fileMgr } = require('./fileStorage'); const { logParser, mmLogger } = require('./loggerService'); const swaggerService = require('./swagger-service'); @@ -17,7 +17,6 @@ const globalMiddleWare = async (app) => { app.use(express.urlencoded({ extended: true })); app.use((req, res, next) => logParser(log, req, next)); app.use(fileMgr); - app.use('/mm_files', fileServer); app.use(acceptedHeaders); app.use(getEnv); }; diff --git a/resfulservice/src/middlewares/isAuth.js b/resfulservice/src/middlewares/isAuth.js index c54b7e27..d81d5443 100644 --- a/resfulservice/src/middlewares/isAuth.js +++ b/resfulservice/src/middlewares/isAuth.js @@ -1,5 +1,5 @@ const { decodeToken } = require('../utils/jwtService'); -const deleteFile = require('../utils/fileManager'); +const { deleteFile } = require('../utils/fileManager'); const { errorWriter } = require('../utils/logWriter'); module.exports = (req, res, next) => { diff --git a/resfulservice/src/middlewares/latencyTimer.js b/resfulservice/src/middlewares/latencyTimer.js new file mode 100644 index 00000000..6732c810 --- /dev/null +++ b/resfulservice/src/middlewares/latencyTimer.js @@ -0,0 +1,12 @@ +exports.latencyTimer = (req, res, next) => { + res.header('entryTime', new Date().getTime()); + next(); +}; + +exports.latencyCalculator = (res) => { + const startTime = res.get('entrytime'); + const endTime = new Date().getTime(); + const latency = Math.floor((endTime - startTime)); + res.header('endTime', new Date(endTime)); + res.header('latency', `${Math.floor((latency / 1000) % 60)} seconds`); +}; diff --git a/resfulservice/src/middlewares/validations.js b/resfulservice/src/middlewares/validations.js index db8350d2..2f8c3d5e 100644 --- a/resfulservice/src/middlewares/validations.js +++ b/resfulservice/src/middlewares/validations.js @@ -1,6 +1,8 @@ -const { param, validationResult, body, query } = require('express-validator'); +const { param, validationResult, body, query, check } = require('express-validator'); +const { Types: { ObjectId } } = require('mongoose'); const { userRoles } = require('../../config/constant'); const { errorWriter } = require('../utils/logWriter'); +const { SupportedFileTypes } = require('../../config/constant'); exports.validateImageType = [ param('imageType').not().isEmpty().withMessage('image type required').bail().isIn(['tiff', 'tif']).withMessage('only supports tiff & tif migration'), @@ -12,14 +14,46 @@ exports.validateAcceptableUploadType = [ validationErrorHandler ]; +exports.validateFileDownload = [ + query('isFileStore', 'only boolean value allowed').optional().isIn(['true', 'false']), + query('isStore', 'only boolean value allowed').optional().isIn(['true', 'false']), + check('fileId').custom((value, { req }) => { + if (req.query?.isFileStore === 'true' || req.query?.isStore === 'true') { + const filetype = value.split('.').pop(); + if (!SupportedFileTypes.includes(filetype)) { + throw new Error('Unsupported filetype'); + } + return true; + } else if (!req.query.isFileStore && !req.query.isStore) { + if (ObjectId.isValid(value)) { + if ((String)(new ObjectId(value)) === value) { return true; } + throw new Error('Invalid file id'); + } + throw new Error('Invalid file id'); + } + + return true; + }), + validationErrorHandler +]; + +exports.validateFileId = [ + check('fileId').custom((value, { req }) => { + const filetype = value.split('.').pop(); + if (!SupportedFileTypes.includes(filetype)) { + throw new Error('Unsupported filetype'); + } + return true; + }), + validationErrorHandler +]; + exports.validateXlsxObjectUpdate = [ query('xlsxObjectId').not().isEmpty().withMessage('xlsx object ID required').bail().isMongoId().withMessage('invalid xlsx object id'), body('payload').isObject().withMessage('please provide xlsx object for update'), validationErrorHandler ]; -exports.validateImageId = [param('fileId').not().isEmpty().withMessage('image ID required').bail().isMongoId().withMessage('invalid file id'), validationErrorHandler]; - function validationErrorHandler (req, res, next) { const errors = validationResult(req); if (!errors.isEmpty()) return res.status(400).json({ success: false, message: 'validation error', data: errors.array() }); @@ -30,6 +64,12 @@ exports.validateIsAdmin = (req, res, next) => !req.user?.roles === userRoles.isA exports.validateXlsxObjectDelete = [ query('xlsxObjectId').if(query('dataset').not().exists()).bail().isMongoId().withMessage('invalid xlsx object id'), - query('dataset').if(query('xlsxObjectId').not().exists()).bail().isMongoId().withMessage('invalid xlsx object id'), + query('dataset').if(query('xlsxObjectId').not().exists()).bail().isMongoId().withMessage('invalid dataset id'), + validationErrorHandler +]; + +exports.validateXlsxObjectGet = [ + query('xlsxObjectId').optional().isMongoId().withMessage('invalid xlsx object id'), + query('xmlId').optional().isMongoId().withMessage('invalid dataset id'), validationErrorHandler ]; diff --git a/resfulservice/src/models/datasetId.js b/resfulservice/src/models/datasetId.js index 35fb0a9b..8aa1df78 100644 --- a/resfulservice/src/models/datasetId.js +++ b/resfulservice/src/models/datasetId.js @@ -8,6 +8,9 @@ const datasetIdSchema = new Schema({ ref: 'User', required: true }, + controlSampleID: { + type: String + }, status: { type: String, enum: DatasetStatusOpt, diff --git a/resfulservice/src/routes/admin.js b/resfulservice/src/routes/admin.js index 56e45b70..6cd24523 100644 --- a/resfulservice/src/routes/admin.js +++ b/resfulservice/src/routes/admin.js @@ -24,6 +24,8 @@ router .put(isAuth, AdminController.loadElasticSearch) .delete(isAuth, AdminController.loadElasticSearch); +router.route('/store').get(AdminController.loadObjectStore); + // Note: Not in use. Deprecated for authService.js route. router.route('/login').post(loginController.login); module.exports = router; diff --git a/resfulservice/src/routes/curation.js b/resfulservice/src/routes/curation.js index 56a12bcf..dd6a4b12 100644 --- a/resfulservice/src/routes/curation.js +++ b/resfulservice/src/routes/curation.js @@ -2,16 +2,19 @@ const express = require('express'); const router = express.Router(); const curationController = require('../controllers/curationController'); const isAuth = require('../middlewares/isAuth'); -const { validateXlsxObjectUpdate, validateXlsxObjectDelete } = require('../middlewares/validations'); +const { latencyTimer } = require('../middlewares/latencyTimer'); +const { validateXlsxObjectUpdate, validateXlsxObjectDelete, validateXlsxObjectGet } = require('../middlewares/validations'); router.route('') - .get(isAuth, curationController.getCurationSchemaObject) - .post(isAuth, curationController.curateXlsxSpreadsheet) - .put(validateXlsxObjectUpdate, isAuth, curationController.updateXlsxCurations) + .get(isAuth, latencyTimer, curationController.getCurationSchemaObject) + .post(isAuth, latencyTimer, curationController.curateXlsxSpreadsheet) + .put(validateXlsxObjectUpdate, isAuth, latencyTimer, curationController.updateXlsxCurations) .delete(validateXlsxObjectDelete, isAuth, curationController.deleteXlsxCurations); -router.route('/get/:xmlId/:xlsxObjectId') - .get(isAuth, curationController.getXlsxCurations); +router.route('/bulk') + .post(isAuth, latencyTimer, curationController.bulkXlsxCurations); +router.route('/get') + .get(validateXlsxObjectGet, isAuth, latencyTimer, curationController.getXlsxCurations); router.route('/admin') .post(isAuth, curationController.approveCuration); diff --git a/resfulservice/src/routes/files.js b/resfulservice/src/routes/files.js index 3bfca29c..5f0fc332 100644 --- a/resfulservice/src/routes/files.js +++ b/resfulservice/src/routes/files.js @@ -2,14 +2,14 @@ const express = require('express'); const router = express.Router(); const fileController = require('../controllers/fileController'); const isAuth = require('../middlewares/isAuth'); -const { validateImageType } = require('../middlewares/validations'); -// TODO: Bring back validation and expand -// const { validateImageType, validateImageId } = require('../middlewares/validations'); +const { latencyTimer } = require('../middlewares/latencyTimer'); +const { minioUpload } = require('../middlewares/fileStorage'); +const { validateImageType, validateFileId, validateFileDownload } = require('../middlewares/validations'); -router.route('/:fileId') - .get(fileController.fileContent) - .delete(isAuth, fileController.deleteFile); -router.route('/image_migration/:imageType').get(validateImageType, fileController.imageMigration); -router.route('/upload').post(fileController.uploadFile); +router.route('/:fileId([^/]*)') + .get(validateFileDownload, latencyTimer, fileController.fileContent) + .delete(isAuth, validateFileId, latencyTimer, fileController.deleteFile); +router.route('/image_migration/:imageType').get(validateImageType, latencyTimer, fileController.imageMigration); +router.route('/upload').post(latencyTimer, minioUpload, fileController.uploadFile); module.exports = router; diff --git a/resfulservice/src/utils/curation-utility.js b/resfulservice/src/utils/curation-utility.js index 1ea243e3..7aa0150a 100644 --- a/resfulservice/src/utils/curation-utility.js +++ b/resfulservice/src/utils/curation-utility.js @@ -1,7 +1,10 @@ +const fs = require('fs'); const readXlsxFile = require('read-excel-file/node'); +const decompress = require('decompress'); +const path = require('path'); const Xmljs = require('xml-js'); -const fs = require('fs'); const csv = require('csv-parser'); +const { deleteFile, deleteFolder } = require('../utils/fileManager'); exports.xlsxFileReader = async (path, sheetName) => { const sheetData = await readXlsxFile(path, { sheet: sheetName }); @@ -29,6 +32,53 @@ exports.parseCSV = async (filename) => { }); }; +exports.unZipFolder = async (req, filename) => { + const logger = req.logger; + try { + const folderPath = `mm_files/bulk-curation-${new Date().getTime()}`; + const allfiles = await decompress(filename, folderPath); + + deleteFile(filename, req); + deleteFolder(`${folderPath}/__MACOSX`, req); + return { folderPath, allfiles }; + } catch (error) { + logger.error(`[unZipFolder]: ${error}`); + error.statusCode = 500; + throw error; + } +}; + +exports.readFolder = (folderPath) => { + const folderContent = fs.readdirSync(folderPath) + .map(fileName => { + return path.join(folderPath, fileName); + }); + + const isFolder = fileName => { + return (fs.lstatSync(fileName).isDirectory() && fileName.split('/').pop() !== '__MACOSX'); + }; + + const isFile = fileName => { + return fs.lstatSync(fileName).isFile(); + }; + + const folders = folderContent.filter(isFolder); + const files = folderContent.filter(isFile); + + const regex = /(?=.*?(master_template))(?=.*?(.xlsx)$)/gi; + const masterTemplates = []; + const curationFiles = []; + files.forEach((file) => { + if (regex.test(file)) { + masterTemplates.push(file); + } else { + curationFiles.push(file); + } + }); + + return { folders, files, masterTemplates, curationFiles }; +}; + const fixUrl = function (val, elementName) { return elementName === 'URL' ? val.replace(/&/gi, '&') : val; }; diff --git a/resfulservice/src/utils/fileManager.js b/resfulservice/src/utils/fileManager.js index 4af92a5f..5b63b26c 100644 --- a/resfulservice/src/utils/fileManager.js +++ b/resfulservice/src/utils/fileManager.js @@ -10,6 +10,29 @@ const deleteFile = (path, req) => { }); }; +const deleteFolder = async (folderPath, req) => { + fs.rm(folderPath, { recursive: true, force: true }, (err) => { + if (err) { + return err; + } else { req.logger?.info('Directory deleted successfully'); } + }); +}; + +const getDirectoryFiles = (filesDirectory, filename) => { + let filesDirectoryValue = filesDirectory; + let parsedFileName = filename; + // Split the path by "/" + const pathArray = filename.split('/'); + + if (pathArray.length) { + parsedFileName = pathArray[pathArray.length - 1]; + pathArray.pop(); + filesDirectoryValue = path.join(filesDirectory, pathArray.join('/')); + } + + return { filesDirectoryValue, parsedFileName }; +}; + async function selectFile (filesDirectory, filename) { const files = await fs.promises.readdir(filesDirectory); @@ -17,33 +40,41 @@ async function selectFile (filesDirectory, filename) { const filePath = path.join(filesDirectory, file); const stats = await fs.promises.lstat(filePath); - if (stats.isDirectory()) { + if (file === filename) { + return file; + } else if (stats.isDirectory()) { const foundFile = await selectFile(filePath, filename); if (foundFile) return foundFile; - } else if (file === filename) { - return file; } } return null; } +function getFileExtension (file) { + return path.parse(file); +} + async function findFile (req) { - if (!req.env?.FILES_DIRECTORY) { + const { fileId } = req.params; + + if (!req.env?.FILES_DIRECTORY || !fileId) { return null; } - const { fileId } = req.params; - const foundFile = await selectFile(req.env?.FILES_DIRECTORY, fileId); + const { filesDirectoryValue, parsedFileName } = getDirectoryFiles(req.env?.FILES_DIRECTORY, fileId); + + const foundFile = await selectFile(filesDirectoryValue, parsedFileName); if (!foundFile) { return null; } - const filePath = path.join(req.env?.FILES_DIRECTORY, foundFile); + const filePath = path.join(filesDirectoryValue, parsedFileName); + const { ext } = getFileExtension(foundFile); - // Stream the file to the client response - return fs.createReadStream(filePath); + // Stream the file to the client response and send file extension + return { fileStream: fs.createReadStream(filePath), ext }; } -module.exports = { deleteFile, findFile }; +module.exports = { deleteFile, findFile, getFileExtension, deleteFolder }; diff --git a/resfulservice/src/utils/minio.js b/resfulservice/src/utils/minio.js new file mode 100644 index 00000000..6282cbc3 --- /dev/null +++ b/resfulservice/src/utils/minio.js @@ -0,0 +1,28 @@ +const Minio = require('minio'); +const { MinioBucket } = require('../../config/constant'); +const env = process.env; + +const minioClient = new Minio.Client({ + endPoint: 'minio', + port: parseInt(env.MINIO_PORT), + useSSL: false, + accessKey: env.MINIO_ROOT_USER, + secretKey: env.MINIO_ROOT_PASSWORD +}); + +minioClient.bucketExists(env.MINIO_BUCKET ?? MinioBucket, function (err, exists) { + if (err) { + return console.log(err); + } + if (exists) { + return console.log('Bucket exists.'); + } else { + minioClient.makeBucket(env.MINIO_BUCKET ?? MinioBucket, 'us-east-1', function (err) { + if (err) return console.log(err); + + console.log('Bucket created successfully in "us-east-1".'); + }); + } +}); + +module.exports = minioClient;