diff --git a/.github/workflows/full.yml b/.github/workflows/full.yml index 30a926bec736..1ccbd02c0fbf 100644 --- a/.github/workflows/full.yml +++ b/.github/workflows/full.yml @@ -305,6 +305,7 @@ jobs: -f docker-compose.yml \ -f docker-compose.dev.yml \ -f components/serverless/docker-compose.serverless.yml \ + -f tests/docker-compose.minio.yml \ -f tests/docker-compose.file_share.yml up -d - name: Waiting for server diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9e9e4d909724..b571e61d331f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -266,6 +266,7 @@ jobs: -f docker-compose.yml \ -f docker-compose.dev.yml \ -f components/serverless/docker-compose.serverless.yml \ + -f tests/docker-compose.minio.yml \ -f tests/docker-compose.file_share.yml up -d - name: Waiting for server diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml index a435aa05c360..8640c45d60d2 100644 --- a/.github/workflows/schedule.yml +++ b/.github/workflows/schedule.yml @@ -319,6 +319,7 @@ jobs: -f docker-compose.yml \ -f docker-compose.dev.yml \ -f tests/docker-compose.file_share.yml \ + -f tests/docker-compose.minio.yml \ -f components/serverless/docker-compose.serverless.yml up -d - name: Waiting for server diff --git a/cvat-core/src/annotations.ts b/cvat-core/src/annotations.ts index 3b1fdbd33579..046ae0a1ece9 100644 --- a/cvat-core/src/annotations.ts +++ b/cvat-core/src/annotations.ts @@ -4,6 +4,7 @@ // SPDX-License-Identifier: MIT import { Storage } from './storage'; + const serverProxy = require('./server-proxy').default; const Collection = require('./annotations-collection'); const AnnotationsSaver = require('./annotations-saver'); diff --git a/cvat-ui/src/components/select-cloud-storage/select-cloud-storage.tsx b/cvat-ui/src/components/select-cloud-storage/select-cloud-storage.tsx index e4a220a05c68..376fa72f3408 100644 --- a/cvat-ui/src/components/select-cloud-storage/select-cloud-storage.tsx +++ b/cvat-ui/src/components/select-cloud-storage/select-cloud-storage.tsx @@ -127,6 +127,7 @@ function SelectCloudStorage(props: Props): JSX.Element { setSearchPhrase(selectedCloudStorage?.displayName || ''); }} allowClear + className={`cvat-search${!name ? '-' : `-${name[0].replace('Storage', '-storage')}-`}cloud-storage-field`} > diff --git a/cvat-ui/src/components/storage/storage-field.tsx b/cvat-ui/src/components/storage/storage-field.tsx index c9ebf2a0a1d9..410c3218a84d 100644 --- a/cvat-ui/src/components/storage/storage-field.tsx +++ b/cvat-ui/src/components/storage/storage-field.tsx @@ -31,6 +31,11 @@ export default function StorageField(props: Props): JSX.Element { } = props; const [cloudStorage, setCloudStorage] = useState(null); const [potentialCloudStorage, setPotentialCloudStorage] = useState(''); + const [storageType, setStorageType] = useState(''); + + useEffect(() => { + setStorageType(locationName[0].replace('Storage', '-storage')); + }, [locationName]); function renderCloudStorage(): JSX.Element { return ( @@ -65,6 +70,7 @@ export default function StorageField(props: Props): JSX.Element { <> {locationValue === StorageLocation.CLOUD_STORAGE && renderCloudStorage()} diff --git a/site/content/en/docs/contributing/running-tests.md b/site/content/en/docs/contributing/running-tests.md index 7435ebe938a8..a6ccf1b19ceb 100644 --- a/site/content/en/docs/contributing/running-tests.md +++ b/site/content/en/docs/contributing/running-tests.md @@ -14,6 +14,7 @@ description: 'Instructions on how to run all existence tests.' -f docker-compose.yml \ -f docker-compose.dev.yml \ -f components/serverless/docker-compose.serverless.yml \ + -f tests/docker-compose.minio.yml \ -f tests/docker-compose.file_share.yml up -d ``` 1. Add test user in CVAT: diff --git a/tests/cypress/integration/actions_projects_models/case_117_backup_restore_project_to_various_storages.js b/tests/cypress/integration/actions_projects_models/case_117_backup_restore_project_to_various_storages.js new file mode 100644 index 000000000000..8581fb4db70a --- /dev/null +++ b/tests/cypress/integration/actions_projects_models/case_117_backup_restore_project_to_various_storages.js @@ -0,0 +1,165 @@ +// Copyright (C) 2022 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +/// + +context('Tests source & target storage for backups.', () => { + const backupArchiveName = 'project_backup'; + let projectID = ''; + let createdCloudStorageId; + const caseId = '117'; + const taskName = `Case ${caseId}`; + const labelName = 'car'; + const attrName = 'color'; + const textDefaultValue = 'red'; + const imagesCount = 1; + const imageFileName = `image_${taskName.replace(/\s+/g, '_').toLowerCase()}`; + const width = 800; + const height = 800; + const posX = 10; + const posY = 10; + const color = 'gray'; + const dataArchiveName = `${imageFileName}.zip`; + const archivePath = `cypress/fixtures/${dataArchiveName}`; + const imagesFolder = `cypress/fixtures/${imageFileName}`; + const directoryToArchive = imagesFolder; + + const serverHost = Cypress.config('baseUrl').includes('3000') ? 'localhost' : 'minio'; + + const cloudStorageData = { + displayName: 'Demo bucket', + resource: 'public', + manifest: 'manifest.jsonl', + endpointUrl: `http://${serverHost}:9000`, + }; + + const storageConnectedToCloud = { + location: 'Cloud storage', + cloudStorageId: undefined, + }; + + const project = { + name: `Case ${caseId}`, + label: labelName, + attrName: 'color', + attrVaue: 'red', + multiAttrParams: false, + advancedConfiguration: { + sourceStorage: { + ...storageConnectedToCloud, + displayName: cloudStorageData.displayName, + }, + targetStorage: { + ...storageConnectedToCloud, + displayName: cloudStorageData.displayName, + }, + }, + }; + + const task = { + name: taskName, + label: labelName, + attrName, + textDefaultValue, + dataArchiveName, + multiAttrParams: false, + forProject: true, + attachToProject: true, + projectName: project.name, + advancedConfiguration: { + sourceStorage: { + disableSwitch: true, + location: 'Local', + cloudStorageId: null, + }, + targetStorage: { + disableSwitch: true, + location: 'Local', + cloudStorageId: null, + }, + }, + }; + + function getProjectID() { + cy.url().then((url) => { + projectID = Number(url.split('/').slice(-1)[0].split('?')[0]); + }); + } + + before(() => { + cy.visit('auth/login'); + cy.login(); + createdCloudStorageId = cy.attachS3Bucket(cloudStorageData); + cy.imageGenerator(imagesFolder, imageFileName, width, height, color, posX, posY, labelName, imagesCount); + cy.createZipArchive(directoryToArchive, archivePath); + cy.goToProjectsList(); + project.advancedConfiguration.sourceStorage.cloudStorageId = createdCloudStorageId; + project.advancedConfiguration.targetStorage.cloudStorageId = createdCloudStorageId; + + cy.createProjects( + project.name, + project.label, + project.attrName, + project.attrVaue, + project.multiAttrParams, + project.advancedConfiguration, + ); + + cy.goToTaskList(); + cy.createAnnotationTask( + task.name, + task.label, + task.attrName, + task.textDefaultValue, + dataArchiveName, + task.multiAttrParams, + null, + task.forProject, + task.attachToProject, + task.projectName, + ); + cy.openProject(project.name); + getProjectID(); + }); + + after(() => { + cy.goToCloudStoragesPage(); + cy.deleteCloudStorage(cloudStorageData.displayName); + cy.logout(); + cy.getAuthKey().then((authKey) => { + cy.deleteProjects(authKey, [project.name]); + }); + }); + + describe(`Testing case "${caseId}"`, () => { + it('Export project to custom local storage', () => { + cy.goToProjectsList(); + cy.backupProject( + project.name, + backupArchiveName, + { location: 'Local' }, + false, + ); + cy.waitForDownload(); + }); + + it('Export project to default minio bucket', () => { + cy.goToProjectsList(); + cy.backupProject( + project.name, + backupArchiveName, + project.advancedConfiguration.targetStorage, + ); + cy.waitForFileUploadToCloudStorage(); + cy.deleteProject(project.name, projectID); + }); + + it('Import project from minio bucket', () => { + cy.restoreProject( + `${backupArchiveName}.zip`, + project.advancedConfiguration.sourceStorage, + ); + }); + }); +}); diff --git a/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js b/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js index 81f6c36f903d..bacdf12cb3d2 100644 --- a/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js +++ b/tests/cypress/integration/actions_tasks/issue_2473_import_annotations_frames_dots_in_name.js @@ -112,12 +112,12 @@ context('Import annotations for frames with dots in name.', { browser: '!firefox it('Upload annotation with YOLO format to job.', () => { cy.interactMenu('Upload annotations'); + cy.intercept('GET', '/api/jobs/**/annotations?**').as('uploadAnnotationsGet'); uploadAnnotation( dumpType.split(' ')[0], annotationArchiveName, '.cvat-modal-content-load-job-annotation', ); - cy.intercept('GET', '/api/jobs/**/annotations?**').as('uploadAnnotationsGet'); cy.wait('@uploadAnnotationsGet').its('response.statusCode').should('equal', 200); cy.contains('Annotations have been loaded').should('be.visible'); cy.closeNotification('.ant-notification-notice-info'); diff --git a/tests/cypress/integration/actions_tasks3/case_113_use_default_project_storage_for_import_export_annotations.js b/tests/cypress/integration/actions_tasks3/case_113_use_default_project_storage_for_import_export_annotations.js new file mode 100644 index 000000000000..6e54fb0315c5 --- /dev/null +++ b/tests/cypress/integration/actions_tasks3/case_113_use_default_project_storage_for_import_export_annotations.js @@ -0,0 +1,185 @@ +// Copyright (C) 2022 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +/// + +context('Tests for source and target storage.', () => { + let createdCloudStorageId; + + const caseId = '113'; + + const taskName = `Case ${caseId}`; + const labelName = 'car'; + const attrName = 'color'; + const textDefaultValue = 'red'; + const imagesCount = 1; + const imageFileName = `image_${taskName.replace(/\s+/g, '_').toLowerCase()}`; + const width = 800; + const height = 800; + const posX = 10; + const posY = 10; + const color = 'gray'; + const dataArchiveName = `${imageFileName}.zip`; + const archivePath = `cypress/fixtures/${dataArchiveName}`; + const imagesFolder = `cypress/fixtures/${imageFileName}`; + const directoryToArchive = imagesFolder; + const format = 'CVAT for images'; + + const createRectangleShape2Points = { + points: 'By 2 Points', + type: 'Shape', + labelName, + firstX: 250, + firstY: 350, + secondX: 350, + secondY: 450, + }; + + const serverHost = Cypress.config('baseUrl').includes('3000') ? 'localhost' : 'minio'; + + const cloudStorageData = { + displayName: 'Demo bucket', + resource: 'public', + manifest: 'manifest.jsonl', + endpointUrl: `http://${serverHost}:9000`, + }; + + const storageConnectedToCloud = { + location: 'Cloud storage', + cloudStorageId: undefined, + }; + + const project = { + name: `Case ${caseId}`, + label: labelName, + attrName: 'color', + attrVaue: 'red', + multiAttrParams: false, + advancedConfiguration: { + sourceStorage: { + ...storageConnectedToCloud, + displayName: cloudStorageData.displayName, + }, + targetStorage: { + ...storageConnectedToCloud, + displayName: cloudStorageData.displayName, + }, + }, + }; + + const task = { + name: taskName, + label: labelName, + attrName, + textDefaultValue, + dataArchiveName, + multiAttrParams: false, + forProject: true, + attachToProject: true, + projectName: project.name, + advancedConfiguration: { + sourceStorage: { + disableSwitch: true, + location: 'Local', + cloudStorageId: null, + }, + targetStorage: { + disableSwitch: true, + location: 'Local', + cloudStorageId: null, + }, + }, + }; + + before(() => { + cy.visit('auth/login'); + cy.login(); + createdCloudStorageId = cy.attachS3Bucket(cloudStorageData); + cy.imageGenerator(imagesFolder, imageFileName, width, height, color, posX, posY, labelName, imagesCount); + cy.createZipArchive(directoryToArchive, archivePath); + cy.goToProjectsList(); + project.advancedConfiguration.sourceStorage.cloudStorageId = createdCloudStorageId; + project.advancedConfiguration.targetStorage.cloudStorageId = createdCloudStorageId; + + cy.createProjects( + project.name, + project.label, + project.attrName, + project.attrVaue, + project.multiAttrParams, + project.advancedConfiguration, + ); + }); + + after(() => { + cy.goToCloudStoragesPage(); + cy.deleteCloudStorage(cloudStorageData.displayName); + cy.logout(); + cy.getAuthKey().then((authKey) => { + cy.deleteProjects(authKey, [project.name]); + }); + }); + + describe(`Testing case "${caseId}"`, () => { + it('Export job annotations to default minio bucket that was attached to the project.', () => { + // create an annotation task with default project source and target storages + cy.goToTaskList(); + cy.createAnnotationTask( + task.name, + task.label, + task.attrName, + task.textDefaultValue, + dataArchiveName, + task.multiAttrParams, + null, + task.forProject, + task.attachToProject, + task.projectName, + ); + cy.goToTaskList(); + cy.openTask(task.name); + + // create dummy annotations and export them to "public" minio bucket + cy.openJob(); + cy.createRectangle(createRectangleShape2Points).then(() => { + Cypress.config('scrollBehavior', false); + }); + cy.saveJob('PATCH', 200, 'saveJobDump'); + const exportParams = { + type: 'annotations', + format, + archiveCustomeName: 'job_annotations', + targetStorage: project.advancedConfiguration.targetStorage, + }; + cy.exportJob(exportParams); + cy.waitForFileUploadToCloudStorage(); + + // remove annotations + cy.removeAnnotations(); + cy.saveJob('PUT'); + cy.get('#cvat_canvas_shape_1').should('not.exist'); + cy.get('#cvat-objects-sidebar-state-item-1').should('not.exist'); + }); + + it('Import job annotations from default minio bucket that was attached to the project.', () => { + // upload annotations from "public" minio bucket + cy.interactMenu('Upload annotations'); + cy.intercept('GET', '/api/jobs/**/annotations?**').as('uploadAnnotationsGet'); + + cy.uploadAnnotations( + format.split(' ')[0], + 'job_annotations.zip', + '.cvat-modal-content-load-job-annotation', + project.advancedConfiguration.sourceStorage, + ); + + cy.get('.cvat-notification-notice-upload-annotations-fail').should('not.exist'); + cy.get('#cvat_canvas_shape_1').should('exist'); + cy.get('#cvat-objects-sidebar-state-item-1').should('exist'); + + cy.goToTaskList(); + cy.deleteTask(task.name); + }); + }); +}); diff --git a/tests/cypress/integration/actions_tasks3/case_114_use_default_task_storage_for_import_export_annotations.js b/tests/cypress/integration/actions_tasks3/case_114_use_default_task_storage_for_import_export_annotations.js new file mode 100644 index 000000000000..70d01434a2e0 --- /dev/null +++ b/tests/cypress/integration/actions_tasks3/case_114_use_default_task_storage_for_import_export_annotations.js @@ -0,0 +1,194 @@ +// Copyright (C) 2022 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +/// + +context('Tests for source and target storage.', () => { + let annotationsArchiveName = ''; + let createdCloudStorageId; + + const caseId = '114'; + + const taskName = `Case ${caseId}`; + const labelName = 'car'; + const attrName = 'color'; + const textDefaultValue = 'red'; + const imagesCount = 1; + const imageFileName = `image_${taskName.replace(/\s+/g, '_').toLowerCase()}`; + const width = 800; + const height = 800; + const posX = 10; + const posY = 10; + const color = 'gray'; + const dataArchiveName = `${imageFileName}.zip`; + const archivePath = `cypress/fixtures/${dataArchiveName}`; + const imagesFolder = `cypress/fixtures/${imageFileName}`; + const directoryToArchive = imagesFolder; + const format = 'CVAT for images'; + + const createRectangleShape2Points = { + points: 'By 2 Points', + type: 'Shape', + labelName, + firstX: 250, + firstY: 350, + secondX: 350, + secondY: 450, + }; + + const serverHost = Cypress.config('baseUrl').includes('3000') ? 'localhost' : 'minio'; + + const cloudStorageData = { + displayName: 'Demo bucket', + resource: 'public', + manifest: 'manifest.jsonl', + endpointUrl: `http://${serverHost}:9000`, + }; + + const storageConnectedToCloud = { + location: 'Cloud storage', + cloudStorageId: undefined, + }; + + const project = { + name: `Case ${caseId}`, + label: labelName, + attrName: 'color', + attrVaue: 'red', + multiAttrParams: false, + advancedConfiguration: { + sourceStorage: { + ...storageConnectedToCloud, + displayName: cloudStorageData.displayName, + }, + targetStorage: { + ...storageConnectedToCloud, + displayName: cloudStorageData.displayName, + }, + }, + }; + + const task = { + name: taskName, + label: labelName, + attrName, + textDefaultValue, + dataArchiveName, + multiAttrParams: false, + forProject: true, + attachToProject: true, + projectName: project.name, + advancedConfiguration: { + sourceStorage: { + disableSwitch: true, + location: 'Local', + cloudStorageId: null, + }, + targetStorage: { + disableSwitch: true, + location: 'Local', + cloudStorageId: null, + }, + }, + }; + + before(() => { + cy.visit('auth/login'); + cy.login(); + createdCloudStorageId = cy.attachS3Bucket(cloudStorageData); + cy.imageGenerator(imagesFolder, imageFileName, width, height, color, posX, posY, labelName, imagesCount); + cy.createZipArchive(directoryToArchive, archivePath); + cy.goToProjectsList(); + project.advancedConfiguration.sourceStorage.cloudStorageId = createdCloudStorageId; + project.advancedConfiguration.targetStorage.cloudStorageId = createdCloudStorageId; + + cy.createProjects( + project.name, + project.label, + project.attrName, + project.attrVaue, + project.multiAttrParams, + project.advancedConfiguration, + ); + }); + + after(() => { + cy.goToCloudStoragesPage(); + cy.deleteCloudStorage(cloudStorageData.displayName); + cy.logout(); + cy.getAuthKey().then((authKey) => { + cy.deleteProjects(authKey, [project.name]); + }); + }); + + describe(`Testing case "${caseId}"`, () => { + it('Export job annotations to default minio bucket that was attached to the task in the project.', () => { + // create an annotation task with custom local source & target storages + cy.goToTaskList(); + cy.createAnnotationTask( + task.name, + task.label, + task.attrName, + task.textDefaultValue, + dataArchiveName, + task.multiAttrParams, + task.advancedConfiguration, + task.forProject, + task.attachToProject, + task.projectName, + ); + cy.goToTaskList(); + cy.openTask(task.name); + + // create dummy annotations and export them to default local storage + cy.openJob(); + cy.createRectangle(createRectangleShape2Points).then(() => { + Cypress.config('scrollBehavior', false); + }); + + cy.saveJob('PATCH', 200, 'saveJobDump'); + const exportParams = { + type: 'annotations', + format, + archiveCustomeName: 'job_annotations', + targetStorage: project.advancedConfiguration.targetStorage, + }; + cy.exportJob(exportParams); + cy.getDownloadFileName().then((file) => { + annotationsArchiveName = file; + cy.verifyDownload(annotationsArchiveName); + }); + cy.verifyNotification(); + + // remove annotations + cy.removeAnnotations(); + cy.saveJob('PUT'); + cy.get('#cvat_canvas_shape_1').should('not.exist'); + cy.get('#cvat-objects-sidebar-state-item-1').should('not.exist'); + }); + + it('Import job annotations from default minio bucket that was attached to the task in the project.', () => { + cy.goToTaskList(); + cy.openTask(task.name); + cy.openJob(); + + // upload annotations from default local storage + cy.interactMenu('Upload annotations'); + cy.intercept('GET', '/api/jobs/**/annotations?**').as('uploadAnnotationsGet'); + cy.uploadAnnotations( + format.split(' ')[0], + annotationsArchiveName, + '.cvat-modal-content-load-job-annotation', + task.advancedConfiguration.sourceStorage, + ); + + cy.get('.cvat-notification-notice-upload-annotations-fail').should('not.exist'); + cy.get('#cvat_canvas_shape_1').should('exist'); + cy.get('#cvat-objects-sidebar-state-item-1').should('exist'); + + cy.goToTaskList(); + cy.deleteTask(task.name); + }); + }); +}); diff --git a/tests/cypress/integration/actions_tasks3/case_115_use_custom_storage_for_import_export_annotations.js b/tests/cypress/integration/actions_tasks3/case_115_use_custom_storage_for_import_export_annotations.js new file mode 100644 index 000000000000..5fb7e1d816ff --- /dev/null +++ b/tests/cypress/integration/actions_tasks3/case_115_use_custom_storage_for_import_export_annotations.js @@ -0,0 +1,155 @@ +// Copyright (C) 2022 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +/// + +context('Import and export annotations: specify source and target storage in modals.', () => { + let createdCloudStorageId; + const caseId = '115'; + const taskName = `Case ${caseId}`; + const labelName = 'car'; + const attrName = 'color'; + const textDefaultValue = 'red'; + const imagesCount = 1; + const imageFileName = `image_${taskName.replace(/\s+/g, '_').toLowerCase()}`; + const width = 800; + const height = 800; + const posX = 10; + const posY = 10; + const color = 'gray'; + const dataArchiveName = `${imageFileName}.zip`; + const archivePath = `cypress/fixtures/${dataArchiveName}`; + const imagesFolder = `cypress/fixtures/${imageFileName}`; + const directoryToArchive = imagesFolder; + const format = 'CVAT for images'; + + const createRectangleShape2Points = { + points: 'By 2 Points', + type: 'Shape', + labelName, + firstX: 250, + firstY: 350, + secondX: 350, + secondY: 450, + }; + + const serverHost = Cypress.config('baseUrl').includes('3000') ? 'localhost' : 'minio'; + + const cloudStorageData = { + displayName: 'Demo bucket', + resource: 'public', + manifest: 'manifest.jsonl', + endpointUrl: `http://${serverHost}:9000`, + }; + + const project = { + name: `Case ${caseId}`, + label: labelName, + attrName: 'color', + attrVaue: 'red', + multiAttrParams: false, + }; + + const task = { + name: taskName, + label: labelName, + attrName, + textDefaultValue, + dataArchiveName, + multiAttrParams: false, + forProject: true, + attachToProject: true, + projectName: project.name, + }; + + before(() => { + cy.visit('auth/login'); + cy.login(); + createdCloudStorageId = cy.attachS3Bucket(cloudStorageData); + cy.imageGenerator(imagesFolder, imageFileName, width, height, color, posX, posY, labelName, imagesCount); + cy.createZipArchive(directoryToArchive, archivePath); + cy.goToProjectsList(); + + cy.createProjects( + project.name, + project.label, + project.attrName, + project.attrVaue, + project.multiAttrParams, + ); + }); + + after(() => { + cy.goToCloudStoragesPage(); + cy.deleteCloudStorage(cloudStorageData.displayName); + cy.logout(); + cy.getAuthKey().then((authKey) => { + cy.deleteProjects(authKey, [project.name]); + }); + }); + + describe(`Testing case "${caseId}"`, () => { + it('Export job annotations to custom minio bucket', () => { + // create an annotation task with local source & target storages + cy.goToTaskList(); + cy.createAnnotationTask( + task.name, + task.label, + task.attrName, + task.textDefaultValue, + dataArchiveName, + task.multiAttrParams, + null, + task.forProject, + task.attachToProject, + task.projectName, + ); + cy.goToTaskList(); + cy.openTask(task.name); + cy.openJob(); + + // create dummy annotations and export them to "public" minio bucket + cy.createRectangle(createRectangleShape2Points).then(() => { + Cypress.config('scrollBehavior', false); + }); + cy.saveJob('PATCH', 200, 'saveJobDump'); + const exportParams = { + type: 'annotations', + format, + archiveCustomeName: 'job_annotations', + targetStorage: { + location: 'Cloud storage', + cloudStorageId: createdCloudStorageId, + }, + useDefaultLocation: false, + }; + cy.exportJob(exportParams); + cy.waitForFileUploadToCloudStorage(); + + // remove annotations + cy.removeAnnotations(); + cy.saveJob('PUT'); + cy.get('#cvat_canvas_shape_1').should('not.exist'); + cy.get('#cvat-objects-sidebar-state-item-1').should('not.exist'); + }); + + it('Import job annotations from custom minio "public" bucket', () => { + cy.interactMenu('Upload annotations'); + cy.intercept('GET', '/api/jobs/**/annotations?**').as('uploadAnnotationsGet'); + cy.uploadAnnotations( + format.split(' ')[0], + 'job_annotations.zip', + '.cvat-modal-content-load-job-annotation', + { + location: 'Cloud storage', + cloudStorageId: createdCloudStorageId, + }, + false, + ); + cy.get('.cvat-notification-notice-upload-annotations-fail').should('not.exist'); + cy.get('#cvat_canvas_shape_1').should('exist'); + cy.get('#cvat-objects-sidebar-state-item-1').should('exist'); + }); + }); +}); diff --git a/tests/cypress/support/commands.js b/tests/cypress/support/commands.js index b577d2a959e8..2bf4b917fc15 100644 --- a/tests/cypress/support/commands.js +++ b/tests/cypress/support/commands.js @@ -645,6 +645,41 @@ Cypress.Commands.add('advancedConfiguration', (advancedConfigurationParams) => { if (advancedConfigurationParams.overlapSize) { cy.get('#overlapSize').type(advancedConfigurationParams.overlapSize); } + if (advancedConfigurationParams.sourceStorage) { + const { sourceStorage } = advancedConfigurationParams; + + if (sourceStorage.disableSwitch) { + cy.get('.ant-collapse-content-box').find('#useProjectSourceStorage').click(); + } + + cy.get('.cvat-select-source-storage').within(() => { + cy.get('.ant-select-selection-item').click(); + }); + cy.contains('.cvat-select-source-storage-location', sourceStorage.location).should('be.visible').click(); + + if (sourceStorage.cloudStorageId) { + cy.get('.cvat-search-source-storage-cloud-storage-field').click(); + cy.get('.cvat-cloud-storage-select-provider').click(); + } + } + + if (advancedConfigurationParams.targetStorage) { + const { targetStorage } = advancedConfigurationParams; + + if (targetStorage.disableSwitch) { + cy.get('.ant-collapse-content-box').find('#useProjectTargetStorage').click(); + } + + cy.get('.cvat-select-target-storage').within(() => { + cy.get('.ant-select-selection-item').click(); + }); + cy.contains('.cvat-select-target-storage-location', targetStorage.location).should('be.visible').click(); + + if (targetStorage.cloudStorageId) { + cy.get('.cvat-search-target-storage-cloud-storage-field').click(); + cy.get('.cvat-cloud-storage-select-provider').last().click(); + } + } }); Cypress.Commands.add('removeAnnotations', () => { @@ -657,6 +692,57 @@ Cypress.Commands.add('removeAnnotations', () => { }); }); +Cypress.Commands.add('confirmUpdate', (modalWindowClassName) => { + cy.get(modalWindowClassName).should('be.visible').within(() => { + cy.contains('button', 'Update').click(); + }); +}); + +Cypress.Commands.add( + 'uploadAnnotations', ( + format, + file, + confirmModalClassName, + sourceStorage = null, + useDefaultLocation = true, + ) => { + cy.get('.cvat-modal-import-dataset').find('.cvat-modal-import-select').click(); + cy.contains('.cvat-modal-import-dataset-option-item', format).click(); + cy.get('.cvat-modal-import-select').should('contain.text', format); + + if (!useDefaultLocation) { + cy.get('.cvat-modal-import-dataset') + .find('.cvat-modal-import-switch-use-default-storage') + .click(); + cy.get('.cvat-select-source-storage').within(() => { + cy.get('.ant-select-selection-item').click(); + }); + cy.contains('.cvat-select-source-storage-location', sourceStorage.location) + .should('be.visible') + .click(); + if (sourceStorage.cloudStorageId) { + cy.get('.cvat-search-source-storage-cloud-storage-field').click(); + cy.get('.cvat-cloud-storage-select-provider').click(); + } + } + if (sourceStorage && sourceStorage.cloudStorageId) { + cy.get('.cvat-modal-import-dataset') + .find('.cvat-modal-import-filename-input') + .type(file); + } else { + cy.get('input[type="file"]').attachFile(file, { subjectType: 'drag-n-drop' }); + cy.get(`[title="${file}"]`).should('be.visible'); + } + cy.contains('button', 'OK').click(); + cy.confirmUpdate(confirmModalClassName); + cy.get('.cvat-notification-notice-import-annotation-start').should('be.visible'); + cy.closeNotification('.cvat-notification-notice-import-annotation-start'); + cy.wait('@uploadAnnotationsGet').its('response.statusCode').should('equal', 200); + cy.contains('Annotations have been loaded').should('be.visible'); + cy.closeNotification('.ant-notification-notice-info'); + }, +); + Cypress.Commands.add('goToTaskList', () => { cy.get('a[value="tasks"]').click(); cy.url().should('include', '/tasks'); @@ -877,6 +963,7 @@ Cypress.Commands.add('exportTask', ({ Cypress.Commands.add('exportJob', ({ type, format, archiveCustomeName, + targetStorage = null, useDefaultLocation = true, }) => { cy.interactMenu('Export job dataset'); cy.get('.cvat-modal-export-job').should('be.visible').find('.cvat-modal-export-select').click(); @@ -888,6 +975,18 @@ Cypress.Commands.add('exportJob', ({ if (archiveCustomeName) { cy.get('.cvat-modal-export-job').find('.cvat-modal-export-filename-input').type(archiveCustomeName); } + if (!useDefaultLocation) { + cy.get('.cvat-modal-export-job').find('.cvat-settings-switch').click(); + cy.get('.cvat-select-target-storage').within(() => { + cy.get('.ant-select-selection-item').click(); + }); + cy.contains('.cvat-select-target-storage-location', targetStorage.location).should('be.visible').click(); + + if (targetStorage.cloudStorageId) { + cy.get('.cvat-search-target-storage-cloud-storage-field').click(); + cy.get('.cvat-cloud-storage-select-provider').click(); + } + } cy.contains('button', 'OK').click(); cy.get('.cvat-notification-notice-export-job-start').should('be.visible'); cy.closeNotification('.cvat-notification-notice-export-job-start'); @@ -952,6 +1051,25 @@ Cypress.Commands.add('verifyNotification', () => { cy.closeNotification('.ant-notification-notice-info'); }); +Cypress.Commands.add('goToCloudStoragesPage', () => { + cy.get('a[value="cloudstorages"]').click(); + cy.url().should('include', '/cloudstorages'); +}); + +Cypress.Commands.add('deleteCloudStorage', (displayName) => { + cy.get('.cvat-cloud-storage-item-menu-button').trigger('mousemove').trigger('mouseover'); + cy.get('.ant-dropdown') + .not('.ant-dropdown-hidden') + .within(() => { + cy.contains('[role="menuitem"]', 'Delete').click(); + }); + cy.get('.cvat-delete-cloud-storage-modal') + .should('contain', `You are going to remove the cloudstorage "${displayName}"`) + .within(() => { + cy.contains('button', 'Delete').click(); + }); +}); + Cypress.Commands.overwrite('visit', (orig, url, options) => { orig(url, options); cy.closeModalUnsupportedPlatform(); diff --git a/tests/cypress/support/commands_cloud_storages.js b/tests/cypress/support/commands_cloud_storages.js new file mode 100644 index 000000000000..ad65f0bdba32 --- /dev/null +++ b/tests/cypress/support/commands_cloud_storages.js @@ -0,0 +1,46 @@ +// Copyright (C) 2022 CVAT.ai Corp +// +// SPDX-License-Identifier: MIT + +/// + +Cypress.Commands.add('attachS3Bucket', (data) => { + let createdCloudStorageId; + cy.contains('.cvat-header-button', 'Cloud Storages').should('be.visible').click(); + cy.get('.cvat-attach-cloud-storage-button').should('be.visible').click(); + cy.get('#display_name') + .type(data.displayName) + .should('have.attr', 'value', data.displayName); + cy.get('#provider_type').click(); + cy.contains('.cvat-cloud-storage-select-provider', 'AWS').click(); + cy.get('#resource') + .should('exist') + .type(data.resource) + .should('have.attr', 'value', data.resource); + cy.get('#credentials_type').should('exist').click(); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .get('[title="Anonymous access"]') + .should('be.visible') + .click(); + cy.get('#endpoint_url') + .type(data.endpointUrl) + .should('have.attr', 'value', data.endpointUrl); + + cy.get('.cvat-add-manifest-button').should('be.visible').click(); + cy.get('[placeholder="manifest.jsonl"]') + .should('exist') + .should('have.attr', 'value', '') + .type(data.manifest) + .should('have.attr', 'value', data.manifest); + cy.intercept('POST', /\/api\/cloudstorages.*/).as('createCloudStorage'); + cy.get('.cvat-cloud-storage-form').within(() => { + cy.contains('button', 'Submit').click(); + }); + cy.wait('@createCloudStorage').then((interseption) => { + expect(interseption.response.statusCode).to.be.equal(201); + createdCloudStorageId = interseption.response.body.id; + }); + cy.verifyNotification(); + return createdCloudStorageId; +}); diff --git a/tests/cypress/support/commands_projects.js b/tests/cypress/support/commands_projects.js index 21efb8b4f146..a835c88aac10 100644 --- a/tests/cypress/support/commands_projects.js +++ b/tests/cypress/support/commands_projects.js @@ -1,4 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) 2022 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -12,7 +13,11 @@ Cypress.Commands.add('goToProjectsList', () => { Cypress.Commands.add( 'createProjects', - (projectName, labelName, attrName, textDefaultValue, multiAttrParams, expectedResult = 'success') => { + ( + projectName, labelName, attrName, textDefaultValue, + multiAttrParams, advancedConfigurationParams, + expectedResult = 'success', + ) => { cy.get('.cvat-create-project-dropdown').click(); cy.get('.cvat-create-project-button').click(); cy.get('#name').type(projectName); @@ -26,6 +31,9 @@ Cypress.Commands.add( if (multiAttrParams) { cy.updateAttributes(multiAttrParams); } + if (advancedConfigurationParams) { + cy.advancedConfiguration(advancedConfigurationParams); + } cy.contains('button', 'Continue').click(); cy.get('.cvat-create-project-content').within(() => { cy.contains('button', 'Submit & Continue').click(); @@ -136,34 +144,83 @@ Cypress.Commands.add('importProject', ({ cy.get('.cvat-modal-import-dataset-status').should('not.exist'); }); -Cypress.Commands.add('backupProject', (projectName, backupFileName) => { - cy.projectActions(projectName); - cy.get('.cvat-project-actions-menu').contains('Backup Project').click(); - cy.get('.cvat-modal-export-project').should('be.visible'); - if (backupFileName) { - cy.get('.cvat-modal-export-project').find('.cvat-modal-export-filename-input').type(backupFileName); - } - cy.get('.cvat-modal-export-project').contains('button', 'OK').click(); - cy.get('.cvat-notification-notice-export-backup-start').should('be.visible'); - cy.closeNotification('.cvat-notification-notice-export-backup-start'); -}); +Cypress.Commands.add( + 'backupProject', + ( + projectName, + backupFileName, + targetStorage = null, + useDefaultLocation = true, + ) => { + cy.projectActions(projectName); + cy.get('.cvat-project-actions-menu').contains('Backup Project').click(); + cy.get('.cvat-modal-export-project').should('be.visible'); + if (backupFileName) { + cy.get('.cvat-modal-export-project').find('.cvat-modal-export-filename-input').type(backupFileName); + } + if (!useDefaultLocation) { + cy.get('.cvat-modal-export-project') + .find('.cvat-settings-switch') + .click(); + cy.get('.cvat-select-target-storage').within(() => { + cy.get('.ant-select-selection-item').click(); + }); + cy.contains('.cvat-select-target-storage-location', targetStorage.location) + .should('be.visible') + .click(); -Cypress.Commands.add('restoreProject', (archiveWithBackup) => { + if (targetStorage && targetStorage.cloudStorageId) { + cy.get('.cvat-search-target-storage-cloud-storage-field').click(); + cy.get('.cvat-cloud-storage-select-provider').click(); + } + } + cy.get('.cvat-modal-export-project').contains('button', 'OK').click(); + cy.get('.cvat-notification-notice-export-backup-start').should('be.visible'); + cy.closeNotification('.cvat-notification-notice-export-backup-start'); + }, +); + +Cypress.Commands.add('restoreProject', (archiveWithBackup, sourceStorage = null) => { cy.intercept({ method: /PATCH|POST/, url: /\/api\/projects\/backup.*/ }).as('restoreProject'); cy.get('.cvat-create-project-dropdown').click(); cy.get('.cvat-import-project-button').click(); - cy.get('input[type=file]').attachFile(archiveWithBackup, { subjectType: 'drag-n-drop' }); - cy.get(`[title="${archiveWithBackup}"]`).should('be.visible'); + + if (sourceStorage) { + cy.get('.cvat-select-source-storage').within(() => { + cy.get('.ant-select-selection-item').click(); + }); + cy.contains('.cvat-select-source-storage-location', sourceStorage.location) + .should('be.visible') + .click(); + if (sourceStorage.cloudStorageId) { + cy.get('.cvat-search-source-storage-cloud-storage-field').click(); + cy.get('.cvat-cloud-storage-select-provider').click(); + cy.get('.cvat-modal-import-backup') + .find('.cvat-modal-import-filename-input') + .type(archiveWithBackup); + } + } else { + cy.get('input[type=file]').attachFile(archiveWithBackup, { subjectType: 'drag-n-drop' }); + cy.get(`[title="${archiveWithBackup}"]`).should('be.visible'); + } + cy.contains('button', 'OK').click(); cy.get('.cvat-notification-notice-import-backup-start').should('be.visible'); cy.closeNotification('.cvat-notification-notice-import-backup-start'); - cy.wait('@restoreProject').its('response.statusCode').should('equal', 202); - cy.wait('@restoreProject').its('response.statusCode').should('equal', 201); - cy.wait('@restoreProject').its('response.statusCode').should('equal', 204); - cy.wait('@restoreProject').its('response.statusCode').should('equal', 202); - cy.wait('@restoreProject', { timeout: 5000 }).its('response.statusCode').should('equal', 202); - cy.wait('@restoreProject').its('response.statusCode').should('equal', 201); + if (!sourceStorage || !sourceStorage.cloudStorageId) { + cy.wait('@restoreProject').its('response.statusCode').should('equal', 202); + cy.wait('@restoreProject').its('response.statusCode').should('equal', 201); + cy.wait('@restoreProject').its('response.statusCode').should('equal', 204); + cy.wait('@restoreProject').its('response.statusCode').should('equal', 202); + cy.wait('@restoreProject', { timeout: 5000 }).its('response.statusCode').should('equal', 202); + cy.wait('@restoreProject').its('response.statusCode').should('equal', 201); + } else { + cy.wait('@restoreProject').its('response.statusCode').should('equal', 202); + cy.wait('@restoreProject', { timeout: 3000 }).its('response.statusCode').should('equal', 202); + cy.wait('@restoreProject').its('response.statusCode').should('equal', 201); + } + cy.contains('The project has been restored succesfully. Click here to open') .should('exist') .and('be.visible'); @@ -179,6 +236,14 @@ Cypress.Commands.add('getDownloadFileName', () => { }); }); +Cypress.Commands.add('waitForFileUploadToCloudStorage', () => { + cy.intercept('GET', '**=download').as('download'); + cy.wait('@download', { requestTimeout: 7000 }).then((interseption) => { + expect(interseption.response.statusCode).to.be.equal(200); + }); + cy.verifyNotification(); +}); + Cypress.Commands.add('waitForDownload', () => { cy.getDownloadFileName().then((filename) => { cy.verifyDownload(filename); diff --git a/tests/cypress/support/index.js b/tests/cypress/support/index.js index dfa79a691dcd..8fc15b30144a 100644 --- a/tests/cypress/support/index.js +++ b/tests/cypress/support/index.js @@ -1,4 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) 2022 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -10,6 +11,7 @@ require('./commands_filters_feature'); require('./commands_models'); require('./commands_opencv'); require('./commands_organizations'); +require('./commands_cloud_storages'); require('@cypress/code-coverage/support'); require('cypress-real-events/support'); diff --git a/tests/python/shared/docker-compose.minio.yml b/tests/docker-compose.minio.yml similarity index 100% rename from tests/python/shared/docker-compose.minio.yml rename to tests/docker-compose.minio.yml diff --git a/tests/python/shared/fixtures/init.py b/tests/python/shared/fixtures/init.py index ffed4f695d90..4fc1bff59fdf 100644 --- a/tests/python/shared/fixtures/init.py +++ b/tests/python/shared/fixtures/init.py @@ -27,7 +27,7 @@ DC_FILES = [ osp.join(CVAT_ROOT_DIR, dc_file) - for dc_file in ("docker-compose.dev.yml", "tests/python/shared/docker-compose.minio.yml") + for dc_file in ("docker-compose.dev.yml", "tests/docker-compose.minio.yml") ] + CONTAINER_NAME_FILES