[cvat-core] support cloud storage

cvat-core/src/api-implementation.js

@@ -17,7 +17,11 @@
     } = require('./common');
 
     const {
-        TaskStatus, TaskMode, DimensionType, CloudStorageProviderType,
+        TaskStatus,
+        TaskMode,
+        DimensionType,
+        CloudStorageProviderType,
+        CloudStorageCredentialsType,
     } = require('./enums');
 
     const User = require('./user');
@@ -270,16 +274,27 @@
                 page: isInteger,
                 displayName: isString,
                 resourceName: isString,
+                description: isString,
                 id: isInteger,
                 owner: isString,
                 search: isString,
-                provider: isEnum.bind(CloudStorageProviderType),
+                providerType: isEnum.bind(CloudStorageProviderType),
+                credentialsType: isEnum.bind(CloudStorageCredentialsType),
             });
 
             checkExclusiveFields(filter, ['id', 'search'], ['page']);
 
             const searchParams = new URLSearchParams();
-            for (const field of ['displayName', 'owner', 'search', 'id', 'page']) {
+            for (const field of [
+                'displayName',
+                'credentialsType',
+                'providerType',
+                'owner',
+                'search',
+                'id',
+                'page',
+                'description',
+            ]) {
                 if (Object.prototype.hasOwnProperty.call(filter, field)) {
                     searchParams.set(camelToSnake(field), filter[field]);
                 }
@@ -289,11 +304,7 @@
                 searchParams.set('resource', filter.resourceName);
             }
 
-            if (Object.prototype.hasOwnProperty.call(filter, 'provider')) {
-                searchParams.set('provider_type', filter.provider);
-            }
-
-            const cloudStoragesData = await serverProxy.cloudStoarges.get(searchParams.toString());
+            const cloudStoragesData = await serverProxy.cloudStorages.get(searchParams.toString());
             const cloudStorages = cloudStoragesData.map((cloudStorage) => new CloudStorage(cloudStorage));
 
             cloudStorages.count = cloudStoragesData.count;

cvat-core/src/cloud-storage.js

@@ -5,6 +5,7 @@
 (() => {
     const PluginRegistry = require('./plugins');
     const serverProxy = require('./server-proxy');
+    const { isBrowser, isNode } = require('browser-or-node');
     const { ArgumentError } = require('./exceptions');
     const { CloudStorageCredentialsType, CloudStorageProviderType } = require('./enums');
 
@@ -351,59 +352,84 @@
             const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.getContent);
             return result;
         }
+
+        /**
+         * Method returns the cloud storage preview
+         * @method getPreview
+         * @memberof module:API.cvat.classes.CloudStorage
+         * @readonly
+         * @instance
+         * @async
+         * @throws {module:API.cvat.exceptions.ServerError}
+         * @throws {module:API.cvat.exceptions.PluginError}
+         */
+        async getPreview() {
+            const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.getPreview);
+            return result;
+        }
     }
 
     CloudStorage.prototype.save.implementation = async function () {
+        function prepareOptionalFields(cloudStorageInstance) {
+            const data = {};
+            if (cloudStorageInstance.description) {
+                data.description = cloudStorageInstance.description;
+            }
+
+            if (cloudStorageInstance.accountName) {
+                data.account_name = cloudStorageInstance.accountName;
+            }
+
+            if (cloudStorageInstance.accessKey) {
+                data.key = cloudStorageInstance.accessKey;
+            }
+
+            if (cloudStorageInstance.secretKey) {
+                data.secret_key = cloudStorageInstance.secretKey;
+            }
+
+            if (cloudStorageInstance.token) {
+                data.session_token = cloudStorageInstance.token;
+            }
+
+            if (cloudStorageInstance.specificAttibutes) {
+                data.specific_attibutes = cloudStorageInstance.specificAttibutes;
+            }
+            return data;
+        }
         // update
         if (typeof this.id !== 'undefined') {
+            // providr_type and recource should not change;
+            // send to the server only the values that have changed
+            const initialData = {};
+            if (this.displayName) {
+                initialData.display_name = this.displayName;
+            }
+            if (this.credentialsType) {
+                initialData.credentials_type = this.credentialsType;
+            }
+
             const cloudStorageData = {
-                display_name: this.displayName,
-                description: this.description ? this.description : null,
-                credentials_type: this.credentialsType ? this.credentialsType : null,
-                provided_type: this.provider ? this.provider : null,
-                resource: this.resourceName ? this.resourceName : null,
-                secret_key: this.secretKey ? this.secretKey : null,
-                session_token: this.token ? this.token : null,
-                key: this.accessKey ? this.accessKey : null,
-                account_name: this.accountName ? this.accountName : null,
-                specific_attibutes: this.specificAttibutes ? this.specificAttibutes : null,
+                ...initialData,
+                ...prepareOptionalFields(this),
             };
 
             await serverProxy.cloudStorages.update(this.id, cloudStorageData);
             return this;
         }
 
         // create
-        const cloudStorageData = {
+        const initialData = {
             display_name: this.displayName,
             credentials_type: this.credentialsType,
             provider_type: this.provider,
             resource: this.resourceName,
         };
 
-        if (this.description) {
-            cloudStorageData.description = this.description;
-        }
-
-        if (this.accountName) {
-            cloudStorageData.account_name = this.accountName;
-        }
-
-        if (this.accessKey) {
-            cloudStorageData.key = this.accessKey;
-        }
-
-        if (this.secretKey) {
-            cloudStorageData.secret_key = this.secretKey;
-        }
-
-        if (this.token) {
-            cloudStorageData.session_token = this.token;
-        }
-
-        if (this.specificAttibutes) {
-            cloudStorageData.specific_attibutes = this.specificAttibutes;
-        }
+        const cloudStorageData = {
+            ...initialData,
+            ...prepareOptionalFields(this),
+        };
 
         const cloudStorage = await serverProxy.cloudStorages.create(cloudStorageData);
         return new CloudStorage(cloudStorage);
@@ -419,6 +445,27 @@
         return result;
     };
 
+    CloudStorage.prototype.getPreview.implementation = async function getPreview() {
+        return new Promise((resolve, reject) => {
+            serverProxy.cloudStorages
+                .getPreview(this.id)
+                .then((result) => {
+                    if (isNode) {
+                        resolve(global.Buffer.from(result, 'binary').toString('base64'));
+                    } else if (isBrowser) {
+                        const reader = new FileReader();
+                        reader.onload = () => {
+                            resolve(reader.result);
+                        };
+                        reader.readAsDataURL(result);
+                    }
+                })
+                .catch((error) => {
+                    reject(error);
+                });
+        });
+    };
+
     module.exports = {
         CloudStorage,
     };

cvat-core/src/enums.js

@@ -339,12 +339,12 @@
      * @name CloudStorageProviderType
      * @memberof module:API.cvat.enums
      * @property {string} AWS_S3 'AWS_S3_BUCKET'
-     * @property {string} AZURE 'AZURE_BLOB_CONTAINER'
+     * @property {string} AZURE 'AZURE_CONTAINER'
      * @readonly
      */
     const CloudStorageProviderType = Object.freeze({
         AWS_S3_BUCKET: 'AWS_S3_BUCKET',
-        AZURE_BLOB_CONTAINER: 'AZURE_BLOB_CONTAINER',
+        AZURE_CONTAINER: 'AZURE_CONTAINER',
     });
 
     /**

cvat-core/src/server-proxy.js

@@ -1175,6 +1175,9 @@
                 try {
                     const response = await Axios.post(`${backendAPI}/cloudstorages`, JSON.stringify(storageDetail), {
                         proxy: config.proxy,
+                        headers: {
+                            'Content-Type': 'application/json',
+                        },
                     });
                     return response.data;
                 } catch (errorData) {
@@ -1202,14 +1205,15 @@
 
                 let response = null;
                 try {
-                    response = await Axios.get(`${backendAPI}/cloudstorages?page_size=20&${filter}`, {
+                    response = await Axios.get(`${backendAPI}/cloudstorages?page_size=12&${filter}`, {
                         proxy: config.proxy,
                     });
                 } catch (errorData) {
                     throw generateError(errorData);
                 }
 
-                return response.data;
+                response.data.results.count = response.data.count;
+                return response.data.results;
             }
 
             async function getCloudStorageContent(id, manifestPath) {
@@ -1230,6 +1234,24 @@
                 return response.data;
             }
 
+            async function getCloudStoragePreview(id) {
+                const { backendAPI } = config;
+
+                let response = null;
+                try {
+                    const url = `${backendAPI}/cloudstorages/${id}/preview`;
+                    response = await Axios.get(url, {
+                        proxy: config.proxy,
+                        responseType: 'blob',
+                    });
+                } catch (errorData) {
+                    const code = errorData.response ? errorData.response.status : errorData.code;
+                    throw new ServerError(`Could not get preview for the cloud storage ${id} from the server`, code);
+                }
+
+                return response.data;
+            }
+
             async function deleteCloudStorage(id) {
                 const { backendAPI } = config;
 
@@ -1373,6 +1395,7 @@
                         value: Object.freeze({
                             get: getCloudStorages,
                             getContent: getCloudStorageContent,
+                            getPreview: getCloudStoragePreview,
                             create: createCloudStorage,
                             delete: deleteCloudStorage,
                             update: updateCloudStorage,

cvat-core/src/session.js

@@ -1036,6 +1036,7 @@
                 use_cache: undefined,
                 copy_data: undefined,
                 dimension: undefined,
+                cloud_storage_id: undefined,
             };
 
             const updatedFields = new FieldUpdateTrigger({
@@ -1397,7 +1398,7 @@
                         get: () => [...data.jobs],
                     },
                     /**
-                     * List of files from shared resource
+                     * List of files from shared resource or list of cloud storage files
                      * @name serverFiles
                      * @type {string[]}
                      * @memberof module:API.cvat.classes.Task
@@ -1559,6 +1560,21 @@
                          */
                         get: () => data.dimension,
                     },
+                    /**
+                     * @name cloudStorageId
+                     * @type {integer|null}
+                     * @memberof module:API.cvat.classes.Task
+                     * @instance
+                     */
+                    cloudStorageId: {
+                        get: () => data.cloud_storage_id,
+                        set: (cloudStorageId) => {
+                            if (!Number.isInteger(cloudStorageId) || cloudStorageId <= 0) {
+                                throw new ArgumentError('Value must be a positive integer');
+                            }
+                            data.cloud_storage_id = cloudStorageId;
+                        },
+                    },
                     _internalData: {
                         get: () => data,
                     },
@@ -2093,6 +2109,9 @@
         if (typeof this.copyData !== 'undefined') {
             taskDataSpec.copy_data = this.copyData;
         }
+        if (typeof this.cloudStorageId !== 'undefined') {
+            taskDataSpec.cloud_storage_id = this.cloudStorageId;
+        }
 
         const task = await serverProxy.tasks.createTask(taskSpec, taskDataSpec, onUpdate);
         return new Task(task);