Improve task creation with cloud storage and share data (cvat-ai#6074)

### Motivation and context Resolved cvat-ai#6037 Related cvat-ai#4400 Related cvat-ai#6028 ![image](https://user-images.githubusercontent.com/49038720/236890662-c44b578e-5808-4fde-a216-2dcab6e95ab0.png) Co-authored-by: Boris Sekachev <boris.sekachev@yandex.ru> Co-authored-by: Boris Sekachev <sekachev.bs@gmail.com> Co-authored-by: Roman Donchenko <roman@cvat.ai> Co-authored-by: Nikita Manovich <nikita@cvat.ai>
retailnext · Jul 1, 2023 · 423d03f · 423d03f
1 parent 116d7e7
commit 423d03f
Show file tree

Hide file tree

Showing 41 changed files with 1,932 additions and 1,078 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,13 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## \[2.5.0] - Unreleased
 ### Added
-- TDB
+- Support share point/cloud storage files and directories that should be excluded when creating a task (server) (<https://github.com/opencv/cvat/pull/6074>)
+- Support task creation with cloud storage/share directories (<https://github.com/opencv/cvat/pull/6074>)
+- Support task creation with any type of data supported by the server by default from cloud storage
+without use_cache option (<https://github.com/opencv/cvat/pull/6074>)
+- Support task creation with cloud storage data and without use_cache option (<https://github.com/opencv/cvat/pull/6074>)
+
+### Changed
+- Cloud storage manifest file is optional (<https://github.com/opencv/cvat/pull/6074>)
 
 ### Changed
 - TDB
 
 ### Deprecated
-- TDB
+- The endpoint /cloudstorages/{id}/content was deprecated (<https://github.com/opencv/cvat/pull/6074>)
 
 ### Removed
 - TDB

diff --git a/cvat-core/src/api-implementation.ts b/cvat-core/src/api-implementation.ts
@@ -3,6 +3,7 @@
 //
 // SPDX-License-Identifier: MIT
 
+import { omit } from 'lodash';
 import config from './config';
 
 import PluginRegistry from './plugins';
@@ -44,7 +45,7 @@ export default function implementAPI(cvat) {
 
  cvat.server.share.implementation = async (directory) => {
  const result = await serverProxy.server.share(directory);
- return result;
+ return result.map((item) => ({ ...omit(item, 'mime_type'), mimeType: item.mime_type }));
  };
 
  cvat.server.formats.implementation = async () => {

diff --git a/cvat-core/src/cloud-storage.ts b/cvat-core/src/cloud-storage.ts
@@ -3,12 +3,14 @@
 //
 // SPDX-License-Identifier: MIT
 
+import { omit } from 'lodash';
 import PluginRegistry from './plugins';
 import serverProxy from './server-proxy';
 import { ArgumentError } from './exceptions';
 import { CloudStorageCredentialsType, CloudStorageProviderType, CloudStorageStatus } from './enums';
 import User from './user';
 import { decodePreview } from './frames';
+import { SerializedRemoteFile } from './server-response-types';
 
 function validateNotEmptyString(value: string): void {
  if (typeof value !== 'string') {
@@ -164,7 +166,6 @@ export default class CloudStorage {
  manifestPath: {
  get: () => data.manifest_path,
  set: (value) => {
- validateNotEmptyString(value);
  data.manifest_path = value;
  },
  },
@@ -245,8 +246,11 @@ export default class CloudStorage {
  return result;
  }
 
- public async getContent(): Promise<any> {
- const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.getContent);
+ public async getContent(path: string, nextToken?: string): Promise<{
+ next: string | null,
+ content: (Omit<SerializedRemoteFile, 'mime_type'> & { mimeType: string })[],
+ }> {
+ const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.getContent, path, nextToken);
  return result;
  }
 
@@ -361,9 +365,12 @@ Object.defineProperties(CloudStorage.prototype.getContent, {
  implementation: {
  writable: false,
  enumerable: false,
- value: async function implementation(): Promise<any> {
- const result = await serverProxy.cloudStorages.getContent(this.id, this.manifestPath);
- return result;
+ value: async function implementation(path: string, nextToken?: string): ReturnType<CloudStorage['getContent']> {
+ const result = await serverProxy.cloudStorages.getContent(this.id, path, nextToken, this.manifestPath);
+ return {
+ next: result.next,
+ content: result.content.map((item) => ({ ...omit(item, 'mime_type'), mimeType: item.mime_type })),
+ };
  },
  },
 });
@@ -376,7 +383,7 @@ Object.defineProperties(CloudStorage.prototype.getPreview, {
  return new Promise((resolve, reject) => {
  serverProxy.cloudStorages
  .getPreview(this.id)
- .then((result) => decodePreview(result))
+ .then((result) => ((result) ? decodePreview(result) : Promise.resolve(result)))
  .then((decoded) => resolve(decoded))
  .catch((error) => {
  reject(error);

diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts
@@ -10,7 +10,7 @@ import * as tus from 'tus-js-client';
 import {
  SerializedLabel, SerializedAnnotationFormats, ProjectsFilter,
  SerializedProject, SerializedTask, TasksFilter, SerializedUser,
- SerializedAbout, SerializedShare, SerializedUserAgreement,
+ SerializedAbout, SerializedRemoteFile, SerializedUserAgreement,
  SerializedRegister, JobsFilter, SerializedJob,
 } from 'server-response-types';
 import { Storage } from './storage';
@@ -309,7 +309,7 @@ async function about(): Promise<SerializedAbout> {
  return response.data;
 }
 
-async function share(directoryArg: string): Promise<SerializedShare[]> {
+async function share(directoryArg: string): Promise<SerializedRemoteFile[]> {
  const { backendAPI } = config;
 
  let response = null;
@@ -1368,7 +1368,7 @@ async function getUsers(filter = { page_size: 'all' }) {
  return response.data.results;
 }
 
-function getPreview(instance) {
+function getPreview(instance: 'projects' | 'tasks' | 'jobs' | 'cloudstorages') {
  return async function (id: number) {
  const { backendAPI } = config;
 
@@ -1383,7 +1383,7 @@ function getPreview(instance) {
  throw new ServerError(`Could not get preview for "${instance}/${id}"`, code);
  }
 
- return response.data;
+ return (response.status === 200) ? response.data : '';
  };
 }
 
@@ -1860,14 +1860,17 @@ async function getCloudStorages(filter = {}) {
  return response.data.results;
 }
 
-async function getCloudStorageContent(id, manifestPath) {
+async function getCloudStorageContent(id: number, path: string, nextToken?: string, manifestPath?: string):
+Promise<{ content: SerializedRemoteFile[], next: string | null }> {
  const { backendAPI } = config;
 
  let response = null;
  try {
- const url = `${backendAPI}/cloudstorages/${id}/content`;
+ const url = `${backendAPI}/cloudstorages/${id}/content-v2`;
  response = await Axios.get(url, {
  params: {
+ prefix: path,
+ ...(nextToken ? { next_token: nextToken } : {}),
  ...(manifestPath ? { manifest_path: manifestPath } : {}),
  },
  });

diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts
@@ -152,7 +152,7 @@ export interface SerializedAbout {
  version: string;
 }
 
-export interface SerializedShare {
+export interface SerializedRemoteFile {
  name: string;
  type: ShareFileType;
  mime_type: string;

diff --git a/cvat-sdk/cvat_sdk/core/proxies/tasks.py b/cvat-sdk/cvat_sdk/core/proxies/tasks.py
@@ -96,6 +96,7 @@ def upload_data(
  "job_file_mapping",
  "filename_pattern",
  "cloud_storage_id",
+ "server_files_exclude",
  ],
  )
  )

diff --git a/cvat-ui/src/actions/share-actions.ts b/cvat-ui/src/actions/share-actions.ts
diff --git a/cvat-ui/src/components/cloud-storages-page/cloud-storages-filter-configuration.ts b/cvat-ui/src/components/cloud-storages-page/cloud-storages-filter-configuration.ts
@@ -1,4 +1,5 @@
 // Copyright (C) 2022 Intel Corporation
+// Copyright (C) 2023 CVAT.ai Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -46,8 +47,8 @@ export const config: Partial<Config> = {
  valueSources: ['value'],
  operators: ['like'],
  },
- display_name: {
- label: 'Display name',
+ name: {
+ label: 'Name',
  type: 'text',
  valueSources: ['value'],
  operators: ['like'],

diff --git a/cvat-ui/src/components/create-cloud-storage-page/manifests-manager.tsx b/cvat-ui/src/components/create-cloud-storage-page/manifests-manager.tsx
@@ -7,12 +7,13 @@ import React, { useEffect } from 'react';
 import { DeleteOutlined, PlusCircleOutlined, QuestionCircleOutlined } from '@ant-design/icons';
 import Button from 'antd/lib/button';
 import Col from 'antd/lib/col';
-import Form, { RuleObject } from 'antd/lib/form';
+import Form from 'antd/lib/form';
 import { FormListFieldData, FormListOperation } from 'antd/lib/form/FormList';
 import Input from 'antd/lib/input';
 import Row from 'antd/lib/row';
 import Tooltip from 'antd/lib/tooltip';
 import config from 'config';
+import { Alert } from 'antd';
 
 interface Props {
  form: any;
@@ -71,19 +72,9 @@ export default function ManifestsManager(props: Props): JSX.Element {
  </Tooltip>
  </>
  )}
- required
  />
  <Form.List
  name='manifests'
- rules={[
- {
- validator: async (_: RuleObject, names: string[]): Promise<void> => {
- if (!names || !names.length) {
- throw new Error('Please, specify at least one manifest file');
- }
- },
- },
- ]}
  >
  {
  (fields: FormListFieldData[], _: FormListOperation, { errors }: { errors: React.ReactNode[] }) => (
@@ -135,6 +126,13 @@ export default function ManifestsManager(props: Props): JSX.Element {
  </Button>
  </Col>
  </Row>
+ {!manifestNames.length && (
+ <Row>
+ <Col>
+ <Alert type='info' message='We highly recommend attaching a manifest file to reduce the number of requests to the bucket.' />
+ </Col>
+ </Row>
+ )}
  </>
  );
 }