Extension of cloud storage server part

CHANGELOG.md

@@ -15,19 +15,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
  and project with 3D tasks (<https://github.com/openvinotoolkit/cvat/pull/3502>)
 - Additional inline tips in interactors with demo gifs (<https://github.com/openvinotoolkit/cvat/pull/3473>)
 - Added intelligent scissors blocking feature (<https://github.com/openvinotoolkit/cvat/pull/3510>)
+- Support cloud storage status (<https://github.com/openvinotoolkit/cvat/pull/3386>)
+- Support cloud storage preview (<https://github.com/openvinotoolkit/cvat/pull/3386>)
 
 ### Changed
 
 - Non-blocking UI when using interactors (<https://github.com/openvinotoolkit/cvat/pull/3473>)
 - "Selected opacity" slider now defines opacity level for shapes being drawnSelected opacity (<https://github.com/openvinotoolkit/cvat/pull/3473>)
+- Cloud storage creating and updating (<https://github.com/openvinotoolkit/cvat/pull/3386>)
+- Way of working with cloud storage content (<https://github.com/openvinotoolkit/cvat/pull/3386>)
 
 ### Deprecated
 
 - TDB
 
 ### Removed
 
-- TDB
+- Support TEMP_KEY_SECRET_KEY_TOKEN_SET for AWS S3 cloud storage (<https://github.com/openvinotoolkit/cvat/pull/3386>)
 
 ### Fixed
 

cvat/apps/engine/cache.py

@@ -15,7 +15,7 @@
  ImageDatasetManifestReader, VideoDatasetManifestReader)
 from cvat.apps.engine.models import DataChoice, StorageChoice
 from cvat.apps.engine.models import DimensionType
-from cvat.apps.engine.cloud_provider import get_cloud_storage_instance, Credentials
+from cvat.apps.engine.cloud_provider import get_cloud_storage_instance, Credentials, Status
 from cvat.apps.engine.utils import md5_hash
 class CacheInteraction:
  def __init__(self, dimension=DimensionType.DIM_2D):
@@ -71,6 +71,7 @@ def prepare_chunk_buff(self, db_data, quality, chunk_number):
  step=db_data.get_frame_step())
  if db_data.storage == StorageChoice.CLOUD_STORAGE:
  db_cloud_storage = db_data.cloud_storage
+ assert db_cloud_storage, 'Cloud storage instance was deleted'
  credentials = Credentials()
  credentials.convert_from_db({
  'type': db_cloud_storage.credentials_type,
@@ -81,22 +82,38 @@ def prepare_chunk_buff(self, db_data, quality, chunk_number):
  'credentials': credentials,
  'specific_attributes': db_cloud_storage.get_specific_attributes()
  }
- cloud_storage_instance = get_cloud_storage_instance(cloud_provider=db_cloud_storage.provider_type, **details)
- cloud_storage_instance.initialize_content()
- for item in reader:
- name = f"{item['name']}{item['extension']}"
- if name not in cloud_storage_instance:
- raise Exception('{} file was not found on a {} storage'.format(name, cloud_storage_instance.name))
- with NamedTemporaryFile(mode='w+b', prefix='cvat', suffix=name.replace(os.path.sep, '#'), delete=False) as temp_file:
- source_path = temp_file.name
- buf = cloud_storage_instance.download_fileobj(name)
- temp_file.write(buf.getvalue())
- checksum = item.get('checksum', None)
- if not checksum:
- slogger.glob.warning('A manifest file does not contain checksum for image {}'.format(item.get('name')))
- if checksum and not md5_hash(source_path) == checksum:
- slogger.glob.warning('Hash sums of files {} do not match'.format(name))
- images.append((source_path, source_path, None))
+ try:
+ cloud_storage_instance = get_cloud_storage_instance(cloud_provider=db_cloud_storage.provider_type, **details)
+ cloud_storage_instance.initialize_content()
+ for item in reader:
+ file_name = f"{item['name']}{item['extension']}"
+ if file_name not in cloud_storage_instance:
+ raise Exception('{} file was not found on a {} storage'.format(file_name, cloud_storage_instance.name))
+ with NamedTemporaryFile(mode='w+b', prefix='cvat', suffix=file_name.replace(os.path.sep, '#'), delete=False) as temp_file:
+ source_path = temp_file.name
+ buf = cloud_storage_instance.download_fileobj(file_name)
+ temp_file.write(buf.getvalue())
+ checksum = item.get('checksum', None)
+ if not checksum:
+ slogger.cloud_storage[db_cloud_storage.id].warning('A manifest file does not contain checksum for image {}'.format(item.get('name')))
+ if checksum and not md5_hash(source_path) == checksum:
+ slogger.cloud_storage[db_cloud_storage.id].warning('Hash sums of files {} do not match'.format(file_name))
+ images.append((source_path, source_path, None))
+ except Exception as ex:
+ storage_status = cloud_storage_instance.get_status()
+ if storage_status == Status.FORBIDDEN:
+ msg = 'The resource {} is no longer available. Access forbidden.'.format(cloud_storage_instance.name)
+ elif storage_status == Status.NOT_FOUND:
+ msg = 'The resource {} not found. It may have been deleted.'.format(cloud_storage_instance.name)
+ else:
+ # check status of last file
+ file_status = cloud_storage_instance.get_file_status(file_name)
+ if file_status == Status.NOT_FOUND:
+ raise Exception("'{}' not found on the cloud storage '{}'".format(file_name, cloud_storage_instance.name))
+ elif file_status == Status.FORBIDDEN:
+ raise Exception("Access to the file '{}' on the '{}' cloud storage is denied".format(file_name, cloud_storage_instance.name))
+ msg = str(ex)
+ raise Exception(msg)
  else:
  for item in reader:
  source_path = os.path.join(upload_dir, f"{item['name']}{item['extension']}")