Merge branch 'develop' into transt-conda

.github/codecov.yml

@@ -19,3 +19,9 @@ component_management:
  - cvat-sdk/**
  - utils/**
 
+codecov:
+ require_ci_to_pass: yes
+ notify:
+ wait_for_ci: yes
+ after_n_builds: 16
+

.github/workflows/full.yml

@@ -175,6 +175,8 @@ jobs:
 
  - name: Upload coverage reports to Codecov with GitHub Action
  uses: codecov/codecov-action@v3
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
 
  - name: Creating a log file from cvat containers
  if: failure() && steps.run_tests.conclusion == 'failure'
@@ -241,6 +243,8 @@ jobs:
 
  - name: Upload coverage reports to Codecov with GitHub Action
  uses: codecov/codecov-action@v3
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
 
  - name: Creating a log file from cvat containers
  if: failure()
@@ -346,6 +350,8 @@ jobs:
 
  - name: Upload coverage reports to Codecov with GitHub Action
  uses: codecov/codecov-action@v3
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
 
  - name: Creating a log file from "cvat" container logs
  if: failure()

.github/workflows/main.yml

@@ -168,6 +168,8 @@ jobs:
 
  - name: Upload coverage reports to Codecov with GitHub Action
  uses: codecov/codecov-action@v3
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
 
  - name: Creating a log file from cvat containers
  if: failure() && steps.run_tests.conclusion == 'failure'
@@ -232,6 +234,8 @@ jobs:
 
  - name: Upload coverage reports to Codecov with GitHub Action
  uses: codecov/codecov-action@v3
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
 
  - name: Creating a log file from cvat containers
  if: failure()
@@ -335,6 +339,8 @@ jobs:
 
  - name: Upload coverage reports to Codecov with GitHub Action
  uses: codecov/codecov-action@v3
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
 
  - name: Creating a log file from "cvat" container logs
  if: failure()

.github/workflows/schedule.yml

@@ -197,6 +197,8 @@ jobs:
 
  - name: Upload coverage reports to Codecov with GitHub Action
  uses: codecov/codecov-action@v3
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
 
  e2e_testing:
  needs: build
@@ -312,6 +314,8 @@ jobs:
 
  - name: Upload coverage reports to Codecov with GitHub Action
  uses: codecov/codecov-action@v3
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
 
  - name: Creating a log file from "cvat" container logs
  if: failure()

cvat/apps/engine/serializers.py

@@ -16,7 +16,6 @@
 from rest_framework import serializers, exceptions
 from django.contrib.auth.models import User, Group
 from django.db import transaction
-from django.conf import settings
 
 from cvat.apps.dataset_manager.formats.utils import get_label_color
 from cvat.apps.engine import models
@@ -951,15 +950,9 @@ def validate(self, attrs):
  and attrs['start_frame'] > attrs['stop_frame']:
  raise serializers.ValidationError('Stop frame must be more or equal start frame')
 
- if (
- (server_files := attrs.get('server_files'))
- and attrs.get('cloud_storage_id')
- and sum(1 for f in server_files if not f['file'].endswith('.jsonl')) > settings.CLOUD_STORAGE_MAX_FILES_COUNT
- ):
- raise serializers.ValidationError(f'The maximum number of the cloud storage attached files is {settings.CLOUD_STORAGE_MAX_FILES_COUNT}')
-
  filename_pattern = attrs.get('filename_pattern')
  server_files_exclude = attrs.get('server_files_exclude')
+ server_files = attrs.get('server_files', [])
 
  if filename_pattern and len(list(filter(lambda x: not x['file'].endswith('.jsonl'), server_files))):
  raise serializers.ValidationError('The filename_pattern can only be used with specified manifest or without server_files')

cvat/apps/engine/task.py

@@ -580,24 +580,15 @@ def _create_thread(
  )
  if cloud_storage_manifest_prefix:
  additional_files = [os.path.join(cloud_storage_manifest_prefix, f) for f in additional_files]
- if len(data['server_files']) + len(additional_files) > settings.CLOUD_STORAGE_MAX_FILES_COUNT:
- raise ValidationError(
- 'The maximum number of the cloud storage attached files '
- f'is {settings.CLOUD_STORAGE_MAX_FILES_COUNT}')
  else:
- number_of_files = len(data['server_files'])
  while len(dirs):
  directory = dirs.pop()
  for f in cloud_storage_instance.list_files(prefix=directory, _use_flat_listing=True):
  if f['type'] == 'REG':
  additional_files.append(f['name'])
  else:
  dirs.append(f['name'])
- # we check the limit of files on each iteration to reduce the number of possible requests to the bucket
- if (len(additional_files) + len(dirs) + number_of_files) > settings.CLOUD_STORAGE_MAX_FILES_COUNT:
- raise ValidationError(
- 'The maximum number of the cloud storage attached files '
- f'is {settings.CLOUD_STORAGE_MAX_FILES_COUNT}')
+
  data['server_files'].extend(additional_files)
  del additional_files
 
@@ -637,10 +628,6 @@ def _create_thread(
  if not data['filename_pattern'] == '*':
  additional_files = fnmatch.filter(additional_files, data['filename_pattern'])
 
- if (len(additional_files)) > settings.CLOUD_STORAGE_MAX_FILES_COUNT:
- raise ValidationError(
- 'The maximum number of the cloud storage attached files '
- f'is {settings.CLOUD_STORAGE_MAX_FILES_COUNT}')
  data['server_files'].extend(additional_files)
 
  if db_data.storage_method == models.StorageMethodChoice.FILE_SYSTEM or not settings.USE_CACHE:

cvat/settings/base.py

@@ -522,9 +522,6 @@ class CVAT_QUEUES(Enum):
 DATA_UPLOAD_MAX_MEMORY_SIZE = 100 * 1024 * 1024 # 100 MB
 DATA_UPLOAD_MAX_NUMBER_FIELDS = None # this django check disabled
 DATA_UPLOAD_MAX_NUMBER_FILES = None
-LOCAL_LOAD_MAX_FILES_COUNT = 500
-LOCAL_LOAD_MAX_FILES_SIZE = 512 * 1024 * 1024 # 512 MB
-CLOUD_STORAGE_MAX_FILES_COUNT = LOCAL_LOAD_MAX_FILES_COUNT
 
 RESTRICTIONS = {
  # allow access to analytics component to users with business role

site/content/en/docs/administration/advanced/iam_system_roles.md

@@ -7,4 +7,17 @@ weight: 70
 <!--lint disable heading-style-->
 
 ## System roles
+
+By default CVAT users can be assigned to one of the following groups: `admin`, `business`, `user` and `worker`.
+
+Each of these groups gives a set of permissions.
 TBD
+
+## Changing permissions
+
+System permissions are defined using `.rego` files stored in `cvat/apps/iam/rules/`.
+Rego is a declarative language used for defining OPA policies.
+It's syntax is defined in [OPA docs](https://www.openpolicyagent.org/docs/latest/policy-language/).
+
+After changing the `.rego` files, you need to rebuilt and restart the docker compose for the changes to take effect.
+In this case you need to include `docker-compose.dev.yml` compose config file to `docker compose` command.

tests/cypress/e2e/actions_tasks3/case_115_use_custom_storage_for_import_export_annotations.js

@@ -126,6 +126,21 @@ context('Import and export annotations: specify source and target storage in mod
  };
  cy.exportJob(exportParams);
  cy.waitForFileUploadToCloudStorage();
+ });
+
+ it('Export job annotations to custom minio bucket with folder path', () => {
+ const exportParams = {
+ type: 'annotations',
+ format,
+ archiveCustomName: 'some/folder/job_annotations',
+ targetStorage: {
+ location: 'Cloud storage',
+ cloudStorageId: createdCloudStorageId,
+ },
+ useDefaultLocation: false,
+ };
+ cy.exportJob(exportParams);
+ cy.waitForFileUploadToCloudStorage();
 
  // remove annotations
  cy.removeAnnotations();