From e8a31b87e83c055a8c959ed35fc79d4142fb9437 Mon Sep 17 00:00:00 2001 From: Kirill Lakhov Date: Tue, 13 Jun 2023 19:40:09 +0300 Subject: [PATCH 1/4] Test for exporting annotations into CS folder (#6301) ### Motivation and context Test for #6265 ### How has this been tested? ### Checklist - [ ] I submit my changes into the `develop` branch - [ ] I have added a description of my changes into the [CHANGELOG](https://github.com/opencv/cvat/blob/develop/CHANGELOG.md) file - [ ] I have updated the documentation accordingly - [ ] I have added tests to cover my changes - [ ] I have linked related issues (see [GitHub docs]( https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword)) - [ ] I have increased versions of npm packages if it is necessary ([cvat-canvas](https://github.com/opencv/cvat/tree/develop/cvat-canvas#versioning), [cvat-core](https://github.com/opencv/cvat/tree/develop/cvat-core#versioning), [cvat-data](https://github.com/opencv/cvat/tree/develop/cvat-data#versioning) and [cvat-ui](https://github.com/opencv/cvat/tree/develop/cvat-ui#versioning)) ### License - [ ] I submit _my code changes_ under the same [MIT License]( https://github.com/opencv/cvat/blob/develop/LICENSE) that covers the project. Feel free to contact the maintainers if that's a concern. --- ...ustom_storage_for_import_export_annotations.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/cypress/e2e/actions_tasks3/case_115_use_custom_storage_for_import_export_annotations.js b/tests/cypress/e2e/actions_tasks3/case_115_use_custom_storage_for_import_export_annotations.js index e0e8b14a596..353e4f27526 100644 --- a/tests/cypress/e2e/actions_tasks3/case_115_use_custom_storage_for_import_export_annotations.js +++ b/tests/cypress/e2e/actions_tasks3/case_115_use_custom_storage_for_import_export_annotations.js @@ -126,6 +126,21 @@ context('Import and export annotations: specify source and target storage in mod }; cy.exportJob(exportParams); cy.waitForFileUploadToCloudStorage(); + }); + + it('Export job annotations to custom minio bucket with folder path', () => { + const exportParams = { + type: 'annotations', + format, + archiveCustomName: 'some/folder/job_annotations', + targetStorage: { + location: 'Cloud storage', + cloudStorageId: createdCloudStorageId, + }, + useDefaultLocation: false, + }; + cy.exportJob(exportParams); + cy.waitForFileUploadToCloudStorage(); // remove annotations cy.removeAnnotations(); From 44bf2da799f11ad13bf5a7050dfa12f966f3126e Mon Sep 17 00:00:00 2001 From: Anastasia Yasakova Date: Tue, 13 Jun 2023 21:06:31 +0300 Subject: [PATCH 2/4] Codecov report after completion of full CI (#6293) - Added upload to Codecov with token - Codecov report after completion of full CI --- .github/codecov.yml | 6 ++++++ .github/workflows/full.yml | 6 ++++++ .github/workflows/main.yml | 6 ++++++ .github/workflows/schedule.yml | 4 ++++ 4 files changed, 22 insertions(+) diff --git a/.github/codecov.yml b/.github/codecov.yml index 31402461a07..d259debd2cb 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -19,3 +19,9 @@ component_management: - cvat-sdk/** - utils/** +codecov: + require_ci_to_pass: yes + notify: + wait_for_ci: yes + after_n_builds: 16 + diff --git a/.github/workflows/full.yml b/.github/workflows/full.yml index 5faf726e367..598245b9450 100644 --- a/.github/workflows/full.yml +++ b/.github/workflows/full.yml @@ -175,6 +175,8 @@ jobs: - name: Upload coverage reports to Codecov with GitHub Action uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} - name: Creating a log file from cvat containers if: failure() && steps.run_tests.conclusion == 'failure' @@ -241,6 +243,8 @@ jobs: - name: Upload coverage reports to Codecov with GitHub Action uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} - name: Creating a log file from cvat containers if: failure() @@ -346,6 +350,8 @@ jobs: - name: Upload coverage reports to Codecov with GitHub Action uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} - name: Creating a log file from "cvat" container logs if: failure() diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 63a94f25440..928ed0a3b61 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -168,6 +168,8 @@ jobs: - name: Upload coverage reports to Codecov with GitHub Action uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} - name: Creating a log file from cvat containers if: failure() && steps.run_tests.conclusion == 'failure' @@ -232,6 +234,8 @@ jobs: - name: Upload coverage reports to Codecov with GitHub Action uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} - name: Creating a log file from cvat containers if: failure() @@ -335,6 +339,8 @@ jobs: - name: Upload coverage reports to Codecov with GitHub Action uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} - name: Creating a log file from "cvat" container logs if: failure() diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml index ab7297b5601..80808b29159 100644 --- a/.github/workflows/schedule.yml +++ b/.github/workflows/schedule.yml @@ -197,6 +197,8 @@ jobs: - name: Upload coverage reports to Codecov with GitHub Action uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} e2e_testing: needs: build @@ -312,6 +314,8 @@ jobs: - name: Upload coverage reports to Codecov with GitHub Action uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} - name: Creating a log file from "cvat" container logs if: failure() From e95c0a101517c95b4bad6054f0814127fef11457 Mon Sep 17 00:00:00 2001 From: Kirill Sizov Date: Tue, 13 Jun 2023 21:08:26 +0300 Subject: [PATCH 3/4] Remove max file count (#6292) --- cvat/apps/engine/serializers.py | 9 +-------- cvat/apps/engine/task.py | 15 +-------------- cvat/settings/base.py | 3 --- 3 files changed, 2 insertions(+), 25 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 90d5ea8a413..e0fbb650769 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -16,7 +16,6 @@ from rest_framework import serializers, exceptions from django.contrib.auth.models import User, Group from django.db import transaction -from django.conf import settings from cvat.apps.dataset_manager.formats.utils import get_label_color from cvat.apps.engine import models @@ -951,15 +950,9 @@ def validate(self, attrs): and attrs['start_frame'] > attrs['stop_frame']: raise serializers.ValidationError('Stop frame must be more or equal start frame') - if ( - (server_files := attrs.get('server_files')) - and attrs.get('cloud_storage_id') - and sum(1 for f in server_files if not f['file'].endswith('.jsonl')) > settings.CLOUD_STORAGE_MAX_FILES_COUNT - ): - raise serializers.ValidationError(f'The maximum number of the cloud storage attached files is {settings.CLOUD_STORAGE_MAX_FILES_COUNT}') - filename_pattern = attrs.get('filename_pattern') server_files_exclude = attrs.get('server_files_exclude') + server_files = attrs.get('server_files', []) if filename_pattern and len(list(filter(lambda x: not x['file'].endswith('.jsonl'), server_files))): raise serializers.ValidationError('The filename_pattern can only be used with specified manifest or without server_files') diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 46c767f0c28..9d863002359 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -580,12 +580,7 @@ def _create_thread( ) if cloud_storage_manifest_prefix: additional_files = [os.path.join(cloud_storage_manifest_prefix, f) for f in additional_files] - if len(data['server_files']) + len(additional_files) > settings.CLOUD_STORAGE_MAX_FILES_COUNT: - raise ValidationError( - 'The maximum number of the cloud storage attached files ' - f'is {settings.CLOUD_STORAGE_MAX_FILES_COUNT}') else: - number_of_files = len(data['server_files']) while len(dirs): directory = dirs.pop() for f in cloud_storage_instance.list_files(prefix=directory, _use_flat_listing=True): @@ -593,11 +588,7 @@ def _create_thread( additional_files.append(f['name']) else: dirs.append(f['name']) - # we check the limit of files on each iteration to reduce the number of possible requests to the bucket - if (len(additional_files) + len(dirs) + number_of_files) > settings.CLOUD_STORAGE_MAX_FILES_COUNT: - raise ValidationError( - 'The maximum number of the cloud storage attached files ' - f'is {settings.CLOUD_STORAGE_MAX_FILES_COUNT}') + data['server_files'].extend(additional_files) del additional_files @@ -637,10 +628,6 @@ def _create_thread( if not data['filename_pattern'] == '*': additional_files = fnmatch.filter(additional_files, data['filename_pattern']) - if (len(additional_files)) > settings.CLOUD_STORAGE_MAX_FILES_COUNT: - raise ValidationError( - 'The maximum number of the cloud storage attached files ' - f'is {settings.CLOUD_STORAGE_MAX_FILES_COUNT}') data['server_files'].extend(additional_files) if db_data.storage_method == models.StorageMethodChoice.FILE_SYSTEM or not settings.USE_CACHE: diff --git a/cvat/settings/base.py b/cvat/settings/base.py index e900e221116..25f6bfebe60 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -522,9 +522,6 @@ class CVAT_QUEUES(Enum): DATA_UPLOAD_MAX_MEMORY_SIZE = 100 * 1024 * 1024 # 100 MB DATA_UPLOAD_MAX_NUMBER_FIELDS = None # this django check disabled DATA_UPLOAD_MAX_NUMBER_FILES = None -LOCAL_LOAD_MAX_FILES_COUNT = 500 -LOCAL_LOAD_MAX_FILES_SIZE = 512 * 1024 * 1024 # 512 MB -CLOUD_STORAGE_MAX_FILES_COUNT = LOCAL_LOAD_MAX_FILES_COUNT RESTRICTIONS = { # allow access to analytics component to users with business role From f578e4785d1be3128863721f5b149b28cd85ee9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Kotiuk?= Date: Tue, 13 Jun 2023 20:11:17 +0200 Subject: [PATCH 4/4] docs: Describe changing default OPA permissions (#6294) Follow up to discussion from: https://github.com/opencv/cvat/issues/6224 This issue is linked with: https://github.com/opencv/cvat/pull/3788 which introduced IAM, but not document some aspects. . --- .../administration/advanced/iam_system_roles.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/site/content/en/docs/administration/advanced/iam_system_roles.md b/site/content/en/docs/administration/advanced/iam_system_roles.md index 17cd5d79a5f..71ce8eb274b 100644 --- a/site/content/en/docs/administration/advanced/iam_system_roles.md +++ b/site/content/en/docs/administration/advanced/iam_system_roles.md @@ -7,4 +7,17 @@ weight: 70 ## System roles + +By default CVAT users can be assigned to one of the following groups: `admin`, `business`, `user` and `worker`. + +Each of these groups gives a set of permissions. TBD + +## Changing permissions + +System permissions are defined using `.rego` files stored in `cvat/apps/iam/rules/`. +Rego is a declarative language used for defining OPA policies. +It's syntax is defined in [OPA docs](https://www.openpolicyagent.org/docs/latest/policy-language/). + +After changing the `.rego` files, you need to rebuilt and restart the docker compose for the changes to take effect. +In this case you need to include `docker-compose.dev.yml` compose config file to `docker compose` command.