From bc7f03c4f719e50c98d3772127f0e4a272a95ee6 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Tue, 14 Feb 2023 08:39:01 +0200 Subject: [PATCH 01/30] Fix deletion of sublabels --- cvat/apps/engine/serializers.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 2a3e0ae8108..a6e68414972 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -687,12 +687,13 @@ def update_labels(labels, parent_label=None): logger.info(f'label:update Label id:{db_label.id} for spec:{label} with sublabels:{sublabels}, parent_label:{parent_label}') else: logger.info(f'label:delete label:{label} with sublabels:{sublabels}, parent_label:{parent_label}') - update_labels(sublabels, parent_label=db_label) - if label.get('id') is None and db_label.type == str(models.LabelType.SKELETON): - for db_sublabel in list(db_label.sublabels.all()): - svg = svg.replace(f'data-label-name="{db_sublabel.name}"', f'data-label-id="{db_sublabel.id}"') - db_skeleton = models.Skeleton.objects.create(root=db_label, svg=svg) - logger.info(f'label:update Skeleton id:{db_skeleton.id} for label_id:{db_label.id}') + if not label.get('deleted'): + update_labels(sublabels, parent_label=db_label) + if label.get('id') is None and db_label.type == str(models.LabelType.SKELETON): + for db_sublabel in list(db_label.sublabels.all()): + svg = svg.replace(f'data-label-name="{db_sublabel.name}"', f'data-label-id="{db_sublabel.id}"') + db_skeleton = models.Skeleton.objects.create(root=db_label, svg=svg) + logger.info(f'label:update Skeleton id:{db_skeleton.id} for label_id:{db_label.id}') if instance.project_id is None: update_labels(labels) From 754da0ecba3562df8b9582fa5943c21a2c2643d6 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Tue, 14 Feb 2023 12:29:31 +0200 Subject: [PATCH 02/30] Update Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ec4d8605ea..559c2cee761 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,7 @@ Tracks can be exported/imported to/from Datumaro and Sly Pointcloud formats () - \[Server API\] Various errors in the generated schema () - SiamMask and TransT serverless functions () +- Re-deleting of skeleton sublabels () ### Security - Fixed vulnerability with social authentication () From ed87e38f4a3aa713ec510c9f0af85cd2648ea4f0 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Tue, 14 Feb 2023 19:49:58 +0200 Subject: [PATCH 03/30] Update Label model, remove code duplication --- .../0063_alter_label_unique_together.py | 17 ++ cvat/apps/engine/models.py | 2 +- cvat/apps/engine/serializers.py | 174 ++++++++---------- 3 files changed, 90 insertions(+), 103 deletions(-) create mode 100644 cvat/apps/engine/migrations/0063_alter_label_unique_together.py diff --git a/cvat/apps/engine/migrations/0063_alter_label_unique_together.py b/cvat/apps/engine/migrations/0063_alter_label_unique_together.py new file mode 100644 index 00000000000..8bf1e2a1a5d --- /dev/null +++ b/cvat/apps/engine/migrations/0063_alter_label_unique_together.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.17 on 2023-02-14 17:15 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('engine', '0062_delete_previews'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='label', + unique_together={('task', 'project', 'name', 'parent')}, + ), + ] diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index 1084b25466d..5d0e0a95ec7 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -521,7 +521,7 @@ def has_parent_label(self): class Meta: default_permissions = () - unique_together = ('task', 'name', 'parent') + unique_together = ('task', 'project', 'name', 'parent') class Skeleton(models.Model): root = models.OneToOneField(Label, on_delete=models.CASCADE) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index a6e68414972..ee245d4fd68 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -6,23 +6,25 @@ import os import re import shutil - -from tempfile import NamedTemporaryFile import textwrap +from tempfile import NamedTemporaryFile from typing import OrderedDict -from rest_framework import serializers, exceptions -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import Group, User +from django.db import transaction +from drf_spectacular.utils import (OpenApiExample, extend_schema_field, + extend_schema_serializer) +from rest_framework import exceptions, serializers from cvat.apps.dataset_manager.formats.utils import get_label_color from cvat.apps.engine import models -from cvat.apps.engine.cloud_provider import get_cloud_storage_instance, Credentials, Status +from cvat.apps.engine.cloud_provider import (Credentials, Status, + get_cloud_storage_instance) from cvat.apps.engine.log import slogger from cvat.apps.engine.utils import parse_specific_attributes +from cvat.apps.engine.view_utils import (build_field_filter_params, + get_list_view_name, reverse) -from drf_spectacular.utils import OpenApiExample, extend_schema_field, extend_schema_serializer - -from cvat.apps.engine.view_utils import build_field_filter_params, get_list_view_name, reverse @extend_schema_field(serializers.URLField) class HyperlinkedModelViewSerializer(serializers.Serializer): @@ -600,6 +602,7 @@ def to_representation(self, instance): return serializer.data # pylint: disable=no-self-use + @transaction.atomic def create(self, validated_data): project_id = validated_data.get("project_id") if not (validated_data.get("label_set") or project_id): @@ -629,45 +632,20 @@ def create(self, validated_data): **storages, **validated_data) - def create_labels(labels, parent_label=None): - label_colors = list() - for label in labels: - attributes = label.pop('attributespec_set') - if label.get('id', None): - del label['id'] - if not label.get('color', None): - label['color'] = get_label_color(label['name'], label_colors) - label_colors.append(label['color']) - - sublabels = label.pop('sublabels', []) - svg = label.pop('svg', '') - db_label = models.Label.objects.create(task=db_task, parent=parent_label, **label) - logger.info(f'label:create: Label id:{db_label.id} for spec:{label} with sublabels:{sublabels}, parent_label:{parent_label}') - create_labels(sublabels, parent_label=db_label) - if db_label.type == str(models.LabelType.SKELETON): - for db_sublabel in list(db_label.sublabels.all()): - svg = svg.replace(f'data-label-name="{db_sublabel.name}"', f'data-label-id="{db_sublabel.id}"') - db_skeleton = models.Skeleton.objects.create(root=db_label, svg=svg) - logger.info(f'label:create Skeleton id:{db_skeleton.id} for label_id:{db_label.id}') - - for attr in attributes: - if attr.get('id', None): - del attr['id'] - models.AttributeSpec.objects.create(label=db_label, **attr) - task_path = db_task.get_dirname() if os.path.isdir(task_path): shutil.rmtree(task_path) os.makedirs(db_task.get_task_logs_dirname()) os.makedirs(db_task.get_task_artifacts_dirname()) - logger = slogger.task[db_task.id] - create_labels(labels) + + _create_labels(labels, db_task=db_task) db_task.save() return db_task # pylint: disable=no-self-use + @transaction.atomic def update(self, instance, validated_data): instance.name = validated_data.get('name', instance.name) instance.owner_id = validated_data.get('owner_id', instance.owner_id) @@ -677,26 +655,9 @@ def update(self, instance, validated_data): instance.subset = validated_data.get('subset', instance.subset) labels = validated_data.get('label_set', []) - logger = slogger.task[instance.id] - def update_labels(labels, parent_label=None): - for label in labels: - sublabels = label.pop('sublabels', []) - svg = label.pop('svg', '') - db_label = LabelSerializer.update_instance(label, instance, parent_label) - if db_label: - logger.info(f'label:update Label id:{db_label.id} for spec:{label} with sublabels:{sublabels}, parent_label:{parent_label}') - else: - logger.info(f'label:delete label:{label} with sublabels:{sublabels}, parent_label:{parent_label}') - if not label.get('deleted'): - update_labels(sublabels, parent_label=db_label) - if label.get('id') is None and db_label.type == str(models.LabelType.SKELETON): - for db_sublabel in list(db_label.sublabels.all()): - svg = svg.replace(f'data-label-name="{db_sublabel.name}"', f'data-label-id="{db_sublabel.id}"') - db_skeleton = models.Skeleton.objects.create(root=db_label, svg=svg) - logger.info(f'label:update Skeleton id:{db_skeleton.id} for label_id:{db_label.id}') - if instance.project_id is None: - update_labels(labels) + logger = slogger.task[instance.id] + _update_labels(labels, instance, logger) validated_project_id = validated_data.get('project_id') if validated_project_id is not None and validated_project_id != instance.project_id: @@ -854,6 +815,7 @@ def to_representation(self, instance): return serializer.data # pylint: disable=no-self-use + @transaction.atomic def create(self, validated_data): labels = validated_data.pop('label_set') @@ -867,43 +829,17 @@ def create(self, validated_data): **storages, **validated_data) - def create_labels(labels, parent_label=None): - label_colors = list() - for label in labels: - attributes = label.pop('attributespec_set') - if label.get('id', None): - del label['id'] - if not label.get('color', None): - label['color'] = get_label_color(label['name'], label_colors) - label_colors.append(label['color']) - - sublabels = label.pop('sublabels', []) - svg = label.pop('svg', []) - db_label = models.Label.objects.create(project=db_project, parent=parent_label, **label) - logger.info(f'label:create Label id:{db_label.id} for spec:{label} with sublabels:{sublabels}, parent_label:{parent_label}') - create_labels(sublabels, parent_label=db_label) - if db_label.type == str(models.LabelType.SKELETON): - for db_sublabel in list(db_label.sublabels.all()): - svg = svg.replace(f'data-label-name="{db_sublabel.name}"', f'data-label-id="{db_sublabel.id}"') - db_skeleton = models.Skeleton.objects.create(root=db_label, svg=svg) - logger.info(f'label:create Skeleton id:{db_skeleton.id} for label_id:{db_label.id}') - - for attr in attributes: - if attr.get('id', None): - del attr['id'] - models.AttributeSpec.objects.create(label=db_label, **attr) - project_path = db_project.get_dirname() if os.path.isdir(project_path): shutil.rmtree(project_path) os.makedirs(db_project.get_project_logs_dirname()) - logger = slogger.project[db_project.id] - create_labels(labels) + _create_labels(labels, db_project=db_project) return db_project # pylint: disable=no-self-use + @transaction.atomic def update(self, instance, validated_data): instance.name = validated_data.get('name', instance.name) instance.owner_id = validated_data.get('owner_id', instance.owner_id) @@ -912,25 +848,7 @@ def update(self, instance, validated_data): labels = validated_data.get('label_set', []) logger = slogger.project[instance.id] - def update_labels(labels, parent_label=None): - for label in labels: - sublabels = label.pop('sublabels', []) - svg = label.pop('svg', '') - db_label = LabelSerializer.update_instance(label, instance, parent_label) - if db_label: - logger.info(f'label:update Label id:{db_label.id} for spec:{label} with sublabels:{sublabels}, parent_label:{parent_label}') - else: - logger.info(f'label:delete label:{label} with sublabels:{sublabels}, parent_label:{parent_label}') - if not label.get('deleted'): - update_labels(sublabels, parent_label=db_label) - - if label.get('id') is None and db_label.type == str(models.LabelType.SKELETON): - for db_sublabel in list(db_label.sublabels.all()): - svg = svg.replace(f'data-label-name="{db_sublabel.name}"', f'data-label-id="{db_sublabel.id}"') - db_skeleton = models.Skeleton.objects.create(root=db_label, svg=svg) - logger.info(f'label:update: Skeleton id:{db_skeleton.id} for label_id:{db_label.id}') - - update_labels(labels) + _update_labels(labels, instance, logger) # update source and target storages _update_related_storages(instance, validated_data) @@ -1545,3 +1463,55 @@ def _validate_existence_of_cloud_storage(cloud_storage_id): _ = models.CloudStorage.objects.get(id=cloud_storage_id) except models.CloudStorage.DoesNotExist: raise serializers.ValidationError(f'The specified cloud storage {cloud_storage_id} does not exist.') + +def _create_labels(labels, parent_label=None, db_task=None, db_project=None): + label_colors = list() + for label in labels: + attributes = label.pop('attributespec_set') + if label.get('id', None): + del label['id'] + if not label.get('color', None): + label['color'] = get_label_color(label['name'], label_colors) + label_colors.append(label['color']) + + sublabels = label.pop('sublabels', []) + svg = label.pop('svg', '') + + if db_task is not None: + db_label = models.Label.objects.create(task=db_task, parent=parent_label, **label) + logger = slogger.task[db_task.id] + elif db_project is not None: + db_label = models.Label.objects.create(project=db_project, parent=parent_label, **label) + logger = slogger.project[db_project.id] + + logger.info(f'label:create: Label id:{db_label.id} for spec:{label} with sublabels:{sublabels}, parent_label:{parent_label}') + _create_labels(sublabels, parent_label=db_label, db_task=db_task, db_project=db_project) + if db_label.type == str(models.LabelType.SKELETON): + for db_sublabel in list(db_label.sublabels.all()): + svg = svg.replace(f'data-label-name="{db_sublabel.name}"', f'data-label-id="{db_sublabel.id}"') + db_skeleton = models.Skeleton.objects.create(root=db_label, svg=svg) + logger.info(f'label:create Skeleton id:{db_skeleton.id} for label_id:{db_label.id}') + + for attr in attributes: + if attr.get('id', None): + del attr['id'] + models.AttributeSpec.objects.create(label=db_label, **attr) + +def _update_labels(labels, instance, logger, parent_label=None): + for label in labels: + sublabels = label.pop('sublabels', []) + svg = label.pop('svg', '') + db_label = LabelSerializer.update_instance(label, instance, parent_label) + if db_label: + logger.info(f'label:update Label id:{db_label.id} for spec:{label} with sublabels:{sublabels}, parent_label:{parent_label}') + else: + logger.info(f'label:delete label:{label} with sublabels:{sublabels}, parent_label:{parent_label}') + + if not label.get('deleted'): + _update_labels(sublabels, instance, logger, parent_label=db_label) + + if label.get('id') is None and db_label.type == str(models.LabelType.SKELETON): + for db_sublabel in list(db_label.sublabels.all()): + svg = svg.replace(f'data-label-name="{db_sublabel.name}"', f'data-label-id="{db_sublabel.id}"') + db_skeleton = models.Skeleton.objects.create(root=db_label, svg=svg) + logger.info(f'label:update Skeleton id:{db_skeleton.id} for label_id:{db_label.id}') From f6a7797af6ced5c082650ec04b48c44829fe66dd Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 15 Feb 2023 10:49:20 +0200 Subject: [PATCH 04/30] Merge with some code from #5662 --- cvat/apps/engine/serializers.py | 310 ++++++++++++++++++++++---------- 1 file changed, 212 insertions(+), 98 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index ee245d4fd68..131d07d238d 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -3,28 +3,28 @@ # # SPDX-License-Identifier: MIT +from copy import copy import os import re import shutil -import textwrap + from tempfile import NamedTemporaryFile -from typing import OrderedDict +import textwrap +from typing import Any, Dict, Iterable, Optional, OrderedDict, Union -from django.contrib.auth.models import Group, User +from rest_framework import serializers, exceptions +from django.contrib.auth.models import User, Group from django.db import transaction -from drf_spectacular.utils import (OpenApiExample, extend_schema_field, - extend_schema_serializer) -from rest_framework import exceptions, serializers from cvat.apps.dataset_manager.formats.utils import get_label_color from cvat.apps.engine import models -from cvat.apps.engine.cloud_provider import (Credentials, Status, - get_cloud_storage_instance) +from cvat.apps.engine.cloud_provider import get_cloud_storage_instance, Credentials, Status from cvat.apps.engine.log import slogger from cvat.apps.engine.utils import parse_specific_attributes -from cvat.apps.engine.view_utils import (build_field_filter_params, - get_list_view_name, reverse) +from drf_spectacular.utils import OpenApiExample, extend_schema_field, extend_schema_serializer + +from cvat.apps.engine.view_utils import build_field_filter_params, get_list_view_name, reverse @extend_schema_field(serializers.URLField) class HyperlinkedModelViewSerializer(serializers.Serializer): @@ -150,52 +150,94 @@ def to_representation(self, instance): label['svg'] = instance.skeleton.svg return label + def __init__(self, *args, **kwargs): + self._local = kwargs.pop('local', False) + """ + Indicates that the operation is called from the dedicated ViewSet + and not from the parent entity, i.e. a project or task. + """ + + super().__init__(*args, **kwargs) + def validate(self, attrs): + if self._local: + if attrs.get('deleted'): + raise serializers.ValidationError( + 'Labels cannot be deleted by updating in this endpoint. ' + 'Please use the DELETE method instead.' + ) + if attrs.get('deleted') and attrs.get('id') is None: raise serializers.ValidationError('Deleted label must have an ID') return attrs - @staticmethod - def update_instance(validated_data, parent_instance, parent_label=None): + @classmethod + def update_label( + cls, + validated_data: Dict[str, Any], + *, + parent_instance: Union[models.Project, models.Task], + parent_label: Optional[models.Label] = None + ) -> Optional[models.Label]: + parent_info, logger = cls._get_parent_info(parent_instance) + attributes = validated_data.pop('attributespec_set', []) - instance = dict() - if isinstance(parent_instance, models.Project): - instance['project'] = parent_instance - logger = slogger.project[parent_instance.id] - else: - instance['task'] = parent_instance - logger = slogger.task[parent_instance.id] - if not validated_data.get('id') is None: + + if validated_data.get('id') is not None: try: - db_label = models.Label.objects.get(id=validated_data['id'], - **instance) - except models.Label.DoesNotExist: - raise exceptions.NotFound(detail='Not found label with id #{} to change'.format(validated_data['id'])) - db_label.name = validated_data.get('name', db_label.name) - updated_type = validated_data.get('type', db_label.type) - if 'skeleton' not in [db_label.type, updated_type]: - # do not permit to change types from/to "skeleton" + db_label = models.Label.objects.get(id=validated_data['id'], **parent_info) + except models.Label.DoesNotExist as exc: + raise exceptions.NotFound( + detail='Not found label with id #{} to change'.format(validated_data['id']) + ) from exc + + updated_type = validated_data.get('type') or db_label.type + if str(models.LabelType.SKELETON) in [db_label.type, updated_type]: + # do not permit changing types from/to skeleton + logger.warning("Label id {} ({}): an attempt to change label type from {} to {}. " + "Changing from or to '{}' is not allowed, the type won't be changed.".format( + db_label.id, + db_label.name, + db_label.type, + updated_type, + str(models.LabelType.SKELETON), + )) + else: db_label.type = updated_type - logger.info("{}({}) label was updated".format(db_label.name, db_label.id)) + + db_label.name = validated_data.get('name') or db_label.name + + logger.info("Label id {} ({}) was updated".format(db_label.id, db_label.name)) else: - db_label = models.Label.objects.create(name=validated_data.get('name'), type=validated_data.get('type'), - parent=parent_label, **instance) + db_label = models.Label.objects.create( + name=validated_data.get('name'), + type=validated_data.get('type'), + parent=parent_label, + **parent_info + ) logger.info("New {} label was created".format(db_label.name)) + if validated_data.get('deleted'): + assert validated_data['id'] # must be checked in the validate() db_label.delete() - return + return None + if not validated_data.get('color', None): - label_colors = [l.color for l in - instance[tuple(instance.keys())[0]].label_set.exclude(id=db_label.id).order_by('id') + other_label_colors = [ + label.color for label in + parent_instance.label_set.exclude(id=db_label.id).order_by('id') ] - db_label.color = get_label_color(db_label.name, label_colors) + db_label.color = get_label_color(db_label.name, other_label_colors) else: db_label.color = validated_data.get('color', db_label.color) + db_label.save() + for attr in attributes: (db_attr, created) = models.AttributeSpec.objects.get_or_create( - label=db_label, name=attr['name'], defaults=attr) + label=db_label, name=attr['name'], defaults=attr + ) if created: logger.info("New {} attribute for {} label was created" .format(db_attr.name, db_label.name)) @@ -209,8 +251,138 @@ def update_instance(validated_data, parent_instance, parent_label=None): db_attr.input_type = attr.get('input_type', db_attr.input_type) db_attr.values = attr.get('values', db_attr.values) db_attr.save() + return db_label + @classmethod + @transaction.atomic + def create_labels(cls, + labels: Iterable[Dict[str, Any]], + *, + parent_instance: Union[models.Project, models.Task], + parent_label: Optional[models.Label] = None + ): + parent_info, logger = cls._get_parent_info(parent_instance) + + label_colors = list() + + for label in labels: + attributes = label.pop('attributespec_set') + + if label.get('id', None): + del label['id'] + + if not label.get('color', None): + label['color'] = get_label_color(label['name'], label_colors) + label_colors.append(label['color']) + + sublabels = label.pop('sublabels', []) + svg = label.pop('svg', '') + db_label = models.Label.objects.create(**label, **parent_info, parent=parent_label) + logger.info( + f'label:create Label id:{db_label.id} for spec:{label} ' + f'with sublabels:{sublabels}, parent_label:{parent_label}' + ) + + cls.create_labels(sublabels, parent_instance=parent_instance, parent_label=db_label) + + if db_label.type == str(models.LabelType.SKELETON): + for db_sublabel in list(db_label.sublabels.all()): + svg = svg.replace( + f'data-label-name="{db_sublabel.name}"', + f'data-label-id="{db_sublabel.id}"' + ) + db_skeleton = models.Skeleton.objects.create(root=db_label, svg=svg) + logger.info(f'label:create Skeleton id:{db_skeleton.id} for label_id:{db_label.id}') + + for attr in attributes: + if attr.get('id', None): + del attr['id'] + models.AttributeSpec.objects.create(label=db_label, **attr) + + @classmethod + @transaction.atomic + def update_labels(cls, + labels: Iterable[Dict[str, Any]], + *, + parent_instance: Union[models.Project, models.Task], + parent_label: Optional[models.Label] = None + ): + _, logger = cls._get_parent_info(parent_instance) + + for label in labels: + sublabels = label.pop('sublabels', []) + svg = label.pop('svg', '') + db_label = cls.update_label(label, + parent_instance=parent_instance, parent_label=parent_label + ) + if db_label: + logger.info( + f'label:update Label id:{db_label.id} for spec:{label} ' + f'with sublabels:{sublabels}, parent_label:{parent_label}' + ) + else: + logger.info( + f'label:delete label:{label} with ' + f'sublabels:{sublabels}, parent_label:{parent_label}' + ) + + if not label.get('deleted'): + cls.update_labels(sublabels, parent_instance=parent_instance, parent_label=db_label) + + if label.get('id') is None and db_label.type == str(models.LabelType.SKELETON): + for db_sublabel in list(db_label.sublabels.all()): + svg = svg.replace( + f'data-label-name="{db_sublabel.name}"', + f'data-label-id="{db_sublabel.id}"' + ) + db_skeleton = models.Skeleton.objects.create(root=db_label, svg=svg) + logger.info( + f'label:update Skeleton id:{db_skeleton.id} for label_id:{db_label.id}' + ) + + @classmethod + def _get_parent_info(cls, parent_instance: Union[models.Project, models.Task]): + parent_info = {} + if isinstance(parent_instance, models.Project): + parent_info['project'] = parent_instance + logger = slogger.project[parent_instance.id] + elif isinstance(parent_instance, models.Task): + parent_info['task'] = parent_instance + logger = slogger.task[parent_instance.id] + else: + raise TypeError(f"Unexpected parent instance type {type(parent_instance).__name__}") + + return parent_info, logger + + def update(self, instance, validated_data): + if not self._local: + return super().update(instance, validated_data) + + # Here we reuse the parent entity logic to make sure everything is done + # like these entities expect. Initial data (unprocessed) is used to + # avoid introducing premature changes. + data = copy(self.initial_data) + data['id'] = instance.id + data.setdefault('name', instance.name) + parent_query = { 'labels': [data] } + + if isinstance(instance.project, models.Project): + parent_serializer = ProjectWriteSerializer( + instance=instance.project, data=parent_query, partial=True, + ) + elif isinstance(instance.task, models.Task): + parent_serializer = TaskWriteSerializer( + instance=instance.task, data=parent_query, partial=True, + ) + + parent_serializer.is_valid(raise_exception=True) + parent_serializer.save() + + self.instance = models.Label.objects.get(pk=instance.pk) + return self.instance + + class JobCommitSerializer(serializers.ModelSerializer): class Meta: model = models.JobCommit @@ -602,7 +774,6 @@ def to_representation(self, instance): return serializer.data # pylint: disable=no-self-use - @transaction.atomic def create(self, validated_data): project_id = validated_data.get("project_id") if not (validated_data.get("label_set") or project_id): @@ -639,13 +810,12 @@ def create(self, validated_data): os.makedirs(db_task.get_task_logs_dirname()) os.makedirs(db_task.get_task_artifacts_dirname()) - _create_labels(labels, db_task=db_task) + LabelSerializer.create_labels(labels, parent_instance=db_task) db_task.save() return db_task # pylint: disable=no-self-use - @transaction.atomic def update(self, instance, validated_data): instance.name = validated_data.get('name', instance.name) instance.owner_id = validated_data.get('owner_id', instance.owner_id) @@ -656,8 +826,7 @@ def update(self, instance, validated_data): labels = validated_data.get('label_set', []) if instance.project_id is None: - logger = slogger.task[instance.id] - _update_labels(labels, instance, logger) + LabelSerializer.update_labels(labels, parent_instance=instance) validated_project_id = validated_data.get('project_id') if validated_project_id is not None and validated_project_id != instance.project_id: @@ -815,7 +984,6 @@ def to_representation(self, instance): return serializer.data # pylint: disable=no-self-use - @transaction.atomic def create(self, validated_data): labels = validated_data.pop('label_set') @@ -834,12 +1002,11 @@ def create(self, validated_data): shutil.rmtree(project_path) os.makedirs(db_project.get_project_logs_dirname()) - _create_labels(labels, db_project=db_project) + LabelSerializer.create_labels(labels, parent_instance=db_project) return db_project # pylint: disable=no-self-use - @transaction.atomic def update(self, instance, validated_data): instance.name = validated_data.get('name', instance.name) instance.owner_id = validated_data.get('owner_id', instance.owner_id) @@ -847,8 +1014,7 @@ def update(self, instance, validated_data): instance.bug_tracker = validated_data.get('bug_tracker', instance.bug_tracker) labels = validated_data.get('label_set', []) - logger = slogger.project[instance.id] - _update_labels(labels, instance, logger) + LabelSerializer.update_labels(labels, parent_instance=instance) # update source and target storages _update_related_storages(instance, validated_data) @@ -1463,55 +1629,3 @@ def _validate_existence_of_cloud_storage(cloud_storage_id): _ = models.CloudStorage.objects.get(id=cloud_storage_id) except models.CloudStorage.DoesNotExist: raise serializers.ValidationError(f'The specified cloud storage {cloud_storage_id} does not exist.') - -def _create_labels(labels, parent_label=None, db_task=None, db_project=None): - label_colors = list() - for label in labels: - attributes = label.pop('attributespec_set') - if label.get('id', None): - del label['id'] - if not label.get('color', None): - label['color'] = get_label_color(label['name'], label_colors) - label_colors.append(label['color']) - - sublabels = label.pop('sublabels', []) - svg = label.pop('svg', '') - - if db_task is not None: - db_label = models.Label.objects.create(task=db_task, parent=parent_label, **label) - logger = slogger.task[db_task.id] - elif db_project is not None: - db_label = models.Label.objects.create(project=db_project, parent=parent_label, **label) - logger = slogger.project[db_project.id] - - logger.info(f'label:create: Label id:{db_label.id} for spec:{label} with sublabels:{sublabels}, parent_label:{parent_label}') - _create_labels(sublabels, parent_label=db_label, db_task=db_task, db_project=db_project) - if db_label.type == str(models.LabelType.SKELETON): - for db_sublabel in list(db_label.sublabels.all()): - svg = svg.replace(f'data-label-name="{db_sublabel.name}"', f'data-label-id="{db_sublabel.id}"') - db_skeleton = models.Skeleton.objects.create(root=db_label, svg=svg) - logger.info(f'label:create Skeleton id:{db_skeleton.id} for label_id:{db_label.id}') - - for attr in attributes: - if attr.get('id', None): - del attr['id'] - models.AttributeSpec.objects.create(label=db_label, **attr) - -def _update_labels(labels, instance, logger, parent_label=None): - for label in labels: - sublabels = label.pop('sublabels', []) - svg = label.pop('svg', '') - db_label = LabelSerializer.update_instance(label, instance, parent_label) - if db_label: - logger.info(f'label:update Label id:{db_label.id} for spec:{label} with sublabels:{sublabels}, parent_label:{parent_label}') - else: - logger.info(f'label:delete label:{label} with sublabels:{sublabels}, parent_label:{parent_label}') - - if not label.get('deleted'): - _update_labels(sublabels, instance, logger, parent_label=db_label) - - if label.get('id') is None and db_label.type == str(models.LabelType.SKELETON): - for db_sublabel in list(db_label.sublabels.all()): - svg = svg.replace(f'data-label-name="{db_sublabel.name}"', f'data-label-id="{db_sublabel.id}"') - db_skeleton = models.Skeleton.objects.create(root=db_label, svg=svg) - logger.info(f'label:update Skeleton id:{db_skeleton.id} for label_id:{db_label.id}') From 91341118c8e2a22f08b98a2083c2836b08a42208 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 15 Feb 2023 10:50:06 +0200 Subject: [PATCH 05/30] Add tests --- tests/python/rest_api/test_projects.py | 21 +++++++++++++++++++-- tests/python/rest_api/test_tasks.py | 22 +++++++++++++++++++--- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index fcab5ff4565..0fdb8d12175 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -17,10 +17,11 @@ import pytest from cvat_sdk.api_client import ApiClient, Configuration, models from cvat_sdk.api_client.api_client import Endpoint +from cvat_sdk.core.helpers import get_paginated_collection from deepdiff import DeepDiff from PIL import Image - -from shared.utils.config import BASE_URL, USER_PASS, get_method, make_api_client, patch_method +from shared.utils.config import (BASE_URL, USER_PASS, get_method, + make_api_client, patch_method, post_method) from .utils import CollectionSimpleFilterTestBase, export_dataset @@ -389,6 +390,22 @@ def _create_org(cls, api_client: ApiClient, members: Optional[Dict[str, str]] = return org + def test_cannot_create_project_with_two_same_labels(self, projects, admin_user): + project_spec = { + "name": "test cannot create project with two same labels", + "labels": [{"name": "car"}, {"name": "car"}], + } + response = post_method(admin_user, "/projects", project_spec) + assert response.status_code == HTTPStatus.BAD_REQUEST + + projects = list(projects) + with make_api_client(admin_user) as api_client: + results = get_paginated_collection( + api_client.projects_api.list_endpoint, + return_json=True, + ) + assert DeepDiff(projects, results, ignore_order=True) == {} + def _check_cvat_for_video_project_annotations_meta(content, values_to_be_checked): document = ET.fromstring(content) diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index a815b0c0e8a..1c8e7cdb3de 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -16,6 +16,7 @@ from time import sleep import pytest +import shared.utils.s3 as s3 from cvat_sdk import Client, Config from cvat_sdk.api_client import apis, models from cvat_sdk.api_client.api_client import ApiClient, Endpoint @@ -23,10 +24,9 @@ from cvat_sdk.core.proxies.tasks import ResourceType, Task from deepdiff import DeepDiff from PIL import Image - -import shared.utils.s3 as s3 from shared.fixtures.init import get_server_image_tag -from shared.utils.config import BASE_URL, USER_PASS, get_method, make_api_client, patch_method +from shared.utils.config import (BASE_URL, USER_PASS, get_method, + make_api_client, patch_method, post_method) from shared.utils.helpers import generate_image_files from .utils import CollectionSimpleFilterTestBase, export_dataset @@ -926,6 +926,22 @@ def test_can_specify_file_job_mapping(self): start_frame = stop_frame + 1 + def test_cannot_create_task_with_two_same_labels(self, tasks): + task_spec = { + "name": "test cannot create task with two same labels", + "labels": [{"name": "car"}, {"name": "car"}], + } + response = post_method(self._USERNAME, "/tasks", task_spec) + assert response.status_code == HTTPStatus.BAD_REQUEST + + tasks = list(tasks) + with make_api_client(self._USERNAME) as api_client: + results = get_paginated_collection( + api_client.tasks_api.list_endpoint, + return_json=True, + ) + assert DeepDiff(tasks, results, ignore_order=True) == {} + @pytest.mark.usefixtures("restore_db_per_function") @pytest.mark.usefixtures("restore_cvat_data") From c78eafc52662248772cd0c53819795bd259820e4 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 15 Feb 2023 12:25:55 +0200 Subject: [PATCH 06/30] Fix linters --- tests/python/rest_api/test_projects.py | 10 ++++++++-- tests/python/rest_api/test_tasks.py | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index 0fdb8d12175..cec7c59ef02 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -20,8 +20,14 @@ from cvat_sdk.core.helpers import get_paginated_collection from deepdiff import DeepDiff from PIL import Image -from shared.utils.config import (BASE_URL, USER_PASS, get_method, - make_api_client, patch_method, post_method) +from shared.utils.config import ( + BASE_URL, + USER_PASS, + get_method, + make_api_client, + patch_method, + post_method, +) from .utils import CollectionSimpleFilterTestBase, export_dataset diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 1c8e7cdb3de..8d218321b49 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -25,8 +25,14 @@ from deepdiff import DeepDiff from PIL import Image from shared.fixtures.init import get_server_image_tag -from shared.utils.config import (BASE_URL, USER_PASS, get_method, - make_api_client, patch_method, post_method) +from shared.utils.config import ( + BASE_URL, + USER_PASS, + get_method, + make_api_client, + patch_method, + post_method, +) from shared.utils.helpers import generate_image_files from .utils import CollectionSimpleFilterTestBase, export_dataset From 0ebc55f5a0aa03b935bed85ef7ac728cedaca015 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 15 Feb 2023 14:15:34 +0200 Subject: [PATCH 07/30] Fix tests --- tests/python/rest_api/test_projects.py | 1 + tests/python/rest_api/test_tasks.py | 13 +++---------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index cec7c59ef02..d16a55d04ab 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -20,6 +20,7 @@ from cvat_sdk.core.helpers import get_paginated_collection from deepdiff import DeepDiff from PIL import Image + from shared.utils.config import ( BASE_URL, USER_PASS, diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 8d218321b49..7d8480b908b 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -16,7 +16,6 @@ from time import sleep import pytest -import shared.utils.s3 as s3 from cvat_sdk import Client, Config from cvat_sdk.api_client import apis, models from cvat_sdk.api_client.api_client import ApiClient, Endpoint @@ -24,6 +23,8 @@ from cvat_sdk.core.proxies.tasks import ResourceType, Task from deepdiff import DeepDiff from PIL import Image + +import shared.utils.s3 as s3 from shared.fixtures.init import get_server_image_tag from shared.utils.config import ( BASE_URL, @@ -932,7 +933,7 @@ def test_can_specify_file_job_mapping(self): start_frame = stop_frame + 1 - def test_cannot_create_task_with_two_same_labels(self, tasks): + def test_cannot_create_task_with_two_same_labels(self): task_spec = { "name": "test cannot create task with two same labels", "labels": [{"name": "car"}, {"name": "car"}], @@ -940,14 +941,6 @@ def test_cannot_create_task_with_two_same_labels(self, tasks): response = post_method(self._USERNAME, "/tasks", task_spec) assert response.status_code == HTTPStatus.BAD_REQUEST - tasks = list(tasks) - with make_api_client(self._USERNAME) as api_client: - results = get_paginated_collection( - api_client.tasks_api.list_endpoint, - return_json=True, - ) - assert DeepDiff(tasks, results, ignore_order=True) == {} - @pytest.mark.usefixtures("restore_db_per_function") @pytest.mark.usefixtures("restore_cvat_data") From b97d7fac484bccfd0c5d8bc2ca40199b30374dca Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 15 Feb 2023 14:27:48 +0200 Subject: [PATCH 08/30] Update Changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 559c2cee761..03bbad4345f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,7 +63,7 @@ Tracks can be exported/imported to/from Datumaro and Sly Pointcloud formats () - \[Server API\] Various errors in the generated schema () - SiamMask and TransT serverless functions () -- Re-deleting of skeleton sublabels () +- Сreating a project with the same labels and re-deleting of skeleton sublabels () ### Security - Fixed vulnerability with social authentication () From 1594449c8617b6043b4f427bfccf9b47addfdfd1 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 15 Feb 2023 14:28:14 +0200 Subject: [PATCH 09/30] pdate tests --- tests/python/rest_api/test_projects.py | 2 +- tests/python/rest_api/test_tasks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index d16a55d04ab..3af90e07cde 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -397,7 +397,7 @@ def _create_org(cls, api_client: ApiClient, members: Optional[Dict[str, str]] = return org - def test_cannot_create_project_with_two_same_labels(self, projects, admin_user): + def test_cannot_create_project_with_same_labels(self, projects, admin_user): project_spec = { "name": "test cannot create project with two same labels", "labels": [{"name": "car"}, {"name": "car"}], diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 7d8480b908b..05312d29528 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -933,7 +933,7 @@ def test_can_specify_file_job_mapping(self): start_frame = stop_frame + 1 - def test_cannot_create_task_with_two_same_labels(self): + def test_cannot_create_task_with_same_labels(self): task_spec = { "name": "test cannot create task with two same labels", "labels": [{"name": "car"}, {"name": "car"}], From a786f26b5f2e22f0bfea8e9d55c214010fa6a390 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 15 Feb 2023 17:32:40 +0200 Subject: [PATCH 10/30] Fixes --- .../0063_alter_label_unique_together.py | 4 ++-- cvat/apps/engine/models.py | 3 ++- cvat/apps/engine/serializers.py | 21 ++++++++-------- tests/python/rest_api/test_projects.py | 24 ++++++++++--------- tests/python/rest_api/test_tasks.py | 18 ++++++++++---- 5 files changed, 42 insertions(+), 28 deletions(-) diff --git a/cvat/apps/engine/migrations/0063_alter_label_unique_together.py b/cvat/apps/engine/migrations/0063_alter_label_unique_together.py index 8bf1e2a1a5d..a6061361d6b 100644 --- a/cvat/apps/engine/migrations/0063_alter_label_unique_together.py +++ b/cvat/apps/engine/migrations/0063_alter_label_unique_together.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.17 on 2023-02-14 17:15 +# Generated by Django 3.2.17 on 2023-02-15 14:39 from django.db import migrations @@ -12,6 +12,6 @@ class Migration(migrations.Migration): operations = [ migrations.AlterUniqueTogether( name='label', - unique_together={('task', 'project', 'name', 'parent')}, + unique_together={('task', 'name', 'parent'), ('project', 'name', 'parent')}, ), ] diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index 5d0e0a95ec7..57f1347991f 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -521,7 +521,8 @@ def has_parent_label(self): class Meta: default_permissions = () - unique_together = ('task', 'project', 'name', 'parent') + unique_together = (('task', 'name', 'parent'), + ('project', 'name', 'parent')) class Skeleton(models.Model): root = models.OneToOneField(Label, on_delete=models.CASCADE) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 131d07d238d..bdaec50f3f2 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -3,28 +3,29 @@ # # SPDX-License-Identifier: MIT -from copy import copy import os import re import shutil - -from tempfile import NamedTemporaryFile import textwrap +from copy import copy +from tempfile import NamedTemporaryFile from typing import Any, Dict, Iterable, Optional, OrderedDict, Union -from rest_framework import serializers, exceptions -from django.contrib.auth.models import User, Group -from django.db import transaction +from django.contrib.auth.models import Group, User +from django.db import IntegrityError, transaction +from drf_spectacular.utils import (OpenApiExample, extend_schema_field, + extend_schema_serializer) +from rest_framework import exceptions, serializers from cvat.apps.dataset_manager.formats.utils import get_label_color from cvat.apps.engine import models -from cvat.apps.engine.cloud_provider import get_cloud_storage_instance, Credentials, Status +from cvat.apps.engine.cloud_provider import (Credentials, Status, + get_cloud_storage_instance) from cvat.apps.engine.log import slogger from cvat.apps.engine.utils import parse_specific_attributes +from cvat.apps.engine.view_utils import (build_field_filter_params, + get_list_view_name, reverse) -from drf_spectacular.utils import OpenApiExample, extend_schema_field, extend_schema_serializer - -from cvat.apps.engine.view_utils import build_field_filter_params, get_list_view_name, reverse @extend_schema_field(serializers.URLField) class HyperlinkedModelViewSerializer(serializers.Serializer): diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index 3af90e07cde..7771252a5cf 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -397,21 +397,23 @@ def _create_org(cls, api_client: ApiClient, members: Optional[Dict[str, str]] = return org - def test_cannot_create_project_with_same_labels(self, projects, admin_user): + def test_cannot_create_project_with_same_labels(self, admin_user): project_spec = { - "name": "test cannot create project with two same labels", - "labels": [{"name": "car"}, {"name": "car"}], + "name": "test cannot create project with same skeletons", + "labels": [{ + "name": "s1", + "type": "skeleton", + "sublabels": [ + {"name": "1"}, + {"name": "1"} + ] + }] } response = post_method(admin_user, "/projects", project_spec) - assert response.status_code == HTTPStatus.BAD_REQUEST + assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR - projects = list(projects) - with make_api_client(admin_user) as api_client: - results = get_paginated_collection( - api_client.projects_api.list_endpoint, - return_json=True, - ) - assert DeepDiff(projects, results, ignore_order=True) == {} + response = post_method(admin_user, "/tasks", project_spec) + assert response.status_code == HTTPStatus.OK def _check_cvat_for_video_project_annotations_meta(content, values_to_be_checked): diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 05312d29528..53105e93fbf 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -933,13 +933,23 @@ def test_can_specify_file_job_mapping(self): start_frame = stop_frame + 1 - def test_cannot_create_task_with_same_labels(self): + def test_cannot_create_task_with_same_skeletons(self): task_spec = { - "name": "test cannot create task with two same labels", - "labels": [{"name": "car"}, {"name": "car"}], + "name": "test cannot create task with same skeletons", + "labels": [{ + "name": "s1", + "type": "skeleton", + "sublabels": [ + {"name": "1"}, + {"name": "1"} + ] + }] } response = post_method(self._USERNAME, "/tasks", task_spec) - assert response.status_code == HTTPStatus.BAD_REQUEST + assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR + + response = get_method(self._USERNAME, "/tasks") + assert response.status_code == HTTPStatus.OK @pytest.mark.usefixtures("restore_db_per_function") From 9d9f2b2ff6930827848cdf201b1dffb317bc220f Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 15 Feb 2023 17:41:02 +0200 Subject: [PATCH 11/30] FIx black --- tests/python/rest_api/test_projects.py | 11 +++-------- tests/python/rest_api/test_tasks.py | 11 +++-------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index 7771252a5cf..91ea1905d0e 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -400,14 +400,9 @@ def _create_org(cls, api_client: ApiClient, members: Optional[Dict[str, str]] = def test_cannot_create_project_with_same_labels(self, admin_user): project_spec = { "name": "test cannot create project with same skeletons", - "labels": [{ - "name": "s1", - "type": "skeleton", - "sublabels": [ - {"name": "1"}, - {"name": "1"} - ] - }] + "labels": [ + {"name": "s1", "type": "skeleton", "sublabels": [{"name": "1"}, {"name": "1"}]} + ], } response = post_method(admin_user, "/projects", project_spec) assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 53105e93fbf..34b4624f0bf 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -936,14 +936,9 @@ def test_can_specify_file_job_mapping(self): def test_cannot_create_task_with_same_skeletons(self): task_spec = { "name": "test cannot create task with same skeletons", - "labels": [{ - "name": "s1", - "type": "skeleton", - "sublabels": [ - {"name": "1"}, - {"name": "1"} - ] - }] + "labels": [ + {"name": "s1", "type": "skeleton", "sublabels": [{"name": "1"}, {"name": "1"}]} + ], } response = post_method(self._USERNAME, "/tasks", task_spec) assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR From 5de8e4b118fc1dd71efff774a3dd425c51c18151 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 15 Feb 2023 18:20:19 +0200 Subject: [PATCH 12/30] Fix pylint --- cvat/apps/engine/serializers.py | 2 +- tests/python/rest_api/test_projects.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index bdaec50f3f2..9b6cb089661 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -12,7 +12,7 @@ from typing import Any, Dict, Iterable, Optional, OrderedDict, Union from django.contrib.auth.models import Group, User -from django.db import IntegrityError, transaction +from django.db import transaction from drf_spectacular.utils import (OpenApiExample, extend_schema_field, extend_schema_serializer) from rest_framework import exceptions, serializers diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index 91ea1905d0e..3a9b1530347 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -17,7 +17,6 @@ import pytest from cvat_sdk.api_client import ApiClient, Configuration, models from cvat_sdk.api_client.api_client import Endpoint -from cvat_sdk.core.helpers import get_paginated_collection from deepdiff import DeepDiff from PIL import Image From 9cecd9e0b69a97067f0ba9bb8db944bd1fca17d3 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 15 Feb 2023 18:22:51 +0200 Subject: [PATCH 13/30] Small fix --- tests/python/rest_api/test_projects.py | 4 ++-- tests/python/rest_api/test_tasks.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index 3a9b1530347..f5217165fc5 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -396,9 +396,9 @@ def _create_org(cls, api_client: ApiClient, members: Optional[Dict[str, str]] = return org - def test_cannot_create_project_with_same_labels(self, admin_user): + def test_cannot_create_project_with_same_skeleton_sublabels(self, admin_user): project_spec = { - "name": "test cannot create project with same skeletons", + "name": "test cannot create project with same skeleton sublabels", "labels": [ {"name": "s1", "type": "skeleton", "sublabels": [{"name": "1"}, {"name": "1"}]} ], diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 34b4624f0bf..4639b63a72d 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -933,9 +933,9 @@ def test_can_specify_file_job_mapping(self): start_frame = stop_frame + 1 - def test_cannot_create_task_with_same_skeletons(self): + def test_cannot_create_task_with_same_skeleton_sublabels(self): task_spec = { - "name": "test cannot create task with same skeletons", + "name": "test cannot create task with same skeleton sublabels", "labels": [ {"name": "s1", "type": "skeleton", "sublabels": [{"name": "1"}, {"name": "1"}]} ], From 8f6b6c788e0134821f58edebe75cacfe0bc9e54b Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 15 Feb 2023 18:34:53 +0200 Subject: [PATCH 14/30] Fix test --- tests/python/rest_api/test_projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index f5217165fc5..207aa576da1 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -406,7 +406,7 @@ def test_cannot_create_project_with_same_skeleton_sublabels(self, admin_user): response = post_method(admin_user, "/projects", project_spec) assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR - response = post_method(admin_user, "/tasks", project_spec) + response = get_method(admin_user, "/projects") assert response.status_code == HTTPStatus.OK From 54e9ed6be4e079803b7bd14563be516902e90c4a Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Mon, 20 Feb 2023 09:29:28 +0200 Subject: [PATCH 15/30] Apply comments --- cvat/apps/engine/serializers.py | 43 --------------------------------- 1 file changed, 43 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 7251fde3610..37ece22579a 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -151,23 +151,7 @@ def to_representation(self, instance): label['svg'] = instance.skeleton.svg return label - def __init__(self, *args, **kwargs): - self._local = kwargs.pop('local', False) - """ - Indicates that the operation is called from the dedicated ViewSet - and not from the parent entity, i.e. a project or task. - """ - - super().__init__(*args, **kwargs) - def validate(self, attrs): - if self._local: - if attrs.get('deleted'): - raise serializers.ValidationError( - 'Labels cannot be deleted by updating in this endpoint. ' - 'Please use the DELETE method instead.' - ) - if attrs.get('deleted') and attrs.get('id') is None: raise serializers.ValidationError('Deleted label must have an ID') @@ -356,33 +340,6 @@ def _get_parent_info(cls, parent_instance: Union[models.Project, models.Task]): return parent_info, logger - def update(self, instance, validated_data): - if not self._local: - return super().update(instance, validated_data) - - # Here we reuse the parent entity logic to make sure everything is done - # like these entities expect. Initial data (unprocessed) is used to - # avoid introducing premature changes. - data = copy(self.initial_data) - data['id'] = instance.id - data.setdefault('name', instance.name) - parent_query = { 'labels': [data] } - - if isinstance(instance.project, models.Project): - parent_serializer = ProjectWriteSerializer( - instance=instance.project, data=parent_query, partial=True, - ) - elif isinstance(instance.task, models.Task): - parent_serializer = TaskWriteSerializer( - instance=instance.task, data=parent_query, partial=True, - ) - - parent_serializer.is_valid(raise_exception=True) - parent_serializer.save() - - self.instance = models.Label.objects.get(pk=instance.pk) - return self.instance - class JobCommitSerializer(serializers.ModelSerializer): class Meta: From 20012c294713024949af663317c2c82e9ddd7c47 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Mon, 20 Feb 2023 11:26:03 +0200 Subject: [PATCH 16/30] Update migrations --- .../migrations/0064_delete_wrong_labels.py | 39 +++++++++++++++++++ ...py => 0065_alter_label_unique_together.py} | 6 +-- cvat/apps/engine/serializers.py | 6 --- 3 files changed, 42 insertions(+), 9 deletions(-) create mode 100644 cvat/apps/engine/migrations/0064_delete_wrong_labels.py rename cvat/apps/engine/migrations/{0063_alter_label_unique_together.py => 0065_alter_label_unique_together.py} (52%) diff --git a/cvat/apps/engine/migrations/0064_delete_wrong_labels.py b/cvat/apps/engine/migrations/0064_delete_wrong_labels.py new file mode 100644 index 00000000000..b11f5aed009 --- /dev/null +++ b/cvat/apps/engine/migrations/0064_delete_wrong_labels.py @@ -0,0 +1,39 @@ +import os + +from django.db import migrations +from cvat.apps.engine.log import get_migration_logger + +def delete_wrong_labels(apps, schema_editor): + migration_name = os.path.splitext(os.path.basename(__file__))[0] + with get_migration_logger(migration_name) as log: + log.info('\nDeleting skeleton Labels without skeletons...') + + Label = apps.get_model('engine', 'Label') + for label in Label.objects.all(): + if label.type == "skeleton" and not hasattr(label, "skeleton"): + label.delete() + + log.info('\nDeleting duplicate Labels...') + for name, parent, project, task in Label.objects.values_list("name", "parent", "project", "task").distinct(): + duplicate_labels = Label.objects.filter(name=name, parent=parent, project=project).values_list('id', "parent") + if task is not None: + duplicate_labels = Label.objects.filter(name=name, parent=parent, task=task).values_list('id', "parent") + + if len(duplicate_labels) > 1: + label = duplicate_labels[0] + if label[1] is not None: + Label.objects.get(pk=label[1]).delete() + else: + Label.objects.get(pk=label[0]).delete() + +class Migration(migrations.Migration): + + dependencies = [ + ('engine', '0063_delete_jobcommit'), + ] + + operations = [ + migrations.RunPython( + code=delete_wrong_labels + ), + ] diff --git a/cvat/apps/engine/migrations/0063_alter_label_unique_together.py b/cvat/apps/engine/migrations/0065_alter_label_unique_together.py similarity index 52% rename from cvat/apps/engine/migrations/0063_alter_label_unique_together.py rename to cvat/apps/engine/migrations/0065_alter_label_unique_together.py index a6061361d6b..98dd66ca748 100644 --- a/cvat/apps/engine/migrations/0063_alter_label_unique_together.py +++ b/cvat/apps/engine/migrations/0065_alter_label_unique_together.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.17 on 2023-02-15 14:39 +# Generated by Django 3.2.18 on 2023-02-20 09:24 from django.db import migrations @@ -6,12 +6,12 @@ class Migration(migrations.Migration): dependencies = [ - ('engine', '0062_delete_previews'), + ('engine', '0064_delete_wrong_labels'), ] operations = [ migrations.AlterUniqueTogether( name='label', - unique_together={('task', 'name', 'parent'), ('project', 'name', 'parent')}, + unique_together={('project', 'name', 'parent'), ('task', 'name', 'parent')}, ), ] diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 37ece22579a..90d86aac5fd 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -341,12 +341,6 @@ def _get_parent_info(cls, parent_instance: Union[models.Project, models.Task]): return parent_info, logger -class JobCommitSerializer(serializers.ModelSerializer): - class Meta: - model = models.JobCommit - fields = ('id', 'owner', 'data', 'timestamp', 'scope') - - class JobReadSerializer(serializers.ModelSerializer): task_id = serializers.ReadOnlyField(source="segment.task.id") project_id = serializers.ReadOnlyField(source="get_project_id", allow_null=True) From 668f865e4622a0e6e2d40126d7b65d8e53b56c1e Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Mon, 20 Feb 2023 11:27:27 +0200 Subject: [PATCH 17/30] Fix linters --- cvat/apps/engine/serializers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 90d86aac5fd..e4e43ad57fd 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -7,7 +7,6 @@ import re import shutil import textwrap -from copy import copy from tempfile import NamedTemporaryFile from typing import Any, Dict, Iterable, Optional, OrderedDict, Union From 8f3db61b02e32b28b8ff09193e94c553ad68ecd6 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Mon, 20 Feb 2023 18:36:41 +0200 Subject: [PATCH 18/30] Update validation --- cvat/apps/engine/serializers.py | 21 ++++++++++++++++----- tests/python/rest_api/test_projects.py | 2 +- tests/python/rest_api/test_tasks.py | 2 +- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index e4e43ad57fd..544d3014e38 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -868,14 +868,20 @@ def validate(self, attrs): for label, sublabels in new_sublabel_names.items(): if sublabels != target_project_sublabel_names.get(label): raise serializers.ValidationError('All task or project label names must be mapped to the target project') - else: - if 'label_set' in attrs.keys(): - label_names = [label['name'] for label in attrs.get('label_set')] - if len(label_names) != len(set(label_names)): - raise serializers.ValidationError('All label names must be unique for the task') return attrs + def validate_labels(self, value): + if value: + label_names = [label['name'] for label in value] + if len(label_names) != len(set(label_names)): + raise serializers.ValidationError('All label names must be unique for the project') + + for label in value: + self.validate_labels(label["sublabels"]) + + return value + class ProjectReadSerializer(serializers.ModelSerializer): labels = LabelSerializer(many=True, source='label_set', partial=True, default=[], read_only=True) @@ -968,7 +974,12 @@ def validate_labels(self, value): label_names = [label['name'] for label in value] if len(label_names) != len(set(label_names)): raise serializers.ValidationError('All label names must be unique for the project') + + for label in value: + self.validate_labels(label["sublabels"]) + return value + class AboutSerializer(serializers.Serializer): name = serializers.CharField(max_length=128) description = serializers.CharField(max_length=2048) diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index 207aa576da1..ab35ac91bf6 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -404,7 +404,7 @@ def test_cannot_create_project_with_same_skeleton_sublabels(self, admin_user): ], } response = post_method(admin_user, "/projects", project_spec) - assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR + assert response.status_code == HTTPStatus.BAD_REQUEST response = get_method(admin_user, "/projects") assert response.status_code == HTTPStatus.OK diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 4639b63a72d..51f73fe33cb 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -941,7 +941,7 @@ def test_cannot_create_task_with_same_skeleton_sublabels(self): ], } response = post_method(self._USERNAME, "/tasks", task_spec) - assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR + assert response.status_code == HTTPStatus.BAD_REQUEST response = get_method(self._USERNAME, "/tasks") assert response.status_code == HTTPStatus.OK From 09be228332297ed7246ed78e50d63683f90c8474 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Mon, 20 Feb 2023 19:37:36 +0200 Subject: [PATCH 19/30] Fix tests --- cvat/apps/engine/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 544d3014e38..07e8f5083e1 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -878,7 +878,7 @@ def validate_labels(self, value): raise serializers.ValidationError('All label names must be unique for the project') for label in value: - self.validate_labels(label["sublabels"]) + self.validate_labels(label.get("sublabels")) return value @@ -976,7 +976,7 @@ def validate_labels(self, value): raise serializers.ValidationError('All label names must be unique for the project') for label in value: - self.validate_labels(label["sublabels"]) + self.validate_labels(label.get("sublabels")) return value From d87103768c65caeecf87eab3db61a8938fed2a9f Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Tue, 21 Feb 2023 11:33:42 +0200 Subject: [PATCH 20/30] Replace unique_together with UniqueConstraint --- .../0065_alter_label_unique_together.py | 17 ---------- .../migrations/0065_auto_20230221_0931.py | 33 +++++++++++++++++++ cvat/apps/engine/models.py | 24 ++++++++++++-- 3 files changed, 55 insertions(+), 19 deletions(-) delete mode 100644 cvat/apps/engine/migrations/0065_alter_label_unique_together.py create mode 100644 cvat/apps/engine/migrations/0065_auto_20230221_0931.py diff --git a/cvat/apps/engine/migrations/0065_alter_label_unique_together.py b/cvat/apps/engine/migrations/0065_alter_label_unique_together.py deleted file mode 100644 index 98dd66ca748..00000000000 --- a/cvat/apps/engine/migrations/0065_alter_label_unique_together.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 3.2.18 on 2023-02-20 09:24 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('engine', '0064_delete_wrong_labels'), - ] - - operations = [ - migrations.AlterUniqueTogether( - name='label', - unique_together={('project', 'name', 'parent'), ('task', 'name', 'parent')}, - ), - ] diff --git a/cvat/apps/engine/migrations/0065_auto_20230221_0931.py b/cvat/apps/engine/migrations/0065_auto_20230221_0931.py new file mode 100644 index 00000000000..beeb261f2cd --- /dev/null +++ b/cvat/apps/engine/migrations/0065_auto_20230221_0931.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.18 on 2023-02-21 09:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('engine', '0064_delete_wrong_labels'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='label', + unique_together=set(), + ), + migrations.AddConstraint( + model_name='label', + constraint=models.UniqueConstraint(condition=models.Q(('parent__isnull', True), ('task__isnull', True)), fields=('project', 'name'), name='project_name_unique'), + ), + migrations.AddConstraint( + model_name='label', + constraint=models.UniqueConstraint(condition=models.Q(('parent__isnull', True), ('project__isnull', True)), fields=('task', 'name'), name='task_name_unique'), + ), + migrations.AddConstraint( + model_name='label', + constraint=models.UniqueConstraint(condition=models.Q(('task__isnull', True)), fields=('project', 'name', 'parent'), name='project_name_parent_unique'), + ), + migrations.AddConstraint( + model_name='label', + constraint=models.UniqueConstraint(condition=models.Q(('project__isnull', True)), fields=('task', 'name', 'parent'), name='task_name_parent_unique'), + ), + ] diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index c33e7d38462..e4ffab9c9a5 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -514,8 +514,28 @@ def has_parent_label(self): class Meta: default_permissions = () - unique_together = (('task', 'name', 'parent'), - ('project', 'name', 'parent')) + constraints = [ + models.UniqueConstraint( + name='project_name_unique', + fields=('project', 'name'), + condition=models.Q(task__isnull=True, parent__isnull=True) + ), + models.UniqueConstraint( + name='task_name_unique', + fields=('task', 'name'), + condition=models.Q(project__isnull=True, parent__isnull=True) + ), + models.UniqueConstraint( + name='project_name_parent_unique', + fields=('project', 'name', 'parent'), + condition=models.Q(task__isnull=True) + ), + models.UniqueConstraint( + name='task_name_parent_unique', + fields=('task', 'name', 'parent'), + condition=models.Q(project__isnull=True) + ) + ] class Skeleton(models.Model): root = models.OneToOneField(Label, on_delete=models.CASCADE) From 9fecbaf3dd0fb1e0cd2591dbdf30474a7a4717c8 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Tue, 21 Feb 2023 18:10:19 +0200 Subject: [PATCH 21/30] Add tests for same labels --- tests/python/rest_api/test_projects.py | 13 +++++++++++++ tests/python/rest_api/test_tasks.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index ab35ac91bf6..880598cc2f9 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -396,6 +396,19 @@ def _create_org(cls, api_client: ApiClient, members: Optional[Dict[str, str]] = return org + def test_cannot_create_project_with_same_labels(self, admin_user): + project_spec = { + "name": "test cannot create project with same labels", + "labels": [ + {"name": "l1"}, {"name": "l1"} + ], + } + response = post_method(admin_user, "/projects", project_spec) + assert response.status_code == HTTPStatus.BAD_REQUEST + + response = get_method(admin_user, "/projects") + assert response.status_code == HTTPStatus.OK + def test_cannot_create_project_with_same_skeleton_sublabels(self, admin_user): project_spec = { "name": "test cannot create project with same skeleton sublabels", diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 51f73fe33cb..a5e0f3479a0 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -933,6 +933,19 @@ def test_can_specify_file_job_mapping(self): start_frame = stop_frame + 1 + def test_cannot_create_task_with_same_labels(self): + task_spec = { + "name": "test cannot create task with same labels", + "labels": [ + {"name": "l1"}, {"name": "l1"} + ], + } + response = post_method(self._USERNAME, "/tasks", task_spec) + assert response.status_code == HTTPStatus.BAD_REQUEST + + response = get_method(self._USERNAME, "/tasks") + assert response.status_code == HTTPStatus.OK + def test_cannot_create_task_with_same_skeleton_sublabels(self): task_spec = { "name": "test cannot create task with same skeleton sublabels", From 143a98d099a01ac515538b20049f9252ecbf4304 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Tue, 21 Feb 2023 18:24:34 +0200 Subject: [PATCH 22/30] Fix linters --- cvat/apps/engine/serializers.py | 21 ++++++--------------- tests/python/rest_api/test_projects.py | 4 +--- tests/python/rest_api/test_tasks.py | 4 +--- 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 981a03c4204..cb60f607850 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -8,33 +8,24 @@ import os import re import shutil -import textwrap -<<<<<<< HEAD -from tempfile import NamedTemporaryFile -from typing import Any, Dict, Iterable, Optional, OrderedDict, Union -from django.contrib.auth.models import Group, User -from django.db import transaction -from drf_spectacular.utils import (OpenApiExample, extend_schema_field, - extend_schema_serializer) -from rest_framework import exceptions, serializers -======= +from tempfile import NamedTemporaryFile +import textwrap from typing import Any, Dict, Iterable, Optional, OrderedDict, Union from rest_framework import serializers, exceptions from django.contrib.auth.models import User, Group from django.db import transaction ->>>>>>> develop from cvat.apps.dataset_manager.formats.utils import get_label_color from cvat.apps.engine import models -from cvat.apps.engine.cloud_provider import (Credentials, Status, - get_cloud_storage_instance) +from cvat.apps.engine.cloud_provider import get_cloud_storage_instance, Credentials, Status from cvat.apps.engine.log import slogger from cvat.apps.engine.utils import parse_specific_attributes -from cvat.apps.engine.view_utils import (build_field_filter_params, - get_list_view_name, reverse) +from drf_spectacular.utils import OpenApiExample, extend_schema_field, extend_schema_serializer + +from cvat.apps.engine.view_utils import build_field_filter_params, get_list_view_name, reverse @extend_schema_field(serializers.URLField) class HyperlinkedEndpointSerializer(serializers.Serializer): diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index edb321109c8..bb8d0d7c763 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -400,9 +400,7 @@ def _create_org(cls, api_client: ApiClient, members: Optional[Dict[str, str]] = def test_cannot_create_project_with_same_labels(self, admin_user): project_spec = { "name": "test cannot create project with same labels", - "labels": [ - {"name": "l1"}, {"name": "l1"} - ], + "labels": [{"name": "l1"}, {"name": "l1"}], } response = post_method(admin_user, "/projects", project_spec) assert response.status_code == HTTPStatus.BAD_REQUEST diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index bce81e01a26..83ca3832815 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -939,9 +939,7 @@ def test_can_specify_file_job_mapping(self): def test_cannot_create_task_with_same_labels(self): task_spec = { "name": "test cannot create task with same labels", - "labels": [ - {"name": "l1"}, {"name": "l1"} - ], + "labels": [{"name": "l1"}, {"name": "l1"}], } response = post_method(self._USERNAME, "/tasks", task_spec) assert response.status_code == HTTPStatus.BAD_REQUEST From a03c4a9b37110c7f9328104e2f43ef8365a5b9c1 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Tue, 21 Feb 2023 20:18:54 +0200 Subject: [PATCH 23/30] Remove invalid checks --- cvat/apps/engine/models.py | 19 ---------- cvat/apps/engine/serializers.py | 65 +++++++-------------------------- 2 files changed, 14 insertions(+), 70 deletions(-) diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index e0b0f12d0e7..d92ee118df3 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -520,25 +520,6 @@ def __str__(self): def has_parent_label(self): return bool(self.parent) - def _check_save_constraints(self) -> None: - # NOTE: constraints don't work for some reason - # https://github.com/opencv/cvat/pull/5700#discussion_r1112276036 - # This method is not 100% reliable because of possible race conditions - # but it should work in relevant cases. - - parent_entity = self.project or self.task - - # Check for possible labels name duplicates in case of saving the new label - existing_labels: models.QuerySet = parent_entity.get_labels() - if self.id: - existing_labels = existing_labels.exclude(id=self.id) - if existing_labels.filter(name=self.name).count(): - raise InvalidLabel(f"Label '{self.name}' already exists") - - def save(self, *args, **kwargs) -> None: - self._check_save_constraints() - return super().save(*args, **kwargs) - class Meta: default_permissions = () constraints = [ diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index cb60f607850..c962316ca7f 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -15,7 +15,7 @@ from rest_framework import serializers, exceptions from django.contrib.auth.models import User, Group -from django.db import transaction +from django.db import IntegrityError, transaction from cvat.apps.dataset_manager.formats.utils import get_label_color from cvat.apps.engine import models @@ -287,12 +287,15 @@ def update_label( logger.info("Label id {} ({}) was updated".format(db_label.id, db_label.name)) else: - db_label = models.Label.objects.create( - name=validated_data.get('name'), - type=validated_data.get('type'), - parent=parent_label, - **parent_info - ) + try: + db_label = models.Label.objects.create( + name=validated_data.get('name'), + type=validated_data.get('type'), + parent=parent_label, + **parent_info + ) + except IntegrityError as exc: + raise exceptions.ValidationError(str(exc)) from exc logger.info("New {} label was created".format(db_label.name)) if validated_data.get('deleted'): @@ -358,7 +361,10 @@ def create_labels(cls, sublabels = label.pop('sublabels', []) svg = label.pop('svg', '') - db_label = models.Label.objects.create(**label, **parent_info, parent=parent_label) + try: + db_label = models.Label.objects.create(**label, **parent_info, parent=parent_label) + except IntegrityError as exc: + raise exceptions.ValidationError(str(exc)) from exc logger.info( f'label:create Label id:{db_label.id} for spec:{label} ' f'with sublabels:{sublabels}, parent_label:{parent_label}' @@ -990,36 +996,9 @@ def validate(self, attrs): for label, sublabels in new_sublabel_names.items(): if sublabels != target_project_sublabel_names.get(label): raise serializers.ValidationError('All task or project label names must be mapped to the target project') - else: - if 'label_set' in attrs.keys(): - # FIXME: doesn't work for renaming just a single label - label_names = [ - label['name'] - for label in attrs.get('label_set') - if not label.get('deleted') - ] - if len(label_names) != len(set(label_names)): - raise serializers.ValidationError('All label names must be unique for the task') return attrs - def validate_labels(self, value): - if value: - # FIXME: doesn't work for renaming just a single label - label_names = [ - label['name'] - for label in value - if not label.get('deleted') - ] - if len(label_names) != len(set(label_names)): - raise serializers.ValidationError('All label names must be unique for the task') - - for label in value: - self.validate_labels(label.get("sublabels")) - - return value - - class ProjectReadSerializer(serializers.ModelSerializer): owner = BasicUserSerializer(required=False, read_only=True) assignee = BasicUserSerializer(allow_null=True, required=False, read_only=True) @@ -1106,22 +1085,6 @@ def update(self, instance, validated_data): instance.save() return instance - def validate_labels(self, value): - if value: - # FIXME: doesn't work for renaming just a single label - label_names = [ - label['name'] - for label in value - if not label.get('deleted') - ] - if len(label_names) != len(set(label_names)): - raise serializers.ValidationError('All label names must be unique for the project') - - for label in value: - self.validate_labels(label.get("sublabels")) - - return value - class AboutSerializer(serializers.Serializer): name = serializers.CharField(max_length=128) description = serializers.CharField(max_length=2048) From 8b0b6f22bbdd454a7e4f8aebc8babb937d8d4656 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 22 Feb 2023 09:46:01 +0200 Subject: [PATCH 24/30] Add save() and create() methods to Label model --- cvat/apps/engine/models.py | 27 ++++++++++++++++++++++----- cvat/apps/engine/serializers.py | 27 +++++++++------------------ 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index d92ee118df3..086832557ca 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -12,13 +12,16 @@ from django.conf import settings from django.contrib.auth.models import User from django.core.files.storage import FileSystemStorage -from django.db import models +from django.db import IntegrityError, models from django.db.models.fields import FloatField from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema_field +from rest_framework import exceptions + from cvat.apps.engine.utils import parse_specific_attributes from cvat.apps.organizations.models import Organization + class SafeCharField(models.CharField): def get_prep_value(self, value): value = super().get_prep_value(value) @@ -502,10 +505,6 @@ def get_labels(self): class Meta: default_permissions = () - -class InvalidLabel(ValueError): - pass - class Label(models.Model): task = models.ForeignKey(Task, null=True, blank=True, on_delete=models.CASCADE) project = models.ForeignKey(Project, null=True, blank=True, on_delete=models.CASCADE) @@ -520,6 +519,24 @@ def __str__(self): def has_parent_label(self): return bool(self.parent) + def save(self, force_insert=False, force_update=False, using=None, + update_fields=None): + try: + super().save(force_insert, force_update, using, update_fields) + except ValueError as exc: + raise exceptions.ValidationError(str(exc)) from exc + except IntegrityError: + raise exceptions.ValidationError("All label names must be unique") + + @classmethod + def create(cls, **kwargs): + try: + return cls.objects.create(**kwargs) + except ValueError as exc: + raise exceptions.ValidationError(str(exc)) from exc + except IntegrityError: + raise exceptions.ValidationError("All label names must be unique") + class Meta: default_permissions = () constraints = [ diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index c962316ca7f..b87bae6c241 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -15,7 +15,7 @@ from rest_framework import serializers, exceptions from django.contrib.auth.models import User, Group -from django.db import IntegrityError, transaction +from django.db import transaction from cvat.apps.dataset_manager.formats.utils import get_label_color from cvat.apps.engine import models @@ -287,15 +287,12 @@ def update_label( logger.info("Label id {} ({}) was updated".format(db_label.id, db_label.name)) else: - try: - db_label = models.Label.objects.create( - name=validated_data.get('name'), - type=validated_data.get('type'), - parent=parent_label, - **parent_info - ) - except IntegrityError as exc: - raise exceptions.ValidationError(str(exc)) from exc + db_label = models.Label.create( + name=validated_data.get('name'), + type=validated_data.get('type'), + parent=parent_label, + **parent_info + ) logger.info("New {} label was created".format(db_label.name)) if validated_data.get('deleted'): @@ -312,10 +309,7 @@ def update_label( else: db_label.color = validated_data.get('color', db_label.color) - try: - db_label.save() - except models.InvalidLabel as exc: - raise exceptions.ValidationError(str(exc)) from exc + db_label.save() for attr in attributes: (db_attr, created) = models.AttributeSpec.objects.get_or_create( @@ -361,10 +355,7 @@ def create_labels(cls, sublabels = label.pop('sublabels', []) svg = label.pop('svg', '') - try: - db_label = models.Label.objects.create(**label, **parent_info, parent=parent_label) - except IntegrityError as exc: - raise exceptions.ValidationError(str(exc)) from exc + db_label = models.Label.create(**label, **parent_info, parent=parent_label) logger.info( f'label:create Label id:{db_label.id} for spec:{label} ' f'with sublabels:{sublabels}, parent_label:{parent_label}' From 7583d5b31d23d43609671bf3117d345736eb6f4b Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 22 Feb 2023 10:43:27 +0200 Subject: [PATCH 25/30] Fix tests --- tests/python/rest_api/test_labels.py | 2 +- tests/python/rest_api/test_projects.py | 2 +- tests/python/rest_api/test_tasks.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/python/rest_api/test_labels.py b/tests/python/rest_api/test_labels.py index ba1985d3886..6ba36d132c7 100644 --- a/tests/python/rest_api/test_labels.py +++ b/tests/python/rest_api/test_labels.py @@ -698,7 +698,7 @@ def test_cannot_rename_label_to_duplicate_name(self, source_type, user): response = self._test_update_denied( user, lid=labels[0]["id"], data=payload, expected_status=HTTPStatus.BAD_REQUEST ) - assert f"Label '{payload['name']}' already exists" in response.data.decode() + assert "All label names must be unique" in response.data.decode() def test_admin_patch_sandbox_label(self, admin_sandbox_case): label, user = get_attrs(admin_sandbox_case, ["label", "user"]) diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index bb8d0d7c763..09c7c6bfb68 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -720,7 +720,7 @@ def test_cannot_rename_label_to_duplicate_name(self, projects, labels, admin_use admin_user, f'/projects/{project["id"]}', {"labels": [label_payload]} ) assert response.status_code == HTTPStatus.BAD_REQUEST - assert f"Label '{project_labels[0]['name']}' already exists" in response.text + assert "All label names must be unique" in response.text def test_cannot_add_foreign_label(self, projects, labels, admin_user): project = list(projects)[0] diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 83ca3832815..a4d63ff05ed 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -1022,7 +1022,7 @@ def test_cannot_rename_label_to_duplicate_name(self, tasks, labels, admin_user): response = patch_method(admin_user, f'/tasks/{task["id"]}', {"labels": [label_payload]}) assert response.status_code == HTTPStatus.BAD_REQUEST - assert f"Label '{task_labels[0]['name']}' already exists" in response.text + assert "All label names must be unique" in response.text def test_cannot_add_foreign_label(self, tasks, labels, admin_user): task = list(tasks)[0] From 4ea1c0734083466bf586502643082df4df5e3357 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 22 Feb 2023 17:47:33 +0200 Subject: [PATCH 26/30] Apply comments --- cvat/apps/engine/models.py | 11 +++++------ cvat/apps/engine/serializers.py | 26 ++++++++++++++++++-------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index 086832557ca..c063b83e5c9 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -505,6 +505,9 @@ def get_labels(self): class Meta: default_permissions = () +class InvalidLabel(ValueError): + pass + class Label(models.Model): task = models.ForeignKey(Task, null=True, blank=True, on_delete=models.CASCADE) project = models.ForeignKey(Project, null=True, blank=True, on_delete=models.CASCADE) @@ -523,19 +526,15 @@ def save(self, force_insert=False, force_update=False, using=None, update_fields=None): try: super().save(force_insert, force_update, using, update_fields) - except ValueError as exc: - raise exceptions.ValidationError(str(exc)) from exc except IntegrityError: - raise exceptions.ValidationError("All label names must be unique") + raise InvalidLabel("All label names must be unique") @classmethod def create(cls, **kwargs): try: return cls.objects.create(**kwargs) - except ValueError as exc: - raise exceptions.ValidationError(str(exc)) from exc except IntegrityError: - raise exceptions.ValidationError("All label names must be unique") + raise InvalidLabel("All label names must be unique") class Meta: default_permissions = () diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index b87bae6c241..942efba2be2 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -250,6 +250,7 @@ def validate(self, attrs): return attrs @classmethod + @transaction.atomic def update_label( cls, validated_data: Dict[str, Any], @@ -287,12 +288,15 @@ def update_label( logger.info("Label id {} ({}) was updated".format(db_label.id, db_label.name)) else: - db_label = models.Label.create( - name=validated_data.get('name'), - type=validated_data.get('type'), - parent=parent_label, - **parent_info - ) + try: + db_label = models.Label.create( + name=validated_data.get('name'), + type=validated_data.get('type'), + parent=parent_label, + **parent_info + ) + except models.InvalidLabel as exc: + raise exceptions.ValidationError(str(exc)) from exc logger.info("New {} label was created".format(db_label.name)) if validated_data.get('deleted'): @@ -309,7 +313,10 @@ def update_label( else: db_label.color = validated_data.get('color', db_label.color) - db_label.save() + try: + db_label.save() + except models.InvalidLabel as exc: + raise exceptions.ValidationError(str(exc)) from exc for attr in attributes: (db_attr, created) = models.AttributeSpec.objects.get_or_create( @@ -355,7 +362,10 @@ def create_labels(cls, sublabels = label.pop('sublabels', []) svg = label.pop('svg', '') - db_label = models.Label.create(**label, **parent_info, parent=parent_label) + try: + db_label = models.Label.create(**label, **parent_info, parent=parent_label) + except models.InvalidLabel as exc: + raise exceptions.ValidationError(str(exc)) from exc logger.info( f'label:create Label id:{db_label.id} for spec:{label} ' f'with sublabels:{sublabels}, parent_label:{parent_label}' From 0f3cba07835936d4d0507f2199808f4415929bec Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 22 Feb 2023 20:24:50 +0200 Subject: [PATCH 27/30] Update migration --- .../engine/migrations/0064_delete_wrong_labels.py | 14 +++++++------- cvat/apps/engine/models.py | 5 ++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/cvat/apps/engine/migrations/0064_delete_wrong_labels.py b/cvat/apps/engine/migrations/0064_delete_wrong_labels.py index b11f5aed009..7db87558dc7 100644 --- a/cvat/apps/engine/migrations/0064_delete_wrong_labels.py +++ b/cvat/apps/engine/migrations/0064_delete_wrong_labels.py @@ -1,5 +1,3 @@ -import os - from django.db import migrations from cvat.apps.engine.log import get_migration_logger @@ -15,16 +13,18 @@ def delete_wrong_labels(apps, schema_editor): log.info('\nDeleting duplicate Labels...') for name, parent, project, task in Label.objects.values_list("name", "parent", "project", "task").distinct(): - duplicate_labels = Label.objects.filter(name=name, parent=parent, project=project).values_list('id', "parent") + duplicate_labels = Label.objects.filter(name=name, parent=parent, project=project) if task is not None: - duplicate_labels = Label.objects.filter(name=name, parent=parent, task=task).values_list('id', "parent") + duplicate_labels = Label.objects.filter(name=name, parent=parent, task=task) if len(duplicate_labels) > 1: label = duplicate_labels[0] - if label[1] is not None: - Label.objects.get(pk=label[1]).delete() + if label.parent is not None: + label.delete() else: - Label.objects.get(pk=label[0]).delete() + for i, label in enumerate(duplicate_labels[1:]): + label.name = f"{label.name}_duplicate_{i + 1}" + label.save() class Migration(migrations.Migration): diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index c063b83e5c9..ff4e1c1a0e9 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -522,10 +522,9 @@ def __str__(self): def has_parent_label(self): return bool(self.parent) - def save(self, force_insert=False, force_update=False, using=None, - update_fields=None): + def save(self, *args, **kwargs): try: - super().save(force_insert, force_update, using, update_fields) + super().save(*args, **kwargs) except IntegrityError: raise InvalidLabel("All label names must be unique") From fe11bd44b68a06534485dea13331412eb151ee63 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 22 Feb 2023 20:28:42 +0200 Subject: [PATCH 28/30] Fix linters --- cvat/apps/engine/migrations/0064_delete_wrong_labels.py | 2 ++ cvat/apps/engine/models.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cvat/apps/engine/migrations/0064_delete_wrong_labels.py b/cvat/apps/engine/migrations/0064_delete_wrong_labels.py index 7db87558dc7..c2f96035dba 100644 --- a/cvat/apps/engine/migrations/0064_delete_wrong_labels.py +++ b/cvat/apps/engine/migrations/0064_delete_wrong_labels.py @@ -1,3 +1,5 @@ +import os + from django.db import migrations from cvat.apps.engine.log import get_migration_logger diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index ff4e1c1a0e9..db97e6dfa13 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -16,7 +16,6 @@ from django.db.models.fields import FloatField from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema_field -from rest_framework import exceptions from cvat.apps.engine.utils import parse_specific_attributes from cvat.apps.organizations.models import Organization From c463bb18c07f51d54aae43ebf07eff3b02b921f8 Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Wed, 22 Feb 2023 21:50:54 +0200 Subject: [PATCH 29/30] Add @transaction.atomic to tasks and projects serializers --- cvat/apps/engine/serializers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 942efba2be2..15b0f5c787f 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -843,6 +843,7 @@ def to_representation(self, instance): return serializer.data # pylint: disable=no-self-use + @transaction.atomic def create(self, validated_data): project_id = validated_data.get("project_id") if not (validated_data.get("label_set") or project_id): @@ -885,6 +886,7 @@ def create(self, validated_data): return db_task # pylint: disable=no-self-use + @transaction.atomic def update(self, instance, validated_data): instance.name = validated_data.get('name', instance.name) instance.owner_id = validated_data.get('owner_id', instance.owner_id) @@ -1048,6 +1050,7 @@ def to_representation(self, instance): return serializer.data # pylint: disable=no-self-use + @transaction.atomic def create(self, validated_data): labels = validated_data.pop('label_set') @@ -1071,6 +1074,7 @@ def create(self, validated_data): return db_project # pylint: disable=no-self-use + @transaction.atomic def update(self, instance, validated_data): instance.name = validated_data.get('name', instance.name) instance.owner_id = validated_data.get('owner_id', instance.owner_id) From a76954ba3070500c1489af7b670c079918d56aee Mon Sep 17 00:00:00 2001 From: yasakova-anastasia Date: Thu, 23 Feb 2023 08:56:34 +0200 Subject: [PATCH 30/30] Update migrations --- ...rong_labels.py => 0064_delete_or_rename_wrong_labels.py} | 6 +++--- cvat/apps/engine/migrations/0065_auto_20230221_0931.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename cvat/apps/engine/migrations/{0064_delete_wrong_labels.py => 0064_delete_or_rename_wrong_labels.py} (87%) diff --git a/cvat/apps/engine/migrations/0064_delete_wrong_labels.py b/cvat/apps/engine/migrations/0064_delete_or_rename_wrong_labels.py similarity index 87% rename from cvat/apps/engine/migrations/0064_delete_wrong_labels.py rename to cvat/apps/engine/migrations/0064_delete_or_rename_wrong_labels.py index c2f96035dba..63c16738152 100644 --- a/cvat/apps/engine/migrations/0064_delete_wrong_labels.py +++ b/cvat/apps/engine/migrations/0064_delete_or_rename_wrong_labels.py @@ -3,7 +3,7 @@ from django.db import migrations from cvat.apps.engine.log import get_migration_logger -def delete_wrong_labels(apps, schema_editor): +def delete_or_rename_wrong_labels(apps, schema_editor): migration_name = os.path.splitext(os.path.basename(__file__))[0] with get_migration_logger(migration_name) as log: log.info('\nDeleting skeleton Labels without skeletons...') @@ -13,7 +13,7 @@ def delete_wrong_labels(apps, schema_editor): if label.type == "skeleton" and not hasattr(label, "skeleton"): label.delete() - log.info('\nDeleting duplicate Labels...') + log.info('\nDeleting duplicate skeleton sublabels and renaming duplicate Labels...') for name, parent, project, task in Label.objects.values_list("name", "parent", "project", "task").distinct(): duplicate_labels = Label.objects.filter(name=name, parent=parent, project=project) if task is not None: @@ -36,6 +36,6 @@ class Migration(migrations.Migration): operations = [ migrations.RunPython( - code=delete_wrong_labels + code=delete_or_rename_wrong_labels ), ] diff --git a/cvat/apps/engine/migrations/0065_auto_20230221_0931.py b/cvat/apps/engine/migrations/0065_auto_20230221_0931.py index beeb261f2cd..f1b09c4ca23 100644 --- a/cvat/apps/engine/migrations/0065_auto_20230221_0931.py +++ b/cvat/apps/engine/migrations/0065_auto_20230221_0931.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('engine', '0064_delete_wrong_labels'), + ('engine', '0064_delete_or_rename_wrong_labels'), ] operations = [