From d488eca9560d4f61bd8f0deee1f3476f82ea21b6 Mon Sep 17 00:00:00 2001 From: Zhiltsov Max Date: Mon, 18 Nov 2019 14:16:24 +0300 Subject: [PATCH 1/7] Add labelme export --- cvat/apps/annotation/labelme.py | 102 +++++++++++++++++++++++++++++++ cvat/apps/annotation/settings.py | 1 + 2 files changed, 103 insertions(+) create mode 100644 cvat/apps/annotation/labelme.py diff --git a/cvat/apps/annotation/labelme.py b/cvat/apps/annotation/labelme.py new file mode 100644 index 000000000000..9874caf68fa0 --- /dev/null +++ b/cvat/apps/annotation/labelme.py @@ -0,0 +1,102 @@ +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +format_spec = { + "name": "LabelMe", + "dumpers": [ + { + "display_name": "{name} {format} {version} for images", + "format": "XML", + "version": "3.0", + "handler": "dump_as_labelme_annotation" + } + ], + "loaders": [ + { + "display_name": "{name} {format} {version}", + "format": "XML", + "version": "3.0", + "handler": "load", + } + ], +} + +def dump_frame_anno(frame_annotation): + from lxml import etree as ET + + root_elem = ET.Element('annotation') + + ET.SubElement(root_elem, 'filename').text = frame_annotation.name + ET.SubElement(root_elem, 'folder').text = '' + + source_elem = ET.SubElement(root_elem, 'source') + ET.SubElement(source_elem, 'sourceImage').text = 'task #' + ET.SubElement(source_elem, 'sourceAnnotation').text = 'CVAT' + + image_elem = ET.SubElement(root_elem, 'imagesize') + ET.SubElement(image_elem, 'nrows').text = str(frame_annotation.height) + ET.SubElement(image_elem, 'ncols').text = str(frame_annotation.width) + + next_obj_id = 0 + for shape in frame_annotation.labeled_shapes: + obj_elem = ET.SubElement(root_elem, 'object') + ET.SubElement(obj_elem, 'name').text = str(shape.label) + ET.SubElement(obj_elem, 'deleted').text = '0' + ET.SubElement(obj_elem, 'verified').text = '0' + ET.SubElement(obj_elem, 'occluded').text = str(int(shape.occluded)) + ET.SubElement(obj_elem, 'date').text = '0' + ET.SubElement(obj_elem, 'id').text = str(next_obj_id) + next_obj_id += 1 + + parts_elem = ET.SubElement(obj_elem, 'parts') + # TODO: handle groups + # .text = str(shape.group) + ET.SubElement(parts_elem, 'hasparts') + ET.SubElement(parts_elem, 'ispartof') + + if shape.type == 'rectangle': + ET.SubElement(obj_elem, 'type').text = 'bounding_box' + + poly_elem = ET.SubElement(obj_elem, 'polygon') + x0 = shape.points[0] + y0 = shape.points[1] + x1 = shape.points[2] + y1 = shape.points[4] + points = [ (x0, y0), (x1, y0), (x1, y1), (x0, y1) ] + for x, y in points: + point_elem = ET.SubElement(poly_elem, 'pt') + ET.SubElement(point_elem, 'x').text = '%.2f' % x + ET.SubElement(point_elem, 'y').text = '%.2f' % y + elif shape.type == 'polygon': + poly_elem = ET.SubElement(obj_elem, 'polygon') + for x, y in zip(shape.points[::2], shape.points[1::2]): + point_elem = ET.SubElement(poly_elem, 'pt') + ET.SubElement(point_elem, 'x').text = '%.2f' % x + ET.SubElement(point_elem, 'y').text = '%.2f' % y + elif shape.type == 'polyline': + pass + elif shape.type == 'points': + pass + else: + raise NotImplementedError("Unknown shape type '%s'" % shape.type) + + attr_string = '' + for i, attr in enumerate(shape.attributes): + if i: + attr_string += ', ' + attr_string += '%s=%s' % (attr.name, attr.value) + ET.SubElement(obj_elem, 'attributes').text = attr_string + + return ET.tostring(root_elem, encoding='unicode', pretty_print=True) + +def dump_as_labelme_annotation(file_object, annotations): + from zipfile import ZipFile + + with ZipFile(file_object, 'w') as output_zip: + for frame_annotation in annotations.group_by_frame(): + xml_data = dump_frame_anno(frame_annotation) + output_zip.writestr(frame_annotation.name + '.xml', xml_data) + +def load(file_object, annotations): + pass \ No newline at end of file diff --git a/cvat/apps/annotation/settings.py b/cvat/apps/annotation/settings.py index 0ac2a38c8ad4..1c42bf3b04aa 100644 --- a/cvat/apps/annotation/settings.py +++ b/cvat/apps/annotation/settings.py @@ -12,4 +12,5 @@ os.path.join(path_prefix, 'coco.py'), os.path.join(path_prefix, 'mask.py'), os.path.join(path_prefix, 'tfrecord.py'), + os.path.join(path_prefix, 'labelme.py'), ) From f264e22faaf9f57267fc2c2fd27af1cf9a0aa18b Mon Sep 17 00:00:00 2001 From: Zhiltsov Max Date: Tue, 19 Nov 2019 14:21:08 +0300 Subject: [PATCH 2/7] Fixes --- cvat/apps/annotation/labelme.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/cvat/apps/annotation/labelme.py b/cvat/apps/annotation/labelme.py index 9874caf68fa0..1b58d93f71bf 100644 --- a/cvat/apps/annotation/labelme.py +++ b/cvat/apps/annotation/labelme.py @@ -7,7 +7,7 @@ "dumpers": [ { "display_name": "{name} {format} {version} for images", - "format": "XML", + "format": "ZIP", "version": "3.0", "handler": "dump_as_labelme_annotation" } @@ -15,7 +15,7 @@ "loaders": [ { "display_name": "{name} {format} {version}", - "format": "XML", + "format": "ZIP", "version": "3.0", "handler": "load", } @@ -31,7 +31,7 @@ def dump_frame_anno(frame_annotation): ET.SubElement(root_elem, 'folder').text = '' source_elem = ET.SubElement(root_elem, 'source') - ET.SubElement(source_elem, 'sourceImage').text = 'task #' + ET.SubElement(source_elem, 'sourceImage').text = '' ET.SubElement(source_elem, 'sourceAnnotation').text = 'CVAT' image_elem = ET.SubElement(root_elem, 'imagesize') @@ -59,10 +59,7 @@ def dump_frame_anno(frame_annotation): ET.SubElement(obj_elem, 'type').text = 'bounding_box' poly_elem = ET.SubElement(obj_elem, 'polygon') - x0 = shape.points[0] - y0 = shape.points[1] - x1 = shape.points[2] - y1 = shape.points[4] + x0, y0, x1, y1 = shape.points points = [ (x0, y0), (x1, y0), (x1, y1), (x0, y1) ] for x, y in points: point_elem = ET.SubElement(poly_elem, 'pt') @@ -91,9 +88,9 @@ def dump_frame_anno(frame_annotation): return ET.tostring(root_elem, encoding='unicode', pretty_print=True) def dump_as_labelme_annotation(file_object, annotations): - from zipfile import ZipFile + from zipfile import ZipFile, ZIP_DEFLATED - with ZipFile(file_object, 'w') as output_zip: + with ZipFile(file_object, 'w', compression=ZIP_DEFLATED) as output_zip: for frame_annotation in annotations.group_by_frame(): xml_data = dump_frame_anno(frame_annotation) output_zip.writestr(frame_annotation.name + '.xml', xml_data) From 6e4d1d5b7a2e0a26be55ba97d049d2c0907531a5 Mon Sep 17 00:00:00 2001 From: Zhiltsov Max Date: Thu, 21 Nov 2019 17:24:53 +0300 Subject: [PATCH 3/7] Add LabelMe import --- cvat/apps/annotation/labelme.py | 238 +++++++++++++++++++++++++++++--- 1 file changed, 222 insertions(+), 16 deletions(-) diff --git a/cvat/apps/annotation/labelme.py b/cvat/apps/annotation/labelme.py index 1b58d93f71bf..05bc985cca0b 100644 --- a/cvat/apps/annotation/labelme.py +++ b/cvat/apps/annotation/labelme.py @@ -22,7 +22,13 @@ ], } + +_DEFAULT_USERNAME = 'cvat' +_MASKS_DIR = 'Masks' + + def dump_frame_anno(frame_annotation): + from collections import defaultdict from lxml import etree as ET root_elem = ET.Element('annotation') @@ -38,22 +44,24 @@ def dump_frame_anno(frame_annotation): ET.SubElement(image_elem, 'nrows').text = str(frame_annotation.height) ET.SubElement(image_elem, 'ncols').text = str(frame_annotation.width) - next_obj_id = 0 - for shape in frame_annotation.labeled_shapes: + groups = defaultdict(list) + + for obj_id, shape in enumerate(frame_annotation.labeled_shapes): obj_elem = ET.SubElement(root_elem, 'object') ET.SubElement(obj_elem, 'name').text = str(shape.label) ET.SubElement(obj_elem, 'deleted').text = '0' ET.SubElement(obj_elem, 'verified').text = '0' - ET.SubElement(obj_elem, 'occluded').text = str(int(shape.occluded)) + ET.SubElement(obj_elem, 'occluded').text = \ + 'yes' if shape.occluded else 'no' ET.SubElement(obj_elem, 'date').text = '0' - ET.SubElement(obj_elem, 'id').text = str(next_obj_id) - next_obj_id += 1 + ET.SubElement(obj_elem, 'id').text = str(obj_id) parts_elem = ET.SubElement(obj_elem, 'parts') - # TODO: handle groups - # .text = str(shape.group) - ET.SubElement(parts_elem, 'hasparts') - ET.SubElement(parts_elem, 'ispartof') + if shape.group: + groups[shape.group].append((obj_id, parts_elem)) + else: + ET.SubElement(parts_elem, 'hasparts').text = '' + ET.SubElement(parts_elem, 'ispartof').text = '' if shape.type == 'rectangle': ET.SubElement(obj_elem, 'type').text = 'bounding_box' @@ -65,12 +73,16 @@ def dump_frame_anno(frame_annotation): point_elem = ET.SubElement(poly_elem, 'pt') ET.SubElement(point_elem, 'x').text = '%.2f' % x ET.SubElement(point_elem, 'y').text = '%.2f' % y + + ET.SubElement(poly_elem, 'username').text = _DEFAULT_USERNAME elif shape.type == 'polygon': poly_elem = ET.SubElement(obj_elem, 'polygon') for x, y in zip(shape.points[::2], shape.points[1::2]): point_elem = ET.SubElement(poly_elem, 'pt') ET.SubElement(point_elem, 'x').text = '%.2f' % x ET.SubElement(point_elem, 'y').text = '%.2f' % y + + ET.SubElement(poly_elem, 'username').text = _DEFAULT_USERNAME elif shape.type == 'polyline': pass elif shape.type == 'points': @@ -78,12 +90,19 @@ def dump_frame_anno(frame_annotation): else: raise NotImplementedError("Unknown shape type '%s'" % shape.type) - attr_string = '' - for i, attr in enumerate(shape.attributes): - if i: - attr_string += ', ' - attr_string += '%s=%s' % (attr.name, attr.value) - ET.SubElement(obj_elem, 'attributes').text = attr_string + attrs = ['%s=%s' % (a.name, a.value) for a in shape.attributes] + ET.SubElement(obj_elem, 'attributes').text = ', '.join(attrs) + + for _, group in groups.items(): + leader_id, leader_parts_elem = group[0] + leader_parts = [str(o_id) for o_id, _ in group[1:]] + ET.SubElement(leader_parts_elem, 'hasparts').text = \ + ','.join(leader_parts) + ET.SubElement(leader_parts_elem, 'ispartof').text = '' + + for obj_id, parts_elem in group[1:]: + ET.SubElement(parts_elem, 'hasparts').text = '' + ET.SubElement(parts_elem, 'ispartof').text = str(leader_id) return ET.tostring(root_elem, encoding='unicode', pretty_print=True) @@ -95,5 +114,192 @@ def dump_as_labelme_annotation(file_object, annotations): xml_data = dump_frame_anno(frame_annotation) output_zip.writestr(frame_annotation.name + '.xml', xml_data) +def parse_xml_annotations(xml_data, annotations, input_zip): + from cvat.apps.annotation.coco import mask_to_polygon + from io import BytesIO + from lxml import etree as ET + import numpy as np + import os.path as osp + from PIL import Image + + def parse_attributes(attributes_string): + parsed = [] + if not attributes_string: + return parsed + + read = attributes_string.split(',') + read = [a.strip() for a in read if a.strip()] + for attr in read: + if '=' in attr: + name, value = attr.split('=', maxsplit=1) + parsed.append(annotations.Attribute(name, value)) + else: + parsed.append(annotations.Attribute(attr, '1')) + + return parsed + + + root_elem = ET.fromstring(xml_data) + + frame_number = annotations.match_frame(root_elem.find('filename').text) + + parsed_annotations = dict() + group_assignments = dict() + root_annotations = set() + for obj_elem in root_elem.iter('object'): + obj_id = int(obj_elem.find('id').text) + + ann_items = [] + + attributes = [] + attributes_elem = obj_elem.find('attributes') + if attributes_elem is not None and attributes_elem.text: + attributes = parse_attributes(attributes_elem.text) + + occluded = False + occluded_elem = obj_elem.find('occluded') + if occluded_elem is not None and occluded_elem.text: + occluded = (occluded_elem.text == 'yes') + + deleted = False + deleted_elem = obj_elem.find('deleted') + if deleted_elem is not None and deleted_elem.text: + deleted = bool(int(deleted_elem.text)) + + poly_elem = obj_elem.find('polygon') + segm_elem = obj_elem.find('segm') + type_elem = obj_elem.find('type') # the only value is 'bounding_box' + if poly_elem is not None: + points = [] + for point_elem in poly_elem.iter('pt'): + x = float(point_elem.find('x').text) + y = float(point_elem.find('y').text) + points.append(x) + points.append(y) + label = obj_elem.find('name').text + if label and attributes: + label_id = annotations._get_label_id(label) + if label_id: + attributes = [a for a in attributes + if annotations._get_attribute_id(label_id, a.name) + ] + else: + attributes = [] + else: + attributes = [] + + if type_elem is not None and type_elem.text == 'bounding_box': + xmin = min(points[::2]) + xmax = max(points[::2]) + ymin = min(points[1::2]) + ymax = max(points[1::2]) + ann_items.append(annotations.LabeledShape( + type='rectangle', + frame=frame_number, + label=label, + points=[xmin, ymin, xmax, ymax], + occluded=occluded, + attributes=attributes, + )) + else: + ann_items.append(annotations.LabeledShape( + type='polygon', + frame=frame_number, + label=label, + points=points, + occluded=occluded, + attributes=attributes, + )) + elif segm_elem is not None: + label = obj_elem.find('name').text + if label and attributes: + label_id = annotations._get_label_id(label) + if label_id: + attributes = [a for a in attributes + if annotations._get_attribute_id(label_id, a.name) + ] + else: + attributes = [] + else: + attributes = [] + + mask_file = segm_elem.find('mask').text + mask = input_zip.read(osp.join(_MASKS_DIR, mask_file)) + mask = np.asarray(Image.open(BytesIO(mask)).convert('L')) + mask = (mask != 0) + polygons = mask_to_polygon(mask) + + for polygon in polygons: + ann_items.append(annotations.LabeledShape( + type='polygon', + frame=frame_number, + label=label, + points=polygon, + occluded=occluded, + attributes=attributes, + )) + + if not deleted: + parsed_annotations[obj_id] = ann_items + + parts_elem = obj_elem.find('parts') + if parts_elem is not None: + children_ids = [] + hasparts_elem = parts_elem.find('hasparts') + if hasparts_elem is not None and hasparts_elem.text: + children_ids = [int(c) for c in hasparts_elem.text.split(',')] + + parent_ids = [] + ispartof_elem = parts_elem.find('ispartof') + if ispartof_elem is not None and ispartof_elem.text: + parent_ids = [int(c) for c in ispartof_elem.text.split(',')] + + if children_ids and not parent_ids and hasparts_elem.text: + root_annotations.add(obj_id) + group_assignments[obj_id] = [None, children_ids] + + # assign a single group to the whole subtree + current_group_id = 0 + annotations_to_visit = list(root_annotations) + while annotations_to_visit: + ann_id = annotations_to_visit.pop() + ann_assignment = group_assignments[ann_id] + group_id, children_ids = ann_assignment + if group_id: + continue + + if ann_id in root_annotations: + current_group_id += 1 # start a new group + + group_id = current_group_id + ann_assignment[0] = group_id + + # continue with children + annotations_to_visit.extend(children_ids) + + assert current_group_id == len(root_annotations) + + for ann_id, ann_items in parsed_annotations.items(): + group_id = 0 + if ann_id in group_assignments: + ann_assignment = group_assignments[ann_id] + group_id = ann_assignment[0] + + for ann_item in ann_items: + if group_id: + ann_item = ann_item._replace(group=group_id) + if isinstance(ann_item, annotations.LabeledShape): + annotations.add_shape(ann_item) + else: + raise NotImplementedError() + def load(file_object, annotations): - pass \ No newline at end of file + from zipfile import ZipFile, ZIP_DEFLATED + + with ZipFile(file_object, 'r') as input_zip: + for filename in input_zip.namelist(): + if not filename.endswith('.xml'): + continue + + xml_data = input_zip.read(filename) + parse_xml_annotations(xml_data, annotations, input_zip) From 7b0df981287076fcca74725b7bb75135270292a3 Mon Sep 17 00:00:00 2001 From: Zhiltsov Max Date: Thu, 21 Nov 2019 17:35:44 +0300 Subject: [PATCH 4/7] Codacy --- cvat/apps/annotation/labelme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/annotation/labelme.py b/cvat/apps/annotation/labelme.py index 05bc985cca0b..58c6354dab15 100644 --- a/cvat/apps/annotation/labelme.py +++ b/cvat/apps/annotation/labelme.py @@ -294,7 +294,7 @@ def parse_attributes(attributes_string): raise NotImplementedError() def load(file_object, annotations): - from zipfile import ZipFile, ZIP_DEFLATED + from zipfile import ZipFile with ZipFile(file_object, 'r') as input_zip: for filename in input_zip.namelist(): From fb4c87ffe2c30e8853f4e397502a1b92b22c101e Mon Sep 17 00:00:00 2001 From: Zhiltsov Max Date: Thu, 21 Nov 2019 19:55:22 +0300 Subject: [PATCH 5/7] Add labelme format to readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 998c288b198c..9bacba82e981 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ Format selection is possible after clicking on the Upload annotation / Dump anno | [MS COCO Object Detection](http://cocodataset.org/#format-data) | X | X | | PNG mask | X | | | [TFrecord](https://www.tensorflow.org/tutorials/load_data/tf_records) | X | X | +| [LabelMe](http://labelme.csail.mit.edu/Release3.0) | X | X | ## Links - [Intel AI blog: New Computer Vision Tool Accelerates Annotation of Digital Images and Video](https://www.intel.ai/introducing-cvat) From 6a088f290c64abab0b7ef4e2c8141c2b1dd3a9dd Mon Sep 17 00:00:00 2001 From: Zhiltsov Max Date: Fri, 22 Nov 2019 11:55:38 +0300 Subject: [PATCH 6/7] Fix filenames and dates --- cvat/apps/annotation/labelme.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cvat/apps/annotation/labelme.py b/cvat/apps/annotation/labelme.py index 58c6354dab15..0128ca739226 100644 --- a/cvat/apps/annotation/labelme.py +++ b/cvat/apps/annotation/labelme.py @@ -53,7 +53,7 @@ def dump_frame_anno(frame_annotation): ET.SubElement(obj_elem, 'verified').text = '0' ET.SubElement(obj_elem, 'occluded').text = \ 'yes' if shape.occluded else 'no' - ET.SubElement(obj_elem, 'date').text = '0' + ET.SubElement(obj_elem, 'date').text = '' ET.SubElement(obj_elem, 'id').text = str(obj_id) parts_elem = ET.SubElement(obj_elem, 'parts') @@ -112,7 +112,9 @@ def dump_as_labelme_annotation(file_object, annotations): with ZipFile(file_object, 'w', compression=ZIP_DEFLATED) as output_zip: for frame_annotation in annotations.group_by_frame(): xml_data = dump_frame_anno(frame_annotation) - output_zip.writestr(frame_annotation.name + '.xml', xml_data) + filename = frame_annotation.name + filename = filename[ : filename.rfind('.')] + '.xml' + output_zip.writestr(filename, xml_data) def parse_xml_annotations(xml_data, annotations, input_zip): from cvat.apps.annotation.coco import mask_to_polygon From a100da18c56c9d8e61d1baf20453bb42717299d3 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Fri, 22 Nov 2019 16:21:07 +0300 Subject: [PATCH 7/7] Updated CHANGELOG.md --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfcfb7c53e6f..04cc754b3f92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Ability to [get basic information about users without admin permissions]( https://github.com/opencv/cvat/issues/750). - Changed REST API: removed PUT and added DELETE methods for /api/v1/users/ID. -- Added Mask-RCNN Auto Annotation Script in OpenVINO format -- Added Yolo Auto Annotation Script +- Mask-RCNN Auto Annotation Script in OpenVINO format +- Yolo Auto Annotation Script - Auto segmentation using Mask_RCNN component (Keras+Tensorflow Mask R-CNN Segmentation) +- Ability to dump/load annotations in LabelMe format from UI ### Changed -