From 2d3db8eec0198b6314cdfc49fdf2348206a822e9 Mon Sep 17 00:00:00 2001 From: sooahleex Date: Mon, 24 Oct 2022 19:50:42 +0900 Subject: [PATCH 1/8] Add jupyternotebook for filter --- notebooks/04_filter.ipynb | 1004 +++++++++++++++++++++++++++++++++++++ 1 file changed, 1004 insertions(+) create mode 100644 notebooks/04_filter.ipynb diff --git a/notebooks/04_filter.ipynb b/notebooks/04_filter.ipynb new file mode 100644 index 0000000000..ea463f2549 --- /dev/null +++ b/notebooks/04_filter.ipynb @@ -0,0 +1,1004 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "652f6b58", + "metadata": {}, + "source": [ + "# Filter datasets" + ] + }, + { + "cell_type": "markdown", + "id": "ca821a19", + "metadata": {}, + "source": [ + "In this notebook example, we'll take a look at Datumaro filter api." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "da198c67", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import datumaro as dm\n", + "from datumaro.components.operations import compute_image_statistics, compute_ann_statistics" + ] + }, + { + "cell_type": "markdown", + "id": "9e2cf885", + "metadata": {}, + "source": [ + "### Filtered by subset" + ] + }, + { + "cell_type": "markdown", + "id": "031f1d62", + "metadata": {}, + "source": [ + "We export sample VOC dataset to filter only train subset." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b9640838", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = dm.Dataset.import_from('./tests/assets/voc_dataset/voc_dataset1', format='voc')" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "d38cfc9b", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "statistics for a sample VOC dataset\n" + ] + }, + { + "data": { + "text/plain": [ + "{'dataset': {'images count': 2,\n", + " 'unique images count': 1,\n", + " 'repeated images count': 1,\n", + " 'repeated images': [[('2007_000001', 'train'), ('2007_000002', 'test')]]},\n", + " 'subsets': {'test': {'images count': 1,\n", + " 'image mean': [0.9999999999999971, 0.9999999999999971, 0.9999999999999971],\n", + " 'image std': [9.411065220006367e-08,\n", + " 9.411065220006367e-08,\n", + " 9.411065220006367e-08]},\n", + " 'train': {'images count': 1,\n", + " 'image mean': [0.9999999999999971, 0.9999999999999971, 0.9999999999999971],\n", + " 'image std': [9.411065220006367e-08,\n", + " 9.411065220006367e-08,\n", + " 9.411065220006367e-08]}}}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('statistics for a sample VOC dataset')\n", + "compute_image_statistics(dataset)" + ] + }, + { + "cell_type": "markdown", + "id": "db0e0346", + "metadata": {}, + "source": [ + "In VOC dataset, there are 'train' and 'test' subset. We will filter only 'train' subset." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "51bf3388", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dm.Dataset.filter(dataset, '/item[subset=\"train\"]')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "fb608396", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "statistics for train subset VOC dataset\n" + ] + }, + { + "data": { + "text/plain": [ + "{'dataset': {'images count': 1,\n", + " 'unique images count': 1,\n", + " 'repeated images count': 0,\n", + " 'repeated images': []},\n", + " 'subsets': {'train': {'images count': 1,\n", + " 'image mean': [0.9999999999999971, 0.9999999999999971, 0.9999999999999971],\n", + " 'image std': [9.411065220006367e-08,\n", + " 9.411065220006367e-08,\n", + " 9.411065220006367e-08]}}}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('statistics for train subset VOC dataset')\n", + "compute_image_statistics(dataset)" + ] + }, + { + "cell_type": "markdown", + "id": "cef29383", + "metadata": {}, + "source": [ + "### Filtered by id" + ] + }, + { + "cell_type": "markdown", + "id": "e3807aff", + "metadata": {}, + "source": [ + "We export sample widerface dataset to filter only dataset which id is 1." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "8993b993", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = dm.Dataset.import_from('./tests/assets/widerface_dataset')" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "11362daf", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "statistics for a sample WiderFace dataset\n" + ] + }, + { + "data": { + "text/plain": [ + "{'dataset': {'images count': 3,\n", + " 'unique images count': 1,\n", + " 'repeated images count': 1,\n", + " 'repeated images': [[('0_Parade_image_01', 'train'),\n", + " ('0_Parade_image_03', 'val'),\n", + " ('1_Handshaking_image_02', 'train')]]},\n", + " 'subsets': {'val': {'images count': 1,\n", + " 'image mean': [0.9999999999999973, 0.9999999999999973, 0.9999999999999973],\n", + " 'image std': [9.058862863930295e-08,\n", + " 9.058862863930295e-08,\n", + " 9.058862863930295e-08]},\n", + " 'train': {'images count': 2,\n", + " 'image mean': [0.9999999999999973, 0.9999999999999973, 0.9999999999999973],\n", + " 'image std': [9.043701576544718e-08,\n", + " 9.043701576544718e-08,\n", + " 9.043701576544718e-08]}}}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('statistics for a sample WiderFace dataset')\n", + "compute_image_statistics(dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "4ed4a847", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dm.Dataset.filter(dataset, '/item[id=\"0_Parade_image_01\"]')" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "d689f5bc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "statistics for WiderFace dataset id == 1\n" + ] + }, + { + "data": { + "text/plain": [ + "{'dataset': {'images count': 1,\n", + " 'unique images count': 1,\n", + " 'repeated images count': 0,\n", + " 'repeated images': []},\n", + " 'subsets': {'train': {'images count': 1,\n", + " 'image mean': [0.9999999999999973, 0.9999999999999973, 0.9999999999999973],\n", + " 'image std': [9.058862863930295e-08,\n", + " 9.058862863930295e-08,\n", + " 9.058862863930295e-08]}}}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('statistics for WiderFace dataset id == 1')\n", + "compute_image_statistics(dataset)" + ] + }, + { + "cell_type": "markdown", + "id": "9b23d1af", + "metadata": {}, + "source": [ + "### Filtered by width and height" + ] + }, + { + "cell_type": "markdown", + "id": "6d6ca87d", + "metadata": {}, + "source": [ + "We export sample dataset to extract a dataset with images with width < height." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "e5e30736", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:Not implemented: Found potentially conflicting source types with labels: person_keypoints, labels, instances, panoptic, stuff. Only one type will be used: instances\n", + "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/labels_train.json' is skipped.\n", + "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/person_keypoints_train.json' is skipped.\n", + "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/stuff_train.json' is skipped.\n", + "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/panoptic_train.json' is skipped.\n", + "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/panoptic_val.json' is skipped.\n", + "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/labels_val.json' is skipped.\n", + "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/person_keypoints_val.json' is skipped.\n", + "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/stuff_val.json' is skipped.\n" + ] + } + ], + "source": [ + "dataset = dm.Dataset.import_from('./tests/assets/coco_dataset/coco')" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "3f8953e3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "statistics for a sample coco dataset\n" + ] + }, + { + "data": { + "text/plain": [ + "{'dataset': {'images count': 2,\n", + " 'unique images count': 1,\n", + " 'repeated images count': 1,\n", + " 'repeated images': [[('a', 'train'), ('b', 'val')]]},\n", + " 'subsets': {'train': {'images count': 1,\n", + " 'image mean': [0.9999999999999987, 0.9999999999999987, 0.9999999999999987],\n", + " 'image std': [6.361265799828938e-08,\n", + " 6.361265799828938e-08,\n", + " 6.361265799828938e-08]},\n", + " 'val': {'images count': 1,\n", + " 'image mean': [0.9999999999999987, 0.9999999999999987, 0.9999999999999987],\n", + " 'image std': [6.361265799828938e-08,\n", + " 6.361265799828938e-08,\n", + " 6.361265799828938e-08]}}}" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('statistics for a sample coco dataset')\n", + "compute_image_statistics(dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "7b693479", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dm.Dataset.filter(dataset, '/item[image/width < image/height]')" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "7479f9c9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "statistics for width < height coco dataset\n" + ] + }, + { + "data": { + "text/plain": [ + "{'dataset': {'images count': 1,\n", + " 'unique images count': 1,\n", + " 'repeated images count': 0,\n", + " 'repeated images': []},\n", + " 'subsets': {'val': {'images count': 1,\n", + " 'image mean': [0.9999999999999987, 0.9999999999999987, 0.9999999999999987],\n", + " 'image std': [6.361265799828938e-08,\n", + " 6.361265799828938e-08,\n", + " 6.361265799828938e-08]}}}" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('statistics for width < height coco dataset')\n", + "compute_image_statistics(dataset)" + ] + }, + { + "cell_type": "markdown", + "id": "cbde6b17", + "metadata": {}, + "source": [ + "### Filtered by label and area" + ] + }, + { + "cell_type": "markdown", + "id": "1e0dc1de", + "metadata": {}, + "source": [ + "We export sample dataset to extract only non-`persons`." + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "0c8248a9", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = dm.Dataset.import_from('./tests/assets/voc_dataset/voc_dataset1')" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "98ad83ad", + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "annotation count for coco dataset\n" + ] + }, + { + "data": { + "text/plain": [ + "15" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('annotation count for voc dataset')\n", + "compute_ann_statistics(dataset)['annotations count']" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "6d74e4cd", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'labels': {'count': 15,\n", + " 'distribution': {'background': [0, 0.0],\n", + " 'aeroplane': [1, 0.06666666666666667],\n", + " 'bicycle': [1, 0.06666666666666667],\n", + " 'bird': [1, 0.06666666666666667],\n", + " 'boat': [0, 0.0],\n", + " 'bottle': [1, 0.06666666666666667],\n", + " 'bus': [0, 0.0],\n", + " 'car': [1, 0.06666666666666667],\n", + " 'cat': [1, 0.06666666666666667],\n", + " 'chair': [1, 0.06666666666666667],\n", + " 'cow': [0, 0.0],\n", + " 'diningtable': [1, 0.06666666666666667],\n", + " 'dog': [0, 0.0],\n", + " 'horse': [1, 0.06666666666666667],\n", + " 'motorbike': [0, 0.0],\n", + " 'person': [2, 0.13333333333333333],\n", + " 'pottedplant': [0, 0.0],\n", + " 'sheep': [1, 0.06666666666666667],\n", + " 'sofa': [0, 0.0],\n", + " 'train': [1, 0.06666666666666667],\n", + " 'tvmonitor': [0, 0.0],\n", + " 'ignored': [1, 0.06666666666666667],\n", + " 'head': [1, 0.06666666666666667],\n", + " 'hand': [0, 0.0],\n", + " 'foot': [0, 0.0]},\n", + " 'attributes': {'difficult': {'count': 2,\n", + " 'values count': 1,\n", + " 'values present': ['False'],\n", + " 'distribution': {'False': [2, 1.0]}},\n", + " 'truncated': {'count': 2,\n", + " 'values count': 2,\n", + " 'values present': ['False', 'True'],\n", + " 'distribution': {'False': [1, 0.5], 'True': [1, 0.5]}},\n", + " 'jumping': {'count': 1,\n", + " 'values count': 1,\n", + " 'values present': ['False'],\n", + " 'distribution': {'False': [1, 1.0]}},\n", + " 'playinginstrument': {'count': 1,\n", + " 'values count': 1,\n", + " 'values present': ['False'],\n", + " 'distribution': {'False': [1, 1.0]}},\n", + " 'reading': {'count': 1,\n", + " 'values count': 1,\n", + " 'values present': ['True'],\n", + " 'distribution': {'True': [1, 1.0]}},\n", + " 'ridingbike': {'count': 1,\n", + " 'values count': 1,\n", + " 'values present': ['False'],\n", + " 'distribution': {'False': [1, 1.0]}},\n", + " 'takingphoto': {'count': 1,\n", + " 'values count': 1,\n", + " 'values present': ['True'],\n", + " 'distribution': {'True': [1, 1.0]}},\n", + " 'usingcomputer': {'count': 1,\n", + " 'values count': 1,\n", + " 'values present': ['False'],\n", + " 'distribution': {'False': [1, 1.0]}},\n", + " 'walking': {'count': 1,\n", + " 'values count': 1,\n", + " 'values present': ['True'],\n", + " 'distribution': {'True': [1, 1.0]}},\n", + " 'running': {'count': 1,\n", + " 'values count': 1,\n", + " 'values present': ['False'],\n", + " 'distribution': {'False': [1, 1.0]}},\n", + " 'other': {'count': 1,\n", + " 'values count': 1,\n", + " 'values present': ['True'],\n", + " 'distribution': {'True': [1, 1.0]}},\n", + " 'phoning': {'count': 1,\n", + " 'values count': 1,\n", + " 'values present': ['True'],\n", + " 'distribution': {'True': [1, 1.0]}},\n", + " 'ridinghorse': {'count': 1,\n", + " 'values count': 1,\n", + " 'values present': ['True'],\n", + " 'distribution': {'True': [1, 1.0]}},\n", + " 'pose': {'count': 1,\n", + " 'values count': 1,\n", + " 'values present': ['Unspecified'],\n", + " 'distribution': {'Unspecified': [1, 1.0]}}}},\n", + " 'segments': {'avg. area': 53.0,\n", + " 'area distribution': [{'min': 4.0, 'max': 23.6, 'count': 3, 'percent': 0.75},\n", + " {'min': 23.6, 'max': 43.2, 'count': 0, 'percent': 0.0},\n", + " {'min': 43.2, 'max': 62.800000000000004, 'count': 0, 'percent': 0.0},\n", + " {'min': 62.800000000000004, 'max': 82.4, 'count': 0, 'percent': 0.0},\n", + " {'min': 82.4, 'max': 102.0, 'count': 0, 'percent': 0.0},\n", + " {'min': 102.0, 'max': 121.60000000000001, 'count': 0, 'percent': 0.0},\n", + " {'min': 121.60000000000001,\n", + " 'max': 141.20000000000002,\n", + " 'count': 0,\n", + " 'percent': 0.0},\n", + " {'min': 141.20000000000002, 'max': 160.8, 'count': 0, 'percent': 0.0},\n", + " {'min': 160.8, 'max': 180.4, 'count': 0, 'percent': 0.0},\n", + " {'min': 180.4, 'max': 200.0, 'count': 1, 'percent': 0.25}],\n", + " 'pixel distribution': {'background': [0, 0.0],\n", + " 'aeroplane': [0, 0.0],\n", + " 'bicycle': [200, 0.9433962264150944],\n", + " 'bird': [0, 0.0],\n", + " 'boat': [0, 0.0],\n", + " 'bottle': [0, 0.0],\n", + " 'bus': [0, 0.0],\n", + " 'car': [0, 0.0],\n", + " 'cat': [4, 0.018867924528301886],\n", + " 'chair': [0, 0.0],\n", + " 'cow': [0, 0.0],\n", + " 'diningtable': [0, 0.0],\n", + " 'dog': [0, 0.0],\n", + " 'horse': [0, 0.0],\n", + " 'motorbike': [0, 0.0],\n", + " 'person': [4, 0.018867924528301886],\n", + " 'pottedplant': [0, 0.0],\n", + " 'sheep': [0, 0.0],\n", + " 'sofa': [0, 0.0],\n", + " 'train': [0, 0.0],\n", + " 'tvmonitor': [0, 0.0],\n", + " 'ignored': [0, 0.0],\n", + " 'head': [4, 0.018867924528301886],\n", + " 'hand': [0, 0.0],\n", + " 'foot': [0, 0.0]}}}" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('annotation statistics for voc dataset whose annotation is label!=\"person\"')\n", + "compute_ann_statistics(dataset)['annotations']" + ] + }, + { + "cell_type": "markdown", + "id": "a3c88c99", + "metadata": {}, + "source": [ + "Indicate `filter_annotations` as `True` if filter needs to apply to annotations. The default value is `False` to items." + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "8dda4168", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dm.Dataset.filter(dataset, '/item/annotation[label!=\"person\"]', filter_annotations=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "67f0b7e9", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "annotation count for coco dataset whose annotation is (label=\"cat\" and area < 9643950) or label!=\"person\"\n" + ] + }, + { + "data": { + "text/plain": [ + "13" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('annotation count for voc dataset whose annotation is label!=\"person\"')\n", + "compute_ann_statistics(dataset)['annotations count']" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "506ae22a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'labels': {'count': 13,\n", + " 'distribution': {'background': [0, 0.0],\n", + " 'aeroplane': [1, 0.07692307692307693],\n", + " 'bicycle': [1, 0.07692307692307693],\n", + " 'bird': [1, 0.07692307692307693],\n", + " 'boat': [0, 0.0],\n", + " 'bottle': [1, 0.07692307692307693],\n", + " 'bus': [0, 0.0],\n", + " 'car': [1, 0.07692307692307693],\n", + " 'cat': [1, 0.07692307692307693],\n", + " 'chair': [1, 0.07692307692307693],\n", + " 'cow': [0, 0.0],\n", + " 'diningtable': [1, 0.07692307692307693],\n", + " 'dog': [0, 0.0],\n", + " 'horse': [1, 0.07692307692307693],\n", + " 'motorbike': [0, 0.0],\n", + " 'person': [0, 0.0],\n", + " 'pottedplant': [0, 0.0],\n", + " 'sheep': [1, 0.07692307692307693],\n", + " 'sofa': [0, 0.0],\n", + " 'train': [1, 0.07692307692307693],\n", + " 'tvmonitor': [0, 0.0],\n", + " 'ignored': [1, 0.07692307692307693],\n", + " 'head': [1, 0.07692307692307693],\n", + " 'hand': [0, 0.0],\n", + " 'foot': [0, 0.0]},\n", + " 'attributes': {'difficult': {'count': 1,\n", + " 'values count': 1,\n", + " 'values present': ['False'],\n", + " 'distribution': {'False': [1, 1.0]}},\n", + " 'truncated': {'count': 1,\n", + " 'values count': 1,\n", + " 'values present': ['True'],\n", + " 'distribution': {'True': [1, 1.0]}},\n", + " 'pose': {'count': 1,\n", + " 'values count': 1,\n", + " 'values present': ['Unspecified'],\n", + " 'distribution': {'Unspecified': [1, 1.0]}}}},\n", + " 'segments': {'avg. area': 69.33333333333333,\n", + " 'area distribution': [{'min': 4.0,\n", + " 'max': 23.6,\n", + " 'count': 2,\n", + " 'percent': 0.6666666666666666},\n", + " {'min': 23.6, 'max': 43.2, 'count': 0, 'percent': 0.0},\n", + " {'min': 43.2, 'max': 62.800000000000004, 'count': 0, 'percent': 0.0},\n", + " {'min': 62.800000000000004, 'max': 82.4, 'count': 0, 'percent': 0.0},\n", + " {'min': 82.4, 'max': 102.0, 'count': 0, 'percent': 0.0},\n", + " {'min': 102.0, 'max': 121.60000000000001, 'count': 0, 'percent': 0.0},\n", + " {'min': 121.60000000000001,\n", + " 'max': 141.20000000000002,\n", + " 'count': 0,\n", + " 'percent': 0.0},\n", + " {'min': 141.20000000000002, 'max': 160.8, 'count': 0, 'percent': 0.0},\n", + " {'min': 160.8, 'max': 180.4, 'count': 0, 'percent': 0.0},\n", + " {'min': 180.4, 'max': 200.0, 'count': 1, 'percent': 0.3333333333333333}],\n", + " 'pixel distribution': {'background': [0, 0.0],\n", + " 'aeroplane': [0, 0.0],\n", + " 'bicycle': [200, 0.9615384615384616],\n", + " 'bird': [0, 0.0],\n", + " 'boat': [0, 0.0],\n", + " 'bottle': [0, 0.0],\n", + " 'bus': [0, 0.0],\n", + " 'car': [0, 0.0],\n", + " 'cat': [4, 0.019230769230769232],\n", + " 'chair': [0, 0.0],\n", + " 'cow': [0, 0.0],\n", + " 'diningtable': [0, 0.0],\n", + " 'dog': [0, 0.0],\n", + " 'horse': [0, 0.0],\n", + " 'motorbike': [0, 0.0],\n", + " 'person': [0, 0.0],\n", + " 'pottedplant': [0, 0.0],\n", + " 'sheep': [0, 0.0],\n", + " 'sofa': [0, 0.0],\n", + " 'train': [0, 0.0],\n", + " 'tvmonitor': [0, 0.0],\n", + " 'ignored': [0, 0.0],\n", + " 'head': [4, 0.019230769230769232],\n", + " 'hand': [0, 0.0],\n", + " 'foot': [0, 0.0]}}}" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "compute_ann_statistics(dataset)['annotations']" + ] + }, + { + "cell_type": "markdown", + "id": "3201198f", + "metadata": {}, + "source": [ + "### Filtered by annotation" + ] + }, + { + "cell_type": "markdown", + "id": "ee1e403a", + "metadata": {}, + "source": [ + "We export sample dataset to extract non-occluded annotations, remove empty images. Use data only from the “s1” source of the project." + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "id": "732face4", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = dm.Dataset.import_from('./tests/assets/voc_dataset/voc_dataset1')" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "id": "7badb290", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "image statistics for sample voc dataset\n" + ] + }, + { + "data": { + "text/plain": [ + "{'dataset': {'images count': 2,\n", + " 'unique images count': 1,\n", + " 'repeated images count': 1,\n", + " 'repeated images': [[('2007_000001', 'train'), ('2007_000002', 'test')]]},\n", + " 'subsets': {'test': {'images count': 1,\n", + " 'image mean': [0.9999999999999971, 0.9999999999999971, 0.9999999999999971],\n", + " 'image std': [9.411065220006367e-08,\n", + " 9.411065220006367e-08,\n", + " 9.411065220006367e-08]},\n", + " 'train': {'images count': 1,\n", + " 'image mean': [0.9999999999999971, 0.9999999999999971, 0.9999999999999971],\n", + " 'image std': [9.411065220006367e-08,\n", + " 9.411065220006367e-08,\n", + " 9.411065220006367e-08]}}}" + ] + }, + "execution_count": 72, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('image statistics for sample voc dataset')\n", + "compute_image_statistics(dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "id": "326af8ad", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "annotation statistics for sample voc dataset\n" + ] + }, + { + "data": { + "text/plain": [ + "15" + ] + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('annotation statistics for sample voc dataset')\n", + "compute_ann_statistics(dataset)['annotations count']" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "id": "f8ddf924", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 74, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dm.Dataset.filter(dataset, '/item/annotation[occluded=\"False\"]', filter_annotations=True, remove_empty=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "id": "3a971546", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "image statistics for non-occluded annotations and empty images removed voc dataset\n" + ] + }, + { + "data": { + "text/plain": [ + "{'dataset': {'images count': 1,\n", + " 'unique images count': 1,\n", + " 'repeated images count': 0,\n", + " 'repeated images': []},\n", + " 'subsets': {'train': {'images count': 1,\n", + " 'image mean': [0.9999999999999971, 0.9999999999999971, 0.9999999999999971],\n", + " 'image std': [9.411065220006367e-08,\n", + " 9.411065220006367e-08,\n", + " 9.411065220006367e-08]}}}" + ] + }, + "execution_count": 75, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('image statistics for non-occluded annotations and empty images removed voc dataset')\n", + "compute_image_statistics(dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "id": "f66b97ac", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "annotation statistics for non-occluded annotations and empty images removed voc dataset\n" + ] + }, + { + "data": { + "text/plain": [ + "2" + ] + }, + "execution_count": 76, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('annotation statistics for non-occluded annotations and empty images removed voc dataset')\n", + "compute_ann_statistics(dataset)['annotations count']" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 4cb9dc9a62c5e0a02b70a740482096b49c836a9b Mon Sep 17 00:00:00 2001 From: sooahleex Date: Mon, 24 Oct 2022 21:49:32 +0900 Subject: [PATCH 2/8] Update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bde225706e..1de81846f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 () - Add jupyter sample introducing how to merge datasets () + - introducing how to filter dataset + () - Add Visualization Python API - Bbox () From 7cd10e8b6ccbb5b880db06491197070cffe397fb Mon Sep 17 00:00:00 2001 From: sooahleex Date: Tue, 25 Oct 2022 21:53:19 +0900 Subject: [PATCH 3/8] Update get_width_height --- notebooks/04_filter.ipynb | 204 +++++++++++++++++++------------------- 1 file changed, 104 insertions(+), 100 deletions(-) diff --git a/notebooks/04_filter.ipynb b/notebooks/04_filter.ipynb index ea463f2549..35863fcf4d 100644 --- a/notebooks/04_filter.ipynb +++ b/notebooks/04_filter.ipynb @@ -18,11 +18,15 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 29, "id": "da198c67", "metadata": {}, "outputs": [], "source": [ + "# Copyright (C) 2022 Intel Corporation\n", + "#\n", + "# SPDX-License-Identifier: MIT\n", + "\n", "import os\n", "import datumaro as dm\n", "from datumaro.components.operations import compute_image_statistics, compute_ann_statistics" @@ -46,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "b9640838", "metadata": {}, "outputs": [], @@ -56,7 +60,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "d38cfc9b", "metadata": { "scrolled": true @@ -88,7 +92,7 @@ " 9.411065220006367e-08]}}}" ] }, - "execution_count": 4, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -108,17 +112,17 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "51bf3388", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 5, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -129,7 +133,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "id": "fb608396", "metadata": { "scrolled": true @@ -156,7 +160,7 @@ " 9.411065220006367e-08]}}}" ] }, - "execution_count": 6, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -179,12 +183,12 @@ "id": "e3807aff", "metadata": {}, "source": [ - "We export sample widerface dataset to filter only dataset which id is 1." + "We export sample widerface dataset to filter only dataset which id is `id=0_Parade_image_01`." ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "id": "8993b993", "metadata": {}, "outputs": [], @@ -194,7 +198,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "id": "11362daf", "metadata": {}, "outputs": [ @@ -226,7 +230,7 @@ " 9.043701576544718e-08]}}}" ] }, - "execution_count": 8, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -238,17 +242,17 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "id": "4ed4a847", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 9, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -259,7 +263,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "id": "d689f5bc", "metadata": {}, "outputs": [ @@ -284,7 +288,7 @@ " 9.058862863930295e-08]}}}" ] }, - "execution_count": 10, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -312,7 +316,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 2, "id": "e5e30736", "metadata": {}, "outputs": [ @@ -320,7 +324,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "WARNING:root:Not implemented: Found potentially conflicting source types with labels: person_keypoints, labels, instances, panoptic, stuff. Only one type will be used: instances\n", + "WARNING:root:Not implemented: Found potentially conflicting source types with labels: labels, instances, person_keypoints, stuff, panoptic. Only one type will be used: instances\n", "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/labels_train.json' is skipped.\n", "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/person_keypoints_train.json' is skipped.\n", "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/stuff_train.json' is skipped.\n", @@ -338,59 +342,60 @@ }, { "cell_type": "code", - "execution_count": 12, - "id": "3f8953e3", + "execution_count": 24, + "id": "87feaf56", + "metadata": {}, + "outputs": [], + "source": [ + "def get_width_height(dataset: dm.Dataset):\n", + " size_dict = {}\n", + " for item in dataset:\n", + " size_dict[item.id] = item.media.size\n", + " return size_dict" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "b821387a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "statistics for a sample coco dataset\n" + "width and height for a sample coco dataset images\n" ] }, { "data": { "text/plain": [ - "{'dataset': {'images count': 2,\n", - " 'unique images count': 1,\n", - " 'repeated images count': 1,\n", - " 'repeated images': [[('a', 'train'), ('b', 'val')]]},\n", - " 'subsets': {'train': {'images count': 1,\n", - " 'image mean': [0.9999999999999987, 0.9999999999999987, 0.9999999999999987],\n", - " 'image std': [6.361265799828938e-08,\n", - " 6.361265799828938e-08,\n", - " 6.361265799828938e-08]},\n", - " 'val': {'images count': 1,\n", - " 'image mean': [0.9999999999999987, 0.9999999999999987, 0.9999999999999987],\n", - " 'image std': [6.361265799828938e-08,\n", - " 6.361265799828938e-08,\n", - " 6.361265799828938e-08]}}}" + "{'a': (5, 10), 'b': (10, 5)}" ] }, - "execution_count": 12, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "print('statistics for a sample coco dataset')\n", - "compute_image_statistics(dataset)" + "print('width and height for a sample coco dataset images')\n", + "get_width_height(dataset)" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 27, "id": "7b693479", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 13, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -401,7 +406,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 28, "id": "7479f9c9", "metadata": {}, "outputs": [ @@ -409,31 +414,23 @@ "name": "stdout", "output_type": "stream", "text": [ - "statistics for width < height coco dataset\n" + "width and height for width < height coco dataset images\n" ] }, { "data": { "text/plain": [ - "{'dataset': {'images count': 1,\n", - " 'unique images count': 1,\n", - " 'repeated images count': 0,\n", - " 'repeated images': []},\n", - " 'subsets': {'val': {'images count': 1,\n", - " 'image mean': [0.9999999999999987, 0.9999999999999987, 0.9999999999999987],\n", - " 'image std': [6.361265799828938e-08,\n", - " 6.361265799828938e-08,\n", - " 6.361265799828938e-08]}}}" + "{'b': (10, 5)}" ] }, - "execution_count": 14, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "print('statistics for width < height coco dataset')\n", - "compute_image_statistics(dataset)" + "print('width and height for width < height coco dataset images')\n", + "get_width_height(dataset)" ] }, { @@ -454,7 +451,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 14, "id": "0c8248a9", "metadata": {}, "outputs": [], @@ -464,7 +461,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 15, "id": "98ad83ad", "metadata": { "scrolled": false @@ -474,7 +471,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "annotation count for coco dataset\n" + "annotation count for voc dataset\n" ] }, { @@ -483,7 +480,7 @@ "15" ] }, - "execution_count": 28, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -495,10 +492,17 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 16, "id": "6d74e4cd", "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "annotation statistics for voc dataset whose annotation is label!=\"person\"\n" + ] + }, { "data": { "text/plain": [ @@ -536,31 +540,27 @@ " 'values count': 2,\n", " 'values present': ['False', 'True'],\n", " 'distribution': {'False': [1, 0.5], 'True': [1, 0.5]}},\n", - " 'jumping': {'count': 1,\n", + " 'ridinghorse': {'count': 1,\n", " 'values count': 1,\n", - " 'values present': ['False'],\n", - " 'distribution': {'False': [1, 1.0]}},\n", - " 'playinginstrument': {'count': 1,\n", + " 'values present': ['True'],\n", + " 'distribution': {'True': [1, 1.0]}},\n", + " 'usingcomputer': {'count': 1,\n", " 'values count': 1,\n", " 'values present': ['False'],\n", " 'distribution': {'False': [1, 1.0]}},\n", - " 'reading': {'count': 1,\n", + " 'other': {'count': 1,\n", " 'values count': 1,\n", " 'values present': ['True'],\n", " 'distribution': {'True': [1, 1.0]}},\n", - " 'ridingbike': {'count': 1,\n", + " 'jumping': {'count': 1,\n", " 'values count': 1,\n", " 'values present': ['False'],\n", " 'distribution': {'False': [1, 1.0]}},\n", - " 'takingphoto': {'count': 1,\n", + " 'phoning': {'count': 1,\n", " 'values count': 1,\n", " 'values present': ['True'],\n", " 'distribution': {'True': [1, 1.0]}},\n", - " 'usingcomputer': {'count': 1,\n", - " 'values count': 1,\n", - " 'values present': ['False'],\n", - " 'distribution': {'False': [1, 1.0]}},\n", - " 'walking': {'count': 1,\n", + " 'takingphoto': {'count': 1,\n", " 'values count': 1,\n", " 'values present': ['True'],\n", " 'distribution': {'True': [1, 1.0]}},\n", @@ -568,18 +568,22 @@ " 'values count': 1,\n", " 'values present': ['False'],\n", " 'distribution': {'False': [1, 1.0]}},\n", - " 'other': {'count': 1,\n", + " 'walking': {'count': 1,\n", " 'values count': 1,\n", " 'values present': ['True'],\n", " 'distribution': {'True': [1, 1.0]}},\n", - " 'phoning': {'count': 1,\n", + " 'reading': {'count': 1,\n", " 'values count': 1,\n", " 'values present': ['True'],\n", " 'distribution': {'True': [1, 1.0]}},\n", - " 'ridinghorse': {'count': 1,\n", + " 'playinginstrument': {'count': 1,\n", " 'values count': 1,\n", - " 'values present': ['True'],\n", - " 'distribution': {'True': [1, 1.0]}},\n", + " 'values present': ['False'],\n", + " 'distribution': {'False': [1, 1.0]}},\n", + " 'ridingbike': {'count': 1,\n", + " 'values count': 1,\n", + " 'values present': ['False'],\n", + " 'distribution': {'False': [1, 1.0]}},\n", " 'pose': {'count': 1,\n", " 'values count': 1,\n", " 'values present': ['Unspecified'],\n", @@ -625,7 +629,7 @@ " 'foot': [0, 0.0]}}}" ] }, - "execution_count": 30, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -645,17 +649,17 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 17, "id": "8dda4168", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 31, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -666,7 +670,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 18, "id": "67f0b7e9", "metadata": { "scrolled": true @@ -676,7 +680,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "annotation count for coco dataset whose annotation is (label=\"cat\" and area < 9643950) or label!=\"person\"\n" + "annotation count for voc dataset whose annotation is label!=\"person\"\n" ] }, { @@ -685,7 +689,7 @@ "13" ] }, - "execution_count": 32, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -697,7 +701,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 19, "id": "506ae22a", "metadata": {}, "outputs": [ @@ -786,7 +790,7 @@ " 'foot': [0, 0.0]}}}" ] }, - "execution_count": 33, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -813,7 +817,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 20, "id": "732face4", "metadata": {}, "outputs": [], @@ -823,7 +827,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 21, "id": "7badb290", "metadata": {}, "outputs": [ @@ -853,7 +857,7 @@ " 9.411065220006367e-08]}}}" ] }, - "execution_count": 72, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -865,7 +869,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 22, "id": "326af8ad", "metadata": {}, "outputs": [ @@ -882,7 +886,7 @@ "15" ] }, - "execution_count": 73, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -894,17 +898,17 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 23, "id": "f8ddf924", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 74, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -915,7 +919,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 24, "id": "3a971546", "metadata": {}, "outputs": [ @@ -940,7 +944,7 @@ " 9.411065220006367e-08]}}}" ] }, - "execution_count": 75, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -952,7 +956,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 25, "id": "f66b97ac", "metadata": {}, "outputs": [ @@ -969,7 +973,7 @@ "2" ] }, - "execution_count": 76, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } From ffbbbb54faf7f48e85bbe512b6158941955061e7 Mon Sep 17 00:00:00 2001 From: sooahleex Date: Tue, 1 Nov 2022 23:12:47 +0900 Subject: [PATCH 4/8] Use __repr__ for dataset instead of compute_image_statistics --- datumaro/components/dataset.py | 2 +- notebooks/04_filter.ipynb | 577 ++++++++------------------------- 2 files changed, 139 insertions(+), 440 deletions(-) diff --git a/datumaro/components/dataset.py b/datumaro/components/dataset.py index b261d953fa..7db39dbe62 100644 --- a/datumaro/components/dataset.py +++ b/datumaro/components/dataset.py @@ -869,7 +869,7 @@ def get_annotated_size(self): def get_subset_info(self): return ( f"{subset_name}: # of items={len(self.get_subset(subset_name))}, " - f"# of annotations={self.get_subset(subset_name).get_annotated_size()}, " + f"# of annotated={self.get_subset(subset_name).get_annotated_size()}, " f"annotation types={self.get_subset(subset_name).get_annotated_type()}\n" for subset_name in sorted(self.subsets().keys()) ) diff --git a/notebooks/04_filter.ipynb b/notebooks/04_filter.ipynb index 35863fcf4d..baa8905697 100644 --- a/notebooks/04_filter.ipynb +++ b/notebooks/04_filter.ipynb @@ -18,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 1, "id": "da198c67", "metadata": {}, "outputs": [], @@ -29,7 +29,7 @@ "\n", "import os\n", "import datumaro as dm\n", - "from datumaro.components.operations import compute_image_statistics, compute_ann_statistics" + "from datumaro.components.operations import compute_ann_statistics" ] }, { @@ -70,26 +70,23 @@ "name": "stdout", "output_type": "stream", "text": [ - "statistics for a sample VOC dataset\n" + "Representation for a sample VOC dataset\n" ] }, { "data": { "text/plain": [ - "{'dataset': {'images count': 2,\n", - " 'unique images count': 1,\n", - " 'repeated images count': 1,\n", - " 'repeated images': [[('2007_000001', 'train'), ('2007_000002', 'test')]]},\n", - " 'subsets': {'test': {'images count': 1,\n", - " 'image mean': [0.9999999999999971, 0.9999999999999971, 0.9999999999999971],\n", - " 'image std': [9.411065220006367e-08,\n", - " 9.411065220006367e-08,\n", - " 9.411065220006367e-08]},\n", - " 'train': {'images count': 1,\n", - " 'image mean': [0.9999999999999971, 0.9999999999999971, 0.9999999999999971],\n", - " 'image std': [9.411065220006367e-08,\n", - " 9.411065220006367e-08,\n", - " 9.411065220006367e-08]}}}" + "Dataset\n", + "\tsize=2\n", + "\tsource_path=./tests/assets/voc_dataset/voc_dataset1\n", + "\tmedia_type=\n", + "\tannotated_count=1\n", + "subsets\n", + "\ttest: # of items=1, # of annotated=0, annotation types=[]\n", + "\ttrain: # of items=1, # of annotated=1, annotation types=['label', 'bbox', 'mask']\n", + "categories\n", + "\tlabel: ['background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor', 'ignored', 'head', 'hand', 'foot']\n", + "\tmask: []" ] }, "execution_count": 3, @@ -98,8 +95,8 @@ } ], "source": [ - "print('statistics for a sample VOC dataset')\n", - "compute_image_statistics(dataset)" + "print('Representation for sample VOC dataset')\n", + "dataset" ] }, { @@ -115,20 +112,9 @@ "execution_count": 4, "id": "51bf3388", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "dm.Dataset.filter(dataset, '/item[subset=\"train\"]')" + "dataset = dm.Dataset.filter(dataset, '/item[subset=\"train\"]')" ] }, { @@ -143,21 +129,22 @@ "name": "stdout", "output_type": "stream", "text": [ - "statistics for train subset VOC dataset\n" + "Representation for `train` subset of sample VOC dataset\n" ] }, { "data": { "text/plain": [ - "{'dataset': {'images count': 1,\n", - " 'unique images count': 1,\n", - " 'repeated images count': 0,\n", - " 'repeated images': []},\n", - " 'subsets': {'train': {'images count': 1,\n", - " 'image mean': [0.9999999999999971, 0.9999999999999971, 0.9999999999999971],\n", - " 'image std': [9.411065220006367e-08,\n", - " 9.411065220006367e-08,\n", - " 9.411065220006367e-08]}}}" + "Dataset\n", + "\tsize=1\n", + "\tsource_path=./tests/assets/voc_dataset/voc_dataset1\n", + "\tmedia_type=\n", + "\tannotated_count=1\n", + "subsets\n", + "\ttrain: # of items=1, # of annotated=1, annotation types=['label', 'bbox', 'mask']\n", + "categories\n", + "\tlabel: ['background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor', 'ignored', 'head', 'hand', 'foot']\n", + "\tmask: []" ] }, "execution_count": 5, @@ -166,8 +153,8 @@ } ], "source": [ - "print('statistics for train subset VOC dataset')\n", - "compute_image_statistics(dataset)" + "print('Representation for `train` subset of sample VOC dataset')\n", + "dataset" ] }, { @@ -206,28 +193,22 @@ "name": "stdout", "output_type": "stream", "text": [ - "statistics for a sample WiderFace dataset\n" + "Representation for a sample WiderFace dataset\n" ] }, { "data": { "text/plain": [ - "{'dataset': {'images count': 3,\n", - " 'unique images count': 1,\n", - " 'repeated images count': 1,\n", - " 'repeated images': [[('0_Parade_image_01', 'train'),\n", - " ('0_Parade_image_03', 'val'),\n", - " ('1_Handshaking_image_02', 'train')]]},\n", - " 'subsets': {'val': {'images count': 1,\n", - " 'image mean': [0.9999999999999973, 0.9999999999999973, 0.9999999999999973],\n", - " 'image std': [9.058862863930295e-08,\n", - " 9.058862863930295e-08,\n", - " 9.058862863930295e-08]},\n", - " 'train': {'images count': 2,\n", - " 'image mean': [0.9999999999999973, 0.9999999999999973, 0.9999999999999973],\n", - " 'image std': [9.043701576544718e-08,\n", - " 9.043701576544718e-08,\n", - " 9.043701576544718e-08]}}}" + "Dataset\n", + "\tsize=3\n", + "\tsource_path=./tests/assets/widerface_dataset\n", + "\tmedia_type=\n", + "\tannotated_count=3\n", + "subsets\n", + "\ttrain: # of items=2, # of annotated=2, annotation types=['label', 'bbox']\n", + "\tval: # of items=1, # of annotated=1, annotation types=['label', 'bbox']\n", + "categories\n", + "\tlabel: ['Parade', 'Handshaking']" ] }, "execution_count": 7, @@ -236,8 +217,8 @@ } ], "source": [ - "print('statistics for a sample WiderFace dataset')\n", - "compute_image_statistics(dataset)" + "print('Representation for sample WiderFace dataset')\n", + "dataset" ] }, { @@ -245,20 +226,9 @@ "execution_count": 8, "id": "4ed4a847", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "dm.Dataset.filter(dataset, '/item[id=\"0_Parade_image_01\"]')" + "dataset = dm.Dataset.filter(dataset, '/item[id=\"0_Parade_image_01\"]')" ] }, { @@ -271,21 +241,21 @@ "name": "stdout", "output_type": "stream", "text": [ - "statistics for WiderFace dataset id == 1\n" + "Representation for `id == 1` dataset of sample WiderFace dataset\n" ] }, { "data": { "text/plain": [ - "{'dataset': {'images count': 1,\n", - " 'unique images count': 1,\n", - " 'repeated images count': 0,\n", - " 'repeated images': []},\n", - " 'subsets': {'train': {'images count': 1,\n", - " 'image mean': [0.9999999999999973, 0.9999999999999973, 0.9999999999999973],\n", - " 'image std': [9.058862863930295e-08,\n", - " 9.058862863930295e-08,\n", - " 9.058862863930295e-08]}}}" + "Dataset\n", + "\tsize=1\n", + "\tsource_path=./tests/assets/widerface_dataset\n", + "\tmedia_type=\n", + "\tannotated_count=1\n", + "subsets\n", + "\ttrain: # of items=1, # of annotated=1, annotation types=['label', 'bbox']\n", + "categories\n", + "\tlabel: ['Parade', 'Handshaking']" ] }, "execution_count": 9, @@ -294,8 +264,8 @@ } ], "source": [ - "print('statistics for WiderFace dataset id == 1')\n", - "compute_image_statistics(dataset)" + "print('Representation for `id == 1` dataset of sample WiderFace dataset')\n", + "dataset" ] }, { @@ -316,7 +286,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 10, "id": "e5e30736", "metadata": {}, "outputs": [ @@ -324,7 +294,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "WARNING:root:Not implemented: Found potentially conflicting source types with labels: labels, instances, person_keypoints, stuff, panoptic. Only one type will be used: instances\n", + "WARNING:root:Not implemented: Found potentially conflicting source types with labels: instances, panoptic, labels, person_keypoints, stuff. Only one type will be used: instances\n", "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/labels_train.json' is skipped.\n", "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/person_keypoints_train.json' is skipped.\n", "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/stuff_train.json' is skipped.\n", @@ -342,7 +312,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 11, "id": "87feaf56", "metadata": {}, "outputs": [], @@ -356,7 +326,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 12, "id": "b821387a", "metadata": {}, "outputs": [ @@ -364,7 +334,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "width and height for a sample coco dataset images\n" + "Width and height for a sample COCO dataset images\n" ] }, { @@ -373,40 +343,29 @@ "{'a': (5, 10), 'b': (10, 5)}" ] }, - "execution_count": 26, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "print('width and height for a sample coco dataset images')\n", + "print('Width and height for sample COCO dataset images')\n", "get_width_height(dataset)" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 13, "id": "7b693479", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "dm.Dataset.filter(dataset, '/item[image/width < image/height]')" + "dataset = dm.Dataset.filter(dataset, '/item[image/width < image/height]')" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 14, "id": "7479f9c9", "metadata": {}, "outputs": [ @@ -414,7 +373,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "width and height for width < height coco dataset images\n" + "Width and height for `width < height` sample COCO dataset images\n" ] }, { @@ -423,13 +382,13 @@ "{'b': (10, 5)}" ] }, - "execution_count": 28, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "print('width and height for width < height coco dataset images')\n", + "print('Width and height for `width < height` sample COCO dataset images')\n", "get_width_height(dataset)" ] }, @@ -451,7 +410,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "id": "0c8248a9", "metadata": {}, "outputs": [], @@ -461,7 +420,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "id": "98ad83ad", "metadata": { "scrolled": false @@ -471,7 +430,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "annotation count for voc dataset\n" + "Annotation count for sample VOC dataset\n" ] }, { @@ -480,163 +439,14 @@ "15" ] }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "print('annotation count for voc dataset')\n", - "compute_ann_statistics(dataset)['annotations count']" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "6d74e4cd", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "annotation statistics for voc dataset whose annotation is label!=\"person\"\n" - ] - }, - { - "data": { - "text/plain": [ - "{'labels': {'count': 15,\n", - " 'distribution': {'background': [0, 0.0],\n", - " 'aeroplane': [1, 0.06666666666666667],\n", - " 'bicycle': [1, 0.06666666666666667],\n", - " 'bird': [1, 0.06666666666666667],\n", - " 'boat': [0, 0.0],\n", - " 'bottle': [1, 0.06666666666666667],\n", - " 'bus': [0, 0.0],\n", - " 'car': [1, 0.06666666666666667],\n", - " 'cat': [1, 0.06666666666666667],\n", - " 'chair': [1, 0.06666666666666667],\n", - " 'cow': [0, 0.0],\n", - " 'diningtable': [1, 0.06666666666666667],\n", - " 'dog': [0, 0.0],\n", - " 'horse': [1, 0.06666666666666667],\n", - " 'motorbike': [0, 0.0],\n", - " 'person': [2, 0.13333333333333333],\n", - " 'pottedplant': [0, 0.0],\n", - " 'sheep': [1, 0.06666666666666667],\n", - " 'sofa': [0, 0.0],\n", - " 'train': [1, 0.06666666666666667],\n", - " 'tvmonitor': [0, 0.0],\n", - " 'ignored': [1, 0.06666666666666667],\n", - " 'head': [1, 0.06666666666666667],\n", - " 'hand': [0, 0.0],\n", - " 'foot': [0, 0.0]},\n", - " 'attributes': {'difficult': {'count': 2,\n", - " 'values count': 1,\n", - " 'values present': ['False'],\n", - " 'distribution': {'False': [2, 1.0]}},\n", - " 'truncated': {'count': 2,\n", - " 'values count': 2,\n", - " 'values present': ['False', 'True'],\n", - " 'distribution': {'False': [1, 0.5], 'True': [1, 0.5]}},\n", - " 'ridinghorse': {'count': 1,\n", - " 'values count': 1,\n", - " 'values present': ['True'],\n", - " 'distribution': {'True': [1, 1.0]}},\n", - " 'usingcomputer': {'count': 1,\n", - " 'values count': 1,\n", - " 'values present': ['False'],\n", - " 'distribution': {'False': [1, 1.0]}},\n", - " 'other': {'count': 1,\n", - " 'values count': 1,\n", - " 'values present': ['True'],\n", - " 'distribution': {'True': [1, 1.0]}},\n", - " 'jumping': {'count': 1,\n", - " 'values count': 1,\n", - " 'values present': ['False'],\n", - " 'distribution': {'False': [1, 1.0]}},\n", - " 'phoning': {'count': 1,\n", - " 'values count': 1,\n", - " 'values present': ['True'],\n", - " 'distribution': {'True': [1, 1.0]}},\n", - " 'takingphoto': {'count': 1,\n", - " 'values count': 1,\n", - " 'values present': ['True'],\n", - " 'distribution': {'True': [1, 1.0]}},\n", - " 'running': {'count': 1,\n", - " 'values count': 1,\n", - " 'values present': ['False'],\n", - " 'distribution': {'False': [1, 1.0]}},\n", - " 'walking': {'count': 1,\n", - " 'values count': 1,\n", - " 'values present': ['True'],\n", - " 'distribution': {'True': [1, 1.0]}},\n", - " 'reading': {'count': 1,\n", - " 'values count': 1,\n", - " 'values present': ['True'],\n", - " 'distribution': {'True': [1, 1.0]}},\n", - " 'playinginstrument': {'count': 1,\n", - " 'values count': 1,\n", - " 'values present': ['False'],\n", - " 'distribution': {'False': [1, 1.0]}},\n", - " 'ridingbike': {'count': 1,\n", - " 'values count': 1,\n", - " 'values present': ['False'],\n", - " 'distribution': {'False': [1, 1.0]}},\n", - " 'pose': {'count': 1,\n", - " 'values count': 1,\n", - " 'values present': ['Unspecified'],\n", - " 'distribution': {'Unspecified': [1, 1.0]}}}},\n", - " 'segments': {'avg. area': 53.0,\n", - " 'area distribution': [{'min': 4.0, 'max': 23.6, 'count': 3, 'percent': 0.75},\n", - " {'min': 23.6, 'max': 43.2, 'count': 0, 'percent': 0.0},\n", - " {'min': 43.2, 'max': 62.800000000000004, 'count': 0, 'percent': 0.0},\n", - " {'min': 62.800000000000004, 'max': 82.4, 'count': 0, 'percent': 0.0},\n", - " {'min': 82.4, 'max': 102.0, 'count': 0, 'percent': 0.0},\n", - " {'min': 102.0, 'max': 121.60000000000001, 'count': 0, 'percent': 0.0},\n", - " {'min': 121.60000000000001,\n", - " 'max': 141.20000000000002,\n", - " 'count': 0,\n", - " 'percent': 0.0},\n", - " {'min': 141.20000000000002, 'max': 160.8, 'count': 0, 'percent': 0.0},\n", - " {'min': 160.8, 'max': 180.4, 'count': 0, 'percent': 0.0},\n", - " {'min': 180.4, 'max': 200.0, 'count': 1, 'percent': 0.25}],\n", - " 'pixel distribution': {'background': [0, 0.0],\n", - " 'aeroplane': [0, 0.0],\n", - " 'bicycle': [200, 0.9433962264150944],\n", - " 'bird': [0, 0.0],\n", - " 'boat': [0, 0.0],\n", - " 'bottle': [0, 0.0],\n", - " 'bus': [0, 0.0],\n", - " 'car': [0, 0.0],\n", - " 'cat': [4, 0.018867924528301886],\n", - " 'chair': [0, 0.0],\n", - " 'cow': [0, 0.0],\n", - " 'diningtable': [0, 0.0],\n", - " 'dog': [0, 0.0],\n", - " 'horse': [0, 0.0],\n", - " 'motorbike': [0, 0.0],\n", - " 'person': [4, 0.018867924528301886],\n", - " 'pottedplant': [0, 0.0],\n", - " 'sheep': [0, 0.0],\n", - " 'sofa': [0, 0.0],\n", - " 'train': [0, 0.0],\n", - " 'tvmonitor': [0, 0.0],\n", - " 'ignored': [0, 0.0],\n", - " 'head': [4, 0.018867924528301886],\n", - " 'hand': [0, 0.0],\n", - " 'foot': [0, 0.0]}}}" - ] - }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "print('annotation statistics for voc dataset whose annotation is label!=\"person\"')\n", - "compute_ann_statistics(dataset)['annotations']" + "print('Annotation count for sample VOC dataset')\n", + "compute_ann_statistics(dataset)['annotations count']" ] }, { @@ -652,20 +462,9 @@ "execution_count": 17, "id": "8dda4168", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "dm.Dataset.filter(dataset, '/item/annotation[label!=\"person\"]', filter_annotations=True)" + "dataset = dm.Dataset.filter(dataset, '/item/annotation[label!=\"person\"]', filter_annotations=True)" ] }, { @@ -680,7 +479,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "annotation count for voc dataset whose annotation is label!=\"person\"\n" + "Representation for sample VOC dataset whose annotation is `label!=\"person\"`\n" ] }, { @@ -695,110 +494,10 @@ } ], "source": [ - "print('annotation count for voc dataset whose annotation is label!=\"person\"')\n", + "print('Representation for sample VOC dataset whose annotation is `label!=\"person\"`')\n", "compute_ann_statistics(dataset)['annotations count']" ] }, - { - "cell_type": "code", - "execution_count": 19, - "id": "506ae22a", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'labels': {'count': 13,\n", - " 'distribution': {'background': [0, 0.0],\n", - " 'aeroplane': [1, 0.07692307692307693],\n", - " 'bicycle': [1, 0.07692307692307693],\n", - " 'bird': [1, 0.07692307692307693],\n", - " 'boat': [0, 0.0],\n", - " 'bottle': [1, 0.07692307692307693],\n", - " 'bus': [0, 0.0],\n", - " 'car': [1, 0.07692307692307693],\n", - " 'cat': [1, 0.07692307692307693],\n", - " 'chair': [1, 0.07692307692307693],\n", - " 'cow': [0, 0.0],\n", - " 'diningtable': [1, 0.07692307692307693],\n", - " 'dog': [0, 0.0],\n", - " 'horse': [1, 0.07692307692307693],\n", - " 'motorbike': [0, 0.0],\n", - " 'person': [0, 0.0],\n", - " 'pottedplant': [0, 0.0],\n", - " 'sheep': [1, 0.07692307692307693],\n", - " 'sofa': [0, 0.0],\n", - " 'train': [1, 0.07692307692307693],\n", - " 'tvmonitor': [0, 0.0],\n", - " 'ignored': [1, 0.07692307692307693],\n", - " 'head': [1, 0.07692307692307693],\n", - " 'hand': [0, 0.0],\n", - " 'foot': [0, 0.0]},\n", - " 'attributes': {'difficult': {'count': 1,\n", - " 'values count': 1,\n", - " 'values present': ['False'],\n", - " 'distribution': {'False': [1, 1.0]}},\n", - " 'truncated': {'count': 1,\n", - " 'values count': 1,\n", - " 'values present': ['True'],\n", - " 'distribution': {'True': [1, 1.0]}},\n", - " 'pose': {'count': 1,\n", - " 'values count': 1,\n", - " 'values present': ['Unspecified'],\n", - " 'distribution': {'Unspecified': [1, 1.0]}}}},\n", - " 'segments': {'avg. area': 69.33333333333333,\n", - " 'area distribution': [{'min': 4.0,\n", - " 'max': 23.6,\n", - " 'count': 2,\n", - " 'percent': 0.6666666666666666},\n", - " {'min': 23.6, 'max': 43.2, 'count': 0, 'percent': 0.0},\n", - " {'min': 43.2, 'max': 62.800000000000004, 'count': 0, 'percent': 0.0},\n", - " {'min': 62.800000000000004, 'max': 82.4, 'count': 0, 'percent': 0.0},\n", - " {'min': 82.4, 'max': 102.0, 'count': 0, 'percent': 0.0},\n", - " {'min': 102.0, 'max': 121.60000000000001, 'count': 0, 'percent': 0.0},\n", - " {'min': 121.60000000000001,\n", - " 'max': 141.20000000000002,\n", - " 'count': 0,\n", - " 'percent': 0.0},\n", - " {'min': 141.20000000000002, 'max': 160.8, 'count': 0, 'percent': 0.0},\n", - " {'min': 160.8, 'max': 180.4, 'count': 0, 'percent': 0.0},\n", - " {'min': 180.4, 'max': 200.0, 'count': 1, 'percent': 0.3333333333333333}],\n", - " 'pixel distribution': {'background': [0, 0.0],\n", - " 'aeroplane': [0, 0.0],\n", - " 'bicycle': [200, 0.9615384615384616],\n", - " 'bird': [0, 0.0],\n", - " 'boat': [0, 0.0],\n", - " 'bottle': [0, 0.0],\n", - " 'bus': [0, 0.0],\n", - " 'car': [0, 0.0],\n", - " 'cat': [4, 0.019230769230769232],\n", - " 'chair': [0, 0.0],\n", - " 'cow': [0, 0.0],\n", - " 'diningtable': [0, 0.0],\n", - " 'dog': [0, 0.0],\n", - " 'horse': [0, 0.0],\n", - " 'motorbike': [0, 0.0],\n", - " 'person': [0, 0.0],\n", - " 'pottedplant': [0, 0.0],\n", - " 'sheep': [0, 0.0],\n", - " 'sofa': [0, 0.0],\n", - " 'train': [0, 0.0],\n", - " 'tvmonitor': [0, 0.0],\n", - " 'ignored': [0, 0.0],\n", - " 'head': [4, 0.019230769230769232],\n", - " 'hand': [0, 0.0],\n", - " 'foot': [0, 0.0]}}}" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "compute_ann_statistics(dataset)['annotations']" - ] - }, { "cell_type": "markdown", "id": "3201198f", @@ -817,7 +516,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "id": "732face4", "metadata": {}, "outputs": [], @@ -827,49 +526,46 @@ }, { "cell_type": "code", - "execution_count": 21, - "id": "7badb290", + "execution_count": 20, + "id": "16555c9c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "image statistics for sample voc dataset\n" + "Representation for sample VOC dataset\n" ] }, { "data": { "text/plain": [ - "{'dataset': {'images count': 2,\n", - " 'unique images count': 1,\n", - " 'repeated images count': 1,\n", - " 'repeated images': [[('2007_000001', 'train'), ('2007_000002', 'test')]]},\n", - " 'subsets': {'test': {'images count': 1,\n", - " 'image mean': [0.9999999999999971, 0.9999999999999971, 0.9999999999999971],\n", - " 'image std': [9.411065220006367e-08,\n", - " 9.411065220006367e-08,\n", - " 9.411065220006367e-08]},\n", - " 'train': {'images count': 1,\n", - " 'image mean': [0.9999999999999971, 0.9999999999999971, 0.9999999999999971],\n", - " 'image std': [9.411065220006367e-08,\n", - " 9.411065220006367e-08,\n", - " 9.411065220006367e-08]}}}" + "Dataset\n", + "\tsize=2\n", + "\tsource_path=./tests/assets/voc_dataset/voc_dataset1\n", + "\tmedia_type=\n", + "\tannotated_count=1\n", + "subsets\n", + "\ttest: # of items=1, # of annotated=0, annotation types=[]\n", + "\ttrain: # of items=1, # of annotated=1, annotation types=['label', 'bbox', 'mask']\n", + "categories\n", + "\tlabel: ['background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor', 'ignored', 'head', 'hand', 'foot']\n", + "\tmask: []" ] }, - "execution_count": 21, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "print('image statistics for sample voc dataset')\n", - "compute_image_statistics(dataset)" + "print('Representation for sample VOC dataset')\n", + "dataset" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "id": "326af8ad", "metadata": {}, "outputs": [ @@ -877,7 +573,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "annotation statistics for sample voc dataset\n" + "Annotation statistics for sample VOC dataset\n" ] }, { @@ -886,40 +582,29 @@ "15" ] }, - "execution_count": 22, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "print('annotation statistics for sample voc dataset')\n", + "print('Annotation statistics for sample VOC dataset')\n", "compute_ann_statistics(dataset)['annotations count']" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 22, "id": "f8ddf924", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "dm.Dataset.filter(dataset, '/item/annotation[occluded=\"False\"]', filter_annotations=True, remove_empty=True)" + "dataset = dm.Dataset.filter(dataset, '/item/annotation[occluded=\"False\"]', filter_annotations=True, remove_empty=True)" ] }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 23, "id": "3a971546", "metadata": {}, "outputs": [ @@ -927,36 +612,37 @@ "name": "stdout", "output_type": "stream", "text": [ - "image statistics for non-occluded annotations and empty images removed voc dataset\n" + "Representation for `non-occluded annotations and empty images removed sample VOC dataset`\n" ] }, { "data": { "text/plain": [ - "{'dataset': {'images count': 1,\n", - " 'unique images count': 1,\n", - " 'repeated images count': 0,\n", - " 'repeated images': []},\n", - " 'subsets': {'train': {'images count': 1,\n", - " 'image mean': [0.9999999999999971, 0.9999999999999971, 0.9999999999999971],\n", - " 'image std': [9.411065220006367e-08,\n", - " 9.411065220006367e-08,\n", - " 9.411065220006367e-08]}}}" + "Dataset\n", + "\tsize=1\n", + "\tsource_path=./tests/assets/voc_dataset/voc_dataset1\n", + "\tmedia_type=\n", + "\tannotated_count=1\n", + "subsets\n", + "\ttrain: # of items=1, # of annotated=1, annotation types=['bbox']\n", + "categories\n", + "\tlabel: ['background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor', 'ignored', 'head', 'hand', 'foot']\n", + "\tmask: []" ] }, - "execution_count": 24, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "print('image statistics for non-occluded annotations and empty images removed voc dataset')\n", - "compute_image_statistics(dataset)" + "print('Representation for `non-occluded annotations and empty images removed sample VOC dataset`')\n", + "dataset" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 24, "id": "f66b97ac", "metadata": {}, "outputs": [ @@ -964,7 +650,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "annotation statistics for non-occluded annotations and empty images removed voc dataset\n" + "Annotation statistics for `non-occluded annotations and empty images removed sample VOC dataset`\n" ] }, { @@ -973,15 +659,23 @@ "2" ] }, - "execution_count": 25, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "print('annotation statistics for non-occluded annotations and empty images removed voc dataset')\n", + "print('Annotation statistics for `non-occluded annotations and empty images removed sample VOC dataset`')\n", "compute_ann_statistics(dataset)['annotations count']" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5be2ee5f", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -1001,6 +695,11 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.13" + }, + "vscode": { + "interpreter": { + "hash": "1e90bae80f8f1f04a7aff772db01befa8d30966fbd5491c30dbbd054d522be4c" + } } }, "nbformat": 4, From e29f709d64a65f963b3f1fe713bbd1ed4bf9a551 Mon Sep 17 00:00:00 2001 From: sooahleex Date: Tue, 1 Nov 2022 23:15:23 +0900 Subject: [PATCH 5/8] Update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1de81846f8..9fe6fb3643 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 () - Add a documentation tab menu for Python API () +- Add __repr__ for Dataset + () ### Changed - Updated `networkx` version to 2.6 From 5373a9ca67089faccc8fbce8200e09e27f5f656a Mon Sep 17 00:00:00 2001 From: sooahleex Date: Tue, 1 Nov 2022 23:18:41 +0900 Subject: [PATCH 6/8] Update CHANGELOG.md --- CHANGELOG.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fe6fb3643..1390157aa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,9 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 () - Add Label, Points, Polygon, PolyLine, and Caption visualization features () -- Add a documentation tab menu for Python API - () -- Add __repr__ for Dataset +- Add `__repr__` for Dataset () ### Changed From 0cbff9d2674562276c2af595fd4503c3c1b02385 Mon Sep 17 00:00:00 2001 From: sooahleex Date: Tue, 1 Nov 2022 23:46:51 +0900 Subject: [PATCH 7/8] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1390157aa0..5c45bae97b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 () - Add Label, Points, Polygon, PolyLine, and Caption visualization features () +- Add a documentation tab menu for Python API + () - Add `__repr__` for Dataset () From 9bcb83c18409af7580f80e5cc6816aab6cc35674 Mon Sep 17 00:00:00 2001 From: sooahleex Date: Wed, 2 Nov 2022 00:30:10 +0900 Subject: [PATCH 8/8] Update __repr__ for Dataset to show annotations info --- datumaro/components/dataset.py | 38 ++++- notebooks/04_filter.ipynb | 267 ++++++++++++++++++++------------- 2 files changed, 193 insertions(+), 112 deletions(-) diff --git a/datumaro/components/dataset.py b/datumaro/components/dataset.py index 7db39dbe62..412a34b156 100644 --- a/datumaro/components/dataset.py +++ b/datumaro/components/dataset.py @@ -114,9 +114,16 @@ def get_subset(self, name): def subsets(self): return self.data - def get_annotated_size(self): + def get_annotated_items(self): return sum(bool(s.annotations) for s in self._traversal_order.values()) + def get_annotations(self): + annotations_by_type = {t.name: {"count": 0} for t in AnnotationType} + for item in self._traversal_order.values(): + for ann in item.annotations: + annotations_by_type[ann.type.name]["count"] += 1 + return sum(t["count"] for t in annotations_by_type.values()) + def __copy__(self): copied = DatasetItemStorage() copied._traversal_order = copy(self._traversal_order) @@ -282,9 +289,16 @@ def categories(self): def media_type(self): return self.parent.media_type() - def get_annotated_size(self): + def get_annotated_items(self): return sum(bool(s.annotations) for s in self.parent._data.get_subset(self.name)) + def get_annotations(self): + annotations_by_type = {t.name: {"count": 0} for t in AnnotationType} + for item in self.parent._data.get_subset(self.name): + for ann in item.annotations: + annotations_by_type[ann.type.name]["count"] += 1 + return sum(t["count"] for t in annotations_by_type.values()) + def get_annotated_type(self): annotation_types = [] for item in self.parent._data.get_subset(self.name): @@ -631,8 +645,11 @@ def subsets(self): # and other cases return self._merged().subsets() - def get_annotated_size(self): - return self._storage.get_annotated_size() + def get_annotated_items(self): + return self._storage.get_annotated_items() + + def get_annotations(self): + return self._storage.get_annotations() def transform(self, method: Type[Transform], *args, **kwargs): # Flush accumulated changes @@ -829,7 +846,8 @@ def __repr__(self) -> str: f"\tsize={len(self._data)}\n" f"\tsource_path={self._source_path}\n" f"\tmedia_type={self.media_type()}\n" - f"\tannotated_count={self.get_annotated_size()}\n" + f"\tannotated_items_count={self.get_annotated_items()}\n" + f"\tannotations_count={self.get_annotations()}\n" f"subsets\n" f"\t{separator.join(self.get_subset_info())}" f"categories\n" @@ -863,13 +881,17 @@ def media_type(self) -> Type[MediaElement]: def get(self, id: str, subset: Optional[str] = None) -> Optional[DatasetItem]: return self._data.get(id, subset) - def get_annotated_size(self): - return self._data.get_annotated_size() + def get_annotated_items(self): + return self._data.get_annotated_items() + + def get_annotations(self): + return self._data.get_annotations() def get_subset_info(self): return ( f"{subset_name}: # of items={len(self.get_subset(subset_name))}, " - f"# of annotated={self.get_subset(subset_name).get_annotated_size()}, " + f"# of annotated items={self.get_subset(subset_name).get_annotated_items()}, " + f"# of annotations={self.get_subset(subset_name).get_annotations()}, " f"annotation types={self.get_subset(subset_name).get_annotated_type()}\n" for subset_name in sorted(self.subsets().keys()) ) diff --git a/notebooks/04_filter.ipynb b/notebooks/04_filter.ipynb index baa8905697..557659d882 100644 --- a/notebooks/04_filter.ipynb +++ b/notebooks/04_filter.ipynb @@ -28,8 +28,7 @@ "# SPDX-License-Identifier: MIT\n", "\n", "import os\n", - "import datumaro as dm\n", - "from datumaro.components.operations import compute_ann_statistics" + "import datumaro as dm" ] }, { @@ -70,7 +69,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Representation for a sample VOC dataset\n" + "Representation for sample VOC dataset\n" ] }, { @@ -80,10 +79,11 @@ "\tsize=2\n", "\tsource_path=./tests/assets/voc_dataset/voc_dataset1\n", "\tmedia_type=\n", - "\tannotated_count=1\n", + "\tannotated_items_count=1\n", + "\tannotations_count=15\n", "subsets\n", - "\ttest: # of items=1, # of annotated=0, annotation types=[]\n", - "\ttrain: # of items=1, # of annotated=1, annotation types=['label', 'bbox', 'mask']\n", + "\ttest: # of items=1, # of annotated items=0, # of annotations=0, annotation types=[]\n", + "\ttrain: # of items=1, # of annotated items=1, # of annotations=15, annotation types=['label', 'bbox', 'mask']\n", "categories\n", "\tlabel: ['background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor', 'ignored', 'head', 'hand', 'foot']\n", "\tmask: []" @@ -139,9 +139,10 @@ "\tsize=1\n", "\tsource_path=./tests/assets/voc_dataset/voc_dataset1\n", "\tmedia_type=\n", - "\tannotated_count=1\n", + "\tannotated_items_count=1\n", + "\tannotations_count=15\n", "subsets\n", - "\ttrain: # of items=1, # of annotated=1, annotation types=['label', 'bbox', 'mask']\n", + "\ttrain: # of items=1, # of annotated items=1, # of annotations=15, annotation types=['label', 'bbox', 'mask']\n", "categories\n", "\tlabel: ['background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor', 'ignored', 'head', 'hand', 'foot']\n", "\tmask: []" @@ -193,7 +194,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Representation for a sample WiderFace dataset\n" + "Representation for sample WiderFace dataset\n" ] }, { @@ -203,10 +204,11 @@ "\tsize=3\n", "\tsource_path=./tests/assets/widerface_dataset\n", "\tmedia_type=\n", - "\tannotated_count=3\n", + "\tannotated_items_count=3\n", + "\tannotations_count=9\n", "subsets\n", - "\ttrain: # of items=2, # of annotated=2, annotation types=['label', 'bbox']\n", - "\tval: # of items=1, # of annotated=1, annotation types=['label', 'bbox']\n", + "\ttrain: # of items=2, # of annotated items=2, # of annotations=5, annotation types=['label', 'bbox']\n", + "\tval: # of items=1, # of annotated items=1, # of annotations=4, annotation types=['label', 'bbox']\n", "categories\n", "\tlabel: ['Parade', 'Handshaking']" ] @@ -251,9 +253,10 @@ "\tsize=1\n", "\tsource_path=./tests/assets/widerface_dataset\n", "\tmedia_type=\n", - "\tannotated_count=1\n", + "\tannotated_items_count=1\n", + "\tannotations_count=2\n", "subsets\n", - "\ttrain: # of items=1, # of annotated=1, annotation types=['label', 'bbox']\n", + "\ttrain: # of items=1, # of annotated items=1, # of annotations=2, annotation types=['label', 'bbox']\n", "categories\n", "\tlabel: ['Parade', 'Handshaking']" ] @@ -294,7 +297,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "WARNING:root:Not implemented: Found potentially conflicting source types with labels: instances, panoptic, labels, person_keypoints, stuff. Only one type will be used: instances\n", + "WARNING:root:Not implemented: Found potentially conflicting source types with labels: labels, panoptic, stuff, person_keypoints, instances. Only one type will be used: instances\n", "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/labels_train.json' is skipped.\n", "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/person_keypoints_train.json' is skipped.\n", "WARNING:root:Not implemented: conflicting source './tests/assets/coco_dataset/coco/annotations/stuff_train.json' is skipped.\n", @@ -327,6 +330,45 @@ { "cell_type": "code", "execution_count": 12, + "id": "e095b117", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Representation for sample COCO dataset\n" + ] + }, + { + "data": { + "text/plain": [ + "Dataset\n", + "\tsize=2\n", + "\tsource_path=./tests/assets/coco_dataset/coco\n", + "\tmedia_type=\n", + "\tannotated_items_count=2\n", + "\tannotations_count=6\n", + "subsets\n", + "\ttrain: # of items=1, # of annotated items=1, # of annotations=2, annotation types=['bbox', 'caption']\n", + "\tval: # of items=1, # of annotated items=1, # of annotations=4, annotation types=['mask', 'caption', 'polygon']\n", + "categories\n", + "\tlabel: ['a', 'b', 'c']" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('Representation for sample COCO dataset')\n", + "dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 13, "id": "b821387a", "metadata": {}, "outputs": [ @@ -334,7 +376,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Width and height for a sample COCO dataset images\n" + "Width and height for sample COCO dataset images\n" ] }, { @@ -343,7 +385,7 @@ "{'a': (5, 10), 'b': (10, 5)}" ] }, - "execution_count": 12, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -355,7 +397,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "id": "7b693479", "metadata": {}, "outputs": [], @@ -365,7 +407,45 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, + "id": "6d30b6c4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Representation for `width < height` sample COCO dataset images\n" + ] + }, + { + "data": { + "text/plain": [ + "Dataset\n", + "\tsize=1\n", + "\tsource_path=./tests/assets/coco_dataset/coco\n", + "\tmedia_type=\n", + "\tannotated_items_count=1\n", + "\tannotations_count=4\n", + "subsets\n", + "\tval: # of items=1, # of annotated items=1, # of annotations=4, annotation types=['mask', 'caption', 'polygon']\n", + "categories\n", + "\tlabel: ['a', 'b', 'c']" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('Representation for `width < height` sample COCO dataset images')\n", + "dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 16, "id": "7479f9c9", "metadata": {}, "outputs": [ @@ -382,7 +462,7 @@ "{'b': (10, 5)}" ] }, - "execution_count": 14, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -410,7 +490,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 17, "id": "0c8248a9", "metadata": {}, "outputs": [], @@ -420,33 +500,44 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 18, "id": "98ad83ad", "metadata": { - "scrolled": false + "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Annotation count for sample VOC dataset\n" + "Representation for sample VOC dataset\n" ] }, { "data": { "text/plain": [ - "15" + "Dataset\n", + "\tsize=2\n", + "\tsource_path=./tests/assets/voc_dataset/voc_dataset1\n", + "\tmedia_type=\n", + "\tannotated_items_count=1\n", + "\tannotations_count=15\n", + "subsets\n", + "\ttest: # of items=1, # of annotated items=0, # of annotations=0, annotation types=[]\n", + "\ttrain: # of items=1, # of annotated items=1, # of annotations=15, annotation types=['label', 'bbox', 'mask']\n", + "categories\n", + "\tlabel: ['background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor', 'ignored', 'head', 'hand', 'foot']\n", + "\tmask: []" ] }, - "execution_count": 16, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "print('Annotation count for sample VOC dataset')\n", - "compute_ann_statistics(dataset)['annotations count']" + "print('Representation for sample VOC dataset')\n", + "dataset" ] }, { @@ -459,7 +550,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 19, "id": "8dda4168", "metadata": {}, "outputs": [], @@ -469,7 +560,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 20, "id": "67f0b7e9", "metadata": { "scrolled": true @@ -485,17 +576,28 @@ { "data": { "text/plain": [ - "13" + "Dataset\n", + "\tsize=2\n", + "\tsource_path=./tests/assets/voc_dataset/voc_dataset1\n", + "\tmedia_type=\n", + "\tannotated_items_count=1\n", + "\tannotations_count=13\n", + "subsets\n", + "\ttest: # of items=1, # of annotated items=0, # of annotations=0, annotation types=[]\n", + "\ttrain: # of items=1, # of annotated items=1, # of annotations=13, annotation types=['label', 'bbox', 'mask']\n", + "categories\n", + "\tlabel: ['background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor', 'ignored', 'head', 'hand', 'foot']\n", + "\tmask: []" ] }, - "execution_count": 18, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "print('Representation for sample VOC dataset whose annotation is `label!=\"person\"`')\n", - "compute_ann_statistics(dataset)['annotations count']" + "dataset" ] }, { @@ -516,7 +618,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 21, "id": "732face4", "metadata": {}, "outputs": [], @@ -526,8 +628,8 @@ }, { "cell_type": "code", - "execution_count": 20, - "id": "16555c9c", + "execution_count": 22, + "id": "7badb290", "metadata": {}, "outputs": [ { @@ -544,16 +646,17 @@ "\tsize=2\n", "\tsource_path=./tests/assets/voc_dataset/voc_dataset1\n", "\tmedia_type=\n", - "\tannotated_count=1\n", + "\tannotated_items_count=1\n", + "\tannotations_count=15\n", "subsets\n", - "\ttest: # of items=1, # of annotated=0, annotation types=[]\n", - "\ttrain: # of items=1, # of annotated=1, annotation types=['label', 'bbox', 'mask']\n", + "\ttest: # of items=1, # of annotated items=0, # of annotations=0, annotation types=[]\n", + "\ttrain: # of items=1, # of annotated items=1, # of annotations=15, annotation types=['label', 'bbox', 'mask']\n", "categories\n", "\tlabel: ['background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor', 'ignored', 'head', 'hand', 'foot']\n", "\tmask: []" ] }, - "execution_count": 20, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -563,58 +666,12 @@ "dataset" ] }, - { - "cell_type": "code", - "execution_count": 21, - "id": "326af8ad", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Annotation statistics for sample VOC dataset\n" - ] - }, - { - "data": { - "text/plain": [ - "15" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "print('Annotation statistics for sample VOC dataset')\n", - "compute_ann_statistics(dataset)['annotations count']" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "f8ddf924", - "metadata": {}, - "outputs": [], - "source": [ - "dataset = dm.Dataset.filter(dataset, '/item/annotation[occluded=\"False\"]', filter_annotations=True, remove_empty=True)" - ] - }, { "cell_type": "code", "execution_count": 23, - "id": "3a971546", + "id": "f8ddf924", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Representation for `non-occluded annotations and empty images removed sample VOC dataset`\n" - ] - }, { "data": { "text/plain": [ @@ -622,9 +679,10 @@ "\tsize=1\n", "\tsource_path=./tests/assets/voc_dataset/voc_dataset1\n", "\tmedia_type=\n", - "\tannotated_count=1\n", + "\tannotated_items_count=1\n", + "\tannotations_count=2\n", "subsets\n", - "\ttrain: # of items=1, # of annotated=1, annotation types=['bbox']\n", + "\ttrain: # of items=1, # of annotated items=1, # of annotations=2, annotation types=['bbox']\n", "categories\n", "\tlabel: ['background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor', 'ignored', 'head', 'hand', 'foot']\n", "\tmask: []" @@ -636,27 +694,36 @@ } ], "source": [ - "print('Representation for `non-occluded annotations and empty images removed sample VOC dataset`')\n", - "dataset" + "dm.Dataset.filter(dataset, '/item/annotation[occluded=\"False\"]', filter_annotations=True, remove_empty=True)" ] }, { "cell_type": "code", "execution_count": 24, - "id": "f66b97ac", + "id": "3a971546", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Annotation statistics for `non-occluded annotations and empty images removed sample VOC dataset`\n" + "Representation for `non-occluded annotations and empty images removed sample VOC dataset`\n" ] }, { "data": { "text/plain": [ - "2" + "Dataset\n", + "\tsize=1\n", + "\tsource_path=./tests/assets/voc_dataset/voc_dataset1\n", + "\tmedia_type=\n", + "\tannotated_items_count=1\n", + "\tannotations_count=2\n", + "subsets\n", + "\ttrain: # of items=1, # of annotated items=1, # of annotations=2, annotation types=['bbox']\n", + "categories\n", + "\tlabel: ['background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor', 'ignored', 'head', 'hand', 'foot']\n", + "\tmask: []" ] }, "execution_count": 24, @@ -665,17 +732,9 @@ } ], "source": [ - "print('Annotation statistics for `non-occluded annotations and empty images removed sample VOC dataset`')\n", - "compute_ann_statistics(dataset)['annotations count']" + "print('Representation for `non-occluded annotations and empty images removed sample VOC dataset`')\n", + "dataset" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5be2ee5f", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": {