From 1594990bde82aab739177c9a7bfd0202f648dc86 Mon Sep 17 00:00:00 2001 From: brimoor Date: Wed, 11 Dec 2024 00:28:45 -0500 Subject: [PATCH 01/12] support on-disk instance segmentations in SDK --- docs/source/user_guide/using_datasets.rst | 8 +- fiftyone/core/collections.py | 3 + fiftyone/core/labels.py | 7 +- fiftyone/utils/data/exporters.py | 102 +++++++++++-------- fiftyone/utils/data/importers.py | 52 ++++++---- fiftyone/utils/labels.py | 113 +++++++++++++++++----- tests/unittests/import_export_tests.py | 109 +++++++++++++++++++++ 7 files changed, 303 insertions(+), 91 deletions(-) diff --git a/docs/source/user_guide/using_datasets.rst b/docs/source/user_guide/using_datasets.rst index 729434e07e..8bab5b83a6 100644 --- a/docs/source/user_guide/using_datasets.rst +++ b/docs/source/user_guide/using_datasets.rst @@ -2542,7 +2542,7 @@ Object detections stored in |Detections| may also have instance segmentation masks. These masks can be stored in one of two ways: either directly in the database -via the :attr:`mask` attribute, or on +via the :attr:`mask ` attribute, or on disk referenced by the :attr:`mask_path ` attribute. @@ -2605,8 +2605,10 @@ object's bounding box when visualizing in the App. , }> -Like all |Label| types, you can also add custom attributes to your detections -by dynamically adding new fields to each |Detection| instance: +Like all |Label| types, you can also add custom attributes to your instance +segmentations by dynamically adding new fields to each |Detection| instance: .. code-block:: python :linenos: diff --git a/fiftyone/core/collections.py b/fiftyone/core/collections.py index d53a25fce1..8200255fb4 100644 --- a/fiftyone/core/collections.py +++ b/fiftyone/core/collections.py @@ -10681,6 +10681,9 @@ def _get_media_fields( app_media_fields.discard("filepath") for field_name, field in schema.items(): + while isinstance(field, fof.ListField): + field = field.field + if field_name in app_media_fields: media_fields[field_name] = None elif isinstance(field, fof.EmbeddedDocumentField) and issubclass( diff --git a/fiftyone/core/labels.py b/fiftyone/core/labels.py index e8b9bd9390..e6b09d8267 100644 --- a/fiftyone/core/labels.py +++ b/fiftyone/core/labels.py @@ -409,7 +409,8 @@ class Detection(_HasAttributesDict, _HasID, _HasMedia, Label): its bounding box, which should be a 2D binary or 0/1 integer numpy array mask_path (None): the absolute path to the instance segmentation image - on disk + on disk, which should be a single-channel PNG image where any + non-zero values represent the instance's extent confidence (None): a confidence in ``[0, 1]`` for the detection index (None): an index for the object attributes ({}): a dict mapping attribute names to :class:`Attribute` @@ -532,8 +533,8 @@ def to_segmentation(self, mask=None, frame_size=None, target=255): """ if not self.has_mask: raise ValueError( - "Only detections with their `mask` attributes populated can " - "be converted to segmentations" + "Only detections with their `mask` or `mask_path` attribute " + "populated can be converted to segmentations" ) mask, target = _parse_segmentation_target(mask, frame_size, target) diff --git a/fiftyone/utils/data/exporters.py b/fiftyone/utils/data/exporters.py index e2a0780380..7a9b7da68e 100644 --- a/fiftyone/utils/data/exporters.py +++ b/fiftyone/utils/data/exporters.py @@ -12,11 +12,13 @@ import warnings from collections import defaultdict +from bson import json_util +import pydash + import eta.core.datasets as etad import eta.core.frameutils as etaf import eta.core.serial as etas import eta.core.utils as etau -from bson import json_util import fiftyone as fo import fiftyone.core.collections as foc @@ -2029,34 +2031,38 @@ def _export_frame_labels(self, sample, uuid): def _export_media_fields(self, sd): for field_name, key in self._media_fields.items(): - value = sd.get(field_name, None) - if value is None: - continue - - if key is not None: - self._export_media_field(value, field_name, key=key) - else: - self._export_media_field(sd, field_name) + self._export_media_field(sd, field_name, key=key) def _export_media_field(self, d, field_name, key=None): - if key is not None: - value = d.get(key, None) - else: - key = field_name - value = d.get(field_name, None) - + value = pydash.get(d, field_name, None) if value is None: return media_exporter = self._get_media_field_exporter(field_name) - outpath, _ = media_exporter.export(value) - if self.abs_paths: - d[key] = outpath - else: - d[key] = fou.safe_relpath( - outpath, self.export_dir, default=outpath - ) + if not isinstance(value, (list, tuple)): + value = [value] + + for _d in value: + if key is not None: + _value = _d.get(key, None) + else: + _value = _d + + if _value is None: + continue + + outpath, _ = media_exporter.export(_value) + + if not self.abs_paths: + outpath = fou.safe_relpath( + outpath, self.export_dir, default=outpath + ) + + if key is not None: + _d[key] = outpath + else: + pydash.set_(d, field_name, outpath) def _get_media_field_exporter(self, field_name): media_exporter = self._media_field_exporters.get(field_name, None) @@ -2333,33 +2339,43 @@ def _prep_sample(sd): def _export_media_fields(self, sd): for field_name, key in self._media_fields.items(): - value = sd.get(field_name, None) - if value is None: - continue + self._export_media_field(sd, field_name, key=key) + + def _export_media_field(self, d, field_name, key=None): + value = pydash.get(d, field_name, None) + if value is None: + return + media_exporter = self._get_media_field_exporter(field_name) + + if not isinstance(value, (list, tuple)): + value = [value] + + for _d in value: if key is not None: - self._export_media_field(value, field_name, key=key) + _value = _d.get(key, None) else: - self._export_media_field(sd, field_name) + _value = _d - def _export_media_field(self, d, field_name, key=None): - if key is not None: - value = d.get(key, None) - else: - key = field_name - value = d.get(field_name, None) + if _value is None: + continue - if value is None: - return + if self.export_media is not False: + # Store relative path + _, uuid = media_exporter.export(_value) + outpath = os.path.join("fields", field_name, uuid) + elif self.rel_dir is not None: + # Remove `rel_dir` prefix from path + outpath = fou.safe_relpath( + _value, self.rel_dir, default=_value + ) + else: + continue - if self.export_media is not False: - # Store relative path - media_exporter = self._get_media_field_exporter(field_name) - _, uuid = media_exporter.export(value) - d[key] = os.path.join("fields", field_name, uuid) - elif self.rel_dir is not None: - # Remove `rel_dir` prefix from path - d[key] = fou.safe_relpath(value, self.rel_dir, default=value) + if key is not None: + _d[key] = outpath + else: + pydash.set_(d, field_name, outpath) def _get_media_field_exporter(self, field_name): media_exporter = self._media_field_exporters.get(field_name, None) diff --git a/fiftyone/utils/data/importers.py b/fiftyone/utils/data/importers.py index 11c50f45a5..299827f3c0 100644 --- a/fiftyone/utils/data/importers.py +++ b/fiftyone/utils/data/importers.py @@ -14,6 +14,7 @@ from bson import json_util from mongoengine.base import get_document +import pydash import eta.core.datasets as etad import eta.core.image as etai @@ -2151,32 +2152,43 @@ def _import_runs(dataset, runs, results_dir, run_cls): def _parse_media_fields(sd, media_fields, rel_dir): for field_name, key in media_fields.items(): - value = sd.get(field_name, None) + value = pydash.get(sd, field_name, None) if value is None: continue if isinstance(value, dict): - if key is False: - try: - _cls = value.get("_cls", None) - key = get_document(_cls)._MEDIA_FIELD - except Exception as e: - logger.warning( - "Failed to infer media field for '%s'. Reason: %s", - field_name, - e, - ) - key = None - - media_fields[field_name] = key - - if key is not None: - path = value.get(key, None) - if path is not None and not os.path.isabs(path): - value[key] = os.path.join(rel_dir, path) + _parse_nested_media_field( + value, media_fields, rel_dir, field_name, key + ) + elif isinstance(value, list): + for d in value: + _parse_nested_media_field( + d, media_fields, rel_dir, field_name, key + ) elif etau.is_str(value): if not os.path.isabs(value): - sd[field_name] = os.path.join(rel_dir, value) + pydash.set_(sd, field_name, os.path.join(rel_dir, value)) + + +def _parse_nested_media_field(d, media_fields, rel_dir, field_name, key): + if key is False: + try: + _cls = d.get("_cls", None) + key = get_document(_cls)._MEDIA_FIELD + except Exception as e: + logger.warning( + "Failed to infer media field for '%s'. Reason: %s", + field_name, + e, + ) + key = None + + media_fields[field_name] = key + + if key is not None: + path = d.get(key, None) + if path is not None and not os.path.isabs(path): + d[key] = os.path.join(rel_dir, path) class ImageDirectoryImporter(UnlabeledImageDatasetImporter): diff --git a/fiftyone/utils/labels.py b/fiftyone/utils/labels.py index 7071d1f001..f28bfac205 100644 --- a/fiftyone/utils/labels.py +++ b/fiftyone/utils/labels.py @@ -155,8 +155,8 @@ def export_segmentations( overwrite=False, progress=None, ): - """Exports the segmentations (or heatmaps) stored as in-database arrays in - the specified field to images on disk. + """Exports the semantic segmentations, instance segmentations, or heatmaps + stored as in-database arrays in the specified field to images on disk. Any labels without in-memory arrays are skipped. @@ -164,7 +164,9 @@ def export_segmentations( sample_collection: a :class:`fiftyone.core.collections.SampleCollection` in_field: the name of the - :class:`fiftyone.core.labels.Segmentation` or + :class:`fiftyone.core.labels.Segmentation`, + :class:`fiftyone.core.labels.Detection`, + :class:`fiftyone.core.labels.Detections`, or :class:`fiftyone.core.labels.Heatmap` field output_dir: the directory in which to write the images rel_dir (None): an optional relative directory to strip from each input @@ -183,7 +185,9 @@ def export_segmentations( """ fov.validate_non_grouped_collection(sample_collection) fov.validate_collection_label_fields( - sample_collection, in_field, (fol.Segmentation, fol.Heatmap) + sample_collection, + in_field, + (fol.Segmentation, fol.Detection, fol.Detections, fol.Heatmap), ) samples = sample_collection.select_fields(in_field) @@ -207,16 +211,31 @@ def export_segmentations( if label is None: continue - outpath = filename_maker.get_output_path( - image.filepath, output_ext=".png" - ) - - if isinstance(label, fol.Heatmap): - if label.map is not None: - label.export_map(outpath, update=update) - else: + if isinstance(label, fol.Segmentation): + if label.mask is not None: + outpath = filename_maker.get_output_path( + image.filepath, output_ext=".png" + ) + label.export_mask(outpath, update=update) + elif isinstance(label, fol.Detection): if label.mask is not None: + outpath = filename_maker.get_output_path( + image.filepath, output_ext=".png" + ) label.export_mask(outpath, update=update) + elif isinstance(label, fol.Detections): + for detection in label.detections: + if detection.mask is not None: + outpath = filename_maker.get_output_path( + image.filepath, output_ext=".png" + ) + detection.export_mask(outpath, update=update) + elif isinstance(label, fol.Heatmap): + if label.map is not None: + outpath = filename_maker.get_output_path( + image.filepath, output_ext=".png" + ) + label.export_map(outpath, update=update) def import_segmentations( @@ -226,8 +245,8 @@ def import_segmentations( delete_images=False, progress=None, ): - """Imports the segmentations (or heatmaps) stored on disk in the specified - field to in-database arrays. + """Imports the semantic segmentations, instance segmentations, or heatmaps + stored on disk in the specified field to in-database arrays. Any labels without images on disk are skipped. @@ -235,7 +254,9 @@ def import_segmentations( sample_collection: a :class:`fiftyone.core.collections.SampleCollection` in_field: the name of the - :class:`fiftyone.core.labels.Segmentation` or + :class:`fiftyone.core.labels.Segmentation`, + :class:`fiftyone.core.labels.Detection`, + :class:`fiftyone.core.labels.Detections`, or :class:`fiftyone.core.labels.Heatmap` field update (True): whether to delete the image paths from the labels delete_images (False): whether to delete any imported images from disk @@ -245,7 +266,9 @@ def import_segmentations( """ fov.validate_non_grouped_collection(sample_collection) fov.validate_collection_label_fields( - sample_collection, in_field, (fol.Segmentation, fol.Heatmap) + sample_collection, + in_field, + (fol.Segmentation, fol.Detection, fol.Detections, fol.Heatmap), ) samples = sample_collection.select_fields(in_field) @@ -262,18 +285,33 @@ def import_segmentations( if label is None: continue - if isinstance(label, fol.Heatmap): - if label.map_path is not None: - del_path = label.map_path if delete_images else None - label.import_map(update=update) + if isinstance(label, fol.Segmentation): + if label.mask_path is not None: + del_path = label.mask_path if delete_images else None + label.import_mask(update=update) if del_path: etau.delete_file(del_path) - else: + elif isinstance(label, fol.Detection): if label.mask_path is not None: del_path = label.mask_path if delete_images else None label.import_mask(update=update) if del_path: etau.delete_file(del_path) + elif isinstance(label, fol.Detections): + for detection in label.detections: + if detection.mask_path is not None: + del_path = ( + detection.mask_path if delete_images else None + ) + detection.import_mask(update=update) + if del_path: + etau.delete_file(del_path) + elif isinstance(label, fol.Heatmap): + if label.map_path is not None: + del_path = label.map_path if delete_images else None + label.import_map(update=update) + if del_path: + etau.delete_file(del_path) def transform_segmentations( @@ -389,6 +427,9 @@ def segmentations_to_detections( out_field, mask_targets=None, mask_types="stuff", + output_dir=None, + rel_dir=None, + overwrite=False, progress=None, ): """Converts the semantic segmentations masks in the specified field of the @@ -423,6 +464,18 @@ def segmentations_to_detections( - ``"thing"`` if all classes are thing classes - a dict mapping pixel values (2D masks) or RGB hex strings (3D masks) to ``"stuff"`` or ``"thing"`` for each class + output_dir (None): an optional output directory in which to write + instance segmentation images. If none is provided, the instance + segmentations are stored in the database + rel_dir (None): an optional relative directory to strip from each input + filepath to generate a unique identifier that is joined with + ``output_dir`` to generate an output path for each instance + segmentation image. This argument allows for populating nested + subdirectories in ``output_dir`` that match the shape of the input + paths. The path is converted to an absolute path (if necessary) via + :func:`fiftyone.core.storage.normalize_path` + overwrite (False): whether to delete ``output_dir`` prior to exporting + if it exists progress (None): whether to render a progress bar (True/False), use the default value ``fiftyone.config.show_progress_bars`` (None), or a progress callback function to invoke instead @@ -438,6 +491,14 @@ def segmentations_to_detections( in_field, processing_frames = samples._handle_frame_field(in_field) out_field, _ = samples._handle_frame_field(out_field) + if overwrite and output_dir is not None: + etau.delete_dir(output_dir) + + if output_dir is not None: + filename_maker = fou.UniqueFilenameMaker( + output_dir=output_dir, rel_dir=rel_dir, idempotent=False + ) + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() @@ -449,9 +510,17 @@ def segmentations_to_detections( if label is None: continue - image[out_field] = label.to_detections( + detections = label.to_detections( mask_targets=mask_targets, mask_types=mask_types ) + if output_dir is not None: + for detection in detections.detections: + mask_path = filename_maker.get_output_path( + image.filepath, output_ext=".png" + ) + detection.export_mask(mask_path, update=True) + + image[out_field] = detections def instances_to_polylines( diff --git a/tests/unittests/import_export_tests.py b/tests/unittests/import_export_tests.py index 896429d8a7..d7f601a9e4 100644 --- a/tests/unittests/import_export_tests.py +++ b/tests/unittests/import_export_tests.py @@ -2218,6 +2218,115 @@ def _test_image_segmentation_fiftyone_dataset(self, dataset_type): dataset2.values("segmentations.mask_path"), ) + @drop_datasets + def test_instance_segmentation_fiftyone_dataset(self): + self._test_instance_segmentation_fiftyone_dataset( + fo.types.FiftyOneDataset + ) + + @drop_datasets + def test_instance_segmentation_legacy_fiftyone_dataset(self): + self._test_instance_segmentation_fiftyone_dataset( + fo.types.LegacyFiftyOneDataset + ) + + def _test_instance_segmentation_fiftyone_dataset(self, dataset_type): + dataset = self._make_dataset() + + # In-database instance segmentations + + export_dir = self._new_dir() + + dataset.export( + export_dir=export_dir, + dataset_type=dataset_type, + ) + + dataset2 = fo.Dataset.from_dir( + dataset_dir=export_dir, + dataset_type=dataset_type, + ) + + self.assertEqual(len(dataset), len(dataset2)) + self.assertEqual(dataset.count("detections.detections.mask_path"), 0) + self.assertEqual(dataset2.count("detections.detections.mask_path"), 0) + self.assertEqual( + dataset.count("detections.detections.mask"), + dataset2.count("detections.detections.mask"), + ) + + # Convert to on-disk instance segmentations + + segmentations_dir = self._new_dir() + + foul.export_segmentations(dataset, "detections", segmentations_dir) + + self.assertEqual(dataset.count("detections.detections.mask"), 0) + for mask_path in dataset.values("detections.detections[].mask_path"): + if mask_path is not None: + self.assertTrue(mask_path.startswith(segmentations_dir)) + + # On-disk instance segmentations + + export_dir = self._new_dir() + field_dir = os.path.join(export_dir, "fields", "detections.detections") + + dataset.export( + export_dir=export_dir, + dataset_type=dataset_type, + ) + + dataset2 = fo.Dataset.from_dir( + dataset_dir=export_dir, + dataset_type=dataset_type, + ) + + self.assertEqual(len(dataset), len(dataset2)) + self.assertEqual(dataset2.count("detections.detections.mask"), 0) + self.assertEqual( + dataset.count("detections.detections.mask_path"), + dataset2.count("detections.detections.mask_path"), + ) + + for mask_path in dataset2.values("detections.detections[].mask_path"): + if mask_path is not None: + self.assertTrue(mask_path.startswith(field_dir)) + + # On-disk instance segmentations (don't export media) + + export_dir = self._new_dir() + + dataset.export( + export_dir=export_dir, + dataset_type=dataset_type, + export_media=False, + ) + + dataset2 = fo.Dataset.from_dir( + dataset_dir=export_dir, + dataset_type=dataset_type, + ) + + self.assertEqual(len(dataset), len(dataset2)) + self.assertListEqual( + dataset.values("filepath"), + dataset2.values("filepath"), + ) + self.assertListEqual( + dataset.values("detections.detections[].mask_path"), + dataset2.values("detections.detections[].mask_path"), + ) + + # Convert to in-database instance segmentations + + foul.import_segmentations(dataset2, "detections") + + self.assertEqual(dataset2.count("detections.detections.mask_path"), 0) + self.assertEqual( + dataset2.count("detections.detections.mask"), + dataset.count("detections.detections.mask_path"), + ) + class DICOMDatasetTests(ImageDatasetTests): def _get_dcm_path(self): From 68eb682a998bebae98ff2a05f6cdffa3a87e98aa Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 12 Dec 2024 14:55:35 -0500 Subject: [PATCH 02/12] handle nested roots --- fiftyone/core/collections.py | 31 +++++++++++++++++++++---------- fiftyone/utils/data/exporters.py | 4 ++-- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/fiftyone/core/collections.py b/fiftyone/core/collections.py index 8200255fb4..65b6a78e39 100644 --- a/fiftyone/core/collections.py +++ b/fiftyone/core/collections.py @@ -10662,9 +10662,7 @@ def _handle_db_fields(self, paths, frames=False): db_fields_map = self._get_db_fields_map(frames=frames) return [db_fields_map.get(p, p) for p in paths] - def _get_media_fields( - self, include_filepath=True, whitelist=None, frames=False - ): + def _get_media_fields(self, whitelist=None, blacklist=None, frames=False): media_fields = {} if frames: @@ -10674,11 +10672,8 @@ def _get_media_fields( schema = self.get_field_schema(flat=True) app_media_fields = set(self._dataset.app_config.media_fields) - if include_filepath: - # 'filepath' should already be in set, but add it just in case - app_media_fields.add("filepath") - else: - app_media_fields.discard("filepath") + # 'filepath' should already be in set, but add it just in case + app_media_fields.add("filepath") for field_name, field in schema.items(): while isinstance(field, fof.ListField): @@ -10698,7 +10693,21 @@ def _get_media_fields( whitelist = {whitelist} media_fields = { - k: v for k, v in media_fields.items() if k in whitelist + k: v + for k, v in media_fields.items() + if any(w == k or k.startswith(w + ".") for w in whitelist) + } + + if blacklist is not None: + if etau.is_container(blacklist): + blacklist = set(blacklist) + else: + blacklist = {blacklist} + + media_fields = { + k: v + for k, v in media_fields.items() + if not any(w == k or k.startswith(w + ".") for w in blacklist) } return media_fields @@ -10714,7 +10723,9 @@ def _resolve_media_field(self, media_field): if leaf is not None: leaf = root + "." + leaf - if _media_field in (root, leaf): + if _media_field in (root, leaf) or root.startswith( + _media_field + "." + ): _resolved_field = leaf if leaf is not None else root if is_frame_field: _resolved_field = self._FRAMES_PREFIX + _resolved_field diff --git a/fiftyone/utils/data/exporters.py b/fiftyone/utils/data/exporters.py index 7a9b7da68e..475e4286b3 100644 --- a/fiftyone/utils/data/exporters.py +++ b/fiftyone/utils/data/exporters.py @@ -1894,7 +1894,7 @@ def log_collection(self, sample_collection): self._metadata["frame_fields"] = schema self._media_fields = sample_collection._get_media_fields( - include_filepath=False + blacklist="filepath", ) info = dict(sample_collection.info) @@ -2202,7 +2202,7 @@ def export_samples(self, sample_collection, progress=None): _sample_collection = sample_collection self._media_fields = sample_collection._get_media_fields( - include_filepath=False + blacklist="filepath" ) logger.info("Exporting samples...") From 833166132b9c2feb494e8f5e58f0ac1276d19735 Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 12 Dec 2024 17:09:40 -0500 Subject: [PATCH 03/12] handle list fields --- fiftyone/core/collections.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/fiftyone/core/collections.py b/fiftyone/core/collections.py index 65b6a78e39..2aafc32ee2 100644 --- a/fiftyone/core/collections.py +++ b/fiftyone/core/collections.py @@ -10712,9 +10712,9 @@ def _get_media_fields(self, whitelist=None, blacklist=None, frames=False): return media_fields - def _resolve_media_field(self, media_field): + def _parse_media_field(self, media_field): if media_field in self._dataset.app_config.media_fields: - return media_field + return media_field, None _media_field, is_frame_field = self._handle_frame_field(media_field) @@ -10730,7 +10730,13 @@ def _resolve_media_field(self, media_field): if is_frame_field: _resolved_field = self._FRAMES_PREFIX + _resolved_field - return _resolved_field + _list_fields = self._parse_field_name( + _resolved_field, auto_unwind=False + )[-2] + if _list_fields: + return _resolved_field, _list_fields[0] + + return _resolved_field, None raise ValueError("'%s' is not a valid media field" % media_field) From 73da3ae62932b9b09189aee2f1e59d827662dd7b Mon Sep 17 00:00:00 2001 From: Benjamin Kane Date: Thu, 12 Dec 2024 18:14:41 -0500 Subject: [PATCH 04/12] use buffers for hasFrame (#5264) --- app/packages/looker/src/lookers/utils.test.ts | 19 +++++++++++++++++++ app/packages/looker/src/lookers/utils.ts | 7 +++++++ app/packages/looker/src/lookers/video.ts | 9 ++------- 3 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 app/packages/looker/src/lookers/utils.test.ts create mode 100644 app/packages/looker/src/lookers/utils.ts diff --git a/app/packages/looker/src/lookers/utils.test.ts b/app/packages/looker/src/lookers/utils.test.ts new file mode 100644 index 0000000000..6c0d307e6e --- /dev/null +++ b/app/packages/looker/src/lookers/utils.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from "vitest"; +import type { Buffers } from "../state"; +import { hasFrame } from "./utils"; + +describe("looker utilities", () => { + it("determines frame availability given a buffer list", () => { + const BUFFERS: Buffers = [ + [1, 3], + [5, 25], + ]; + for (const frameNumber of [1, 10, 25]) { + expect(hasFrame(BUFFERS, frameNumber)).toBe(true); + } + + for (const frameNumber of [0, 4, 26]) { + expect(hasFrame(BUFFERS, frameNumber)).toBe(false); + } + }); +}); diff --git a/app/packages/looker/src/lookers/utils.ts b/app/packages/looker/src/lookers/utils.ts new file mode 100644 index 0000000000..ea645401f0 --- /dev/null +++ b/app/packages/looker/src/lookers/utils.ts @@ -0,0 +1,7 @@ +import type { Buffers } from "../state"; + +export const hasFrame = (buffers: Buffers, frameNumber: number) => { + return buffers.some( + ([start, end]) => start <= frameNumber && frameNumber <= end + ); +}; diff --git a/app/packages/looker/src/lookers/video.ts b/app/packages/looker/src/lookers/video.ts index 24ab04feb0..2fe24f7fab 100644 --- a/app/packages/looker/src/lookers/video.ts +++ b/app/packages/looker/src/lookers/video.ts @@ -19,6 +19,7 @@ import { addToBuffers, removeFromBuffers } from "../util"; import { AbstractLooker } from "./abstract"; import { type Frame, acquireReader, clearReader } from "./frame-reader"; import { LookerUtils, withFrames } from "./shared"; +import { hasFrame } from "./utils"; let LOOKER_WITH_READER: VideoLooker | null = null; @@ -394,13 +395,7 @@ export class VideoLooker extends AbstractLooker { } private hasFrame(frameNumber: number) { - if (frameNumber === this.firstFrameNumber) { - return this.firstFrame; - } - return ( - this.frames.has(frameNumber) && - this.frames.get(frameNumber)?.deref() !== undefined - ); + return hasFrame(this.state.buffers, frameNumber); } private getFrame(frameNumber: number) { From b0388a2f463f9bcdabc9c048c4c42b51c98d5c69 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 12 Dec 2024 17:54:42 -0600 Subject: [PATCH 05/12] use heuristic for detecting grayscale images --- .../looker/src/worker/canvas-decoder.test.ts | 36 +++++++++++++ .../looker/src/worker/canvas-decoder.ts | 50 +++++++++++++------ 2 files changed, 71 insertions(+), 15 deletions(-) create mode 100644 app/packages/looker/src/worker/canvas-decoder.test.ts diff --git a/app/packages/looker/src/worker/canvas-decoder.test.ts b/app/packages/looker/src/worker/canvas-decoder.test.ts new file mode 100644 index 0000000000..57ce3de37b --- /dev/null +++ b/app/packages/looker/src/worker/canvas-decoder.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; +import { isGrayscale } from "./canvas-decoder"; + +const createData = ( + pixels: Array<[number, number, number, number]> +): Uint8ClampedArray => { + return new Uint8ClampedArray(pixels.flat()); +}; + +describe("isGrayscale", () => { + it("should return true for a perfectly grayscale image", () => { + // all pixels are (100, 100, 100, 255) + const data = createData(Array(100).fill([100, 100, 100, 255])); + expect(isGrayscale(data)).toBe(true); + }); + + it("should return false if alpha is not 255", () => { + // one pixel with alpha < 255 + const data = createData([ + [100, 100, 100, 255], + [100, 100, 100, 254], + ...Array(98).fill([100, 100, 100, 255]), + ]); + expect(isGrayscale(data)).toBe(false); + }); + + it("should return false if any pixel is not grayscale", () => { + // one pixel differs in g channel + const data = createData([ + [100, 100, 100, 255], + [100, 101, 100, 255], + ...Array(98).fill([100, 100, 100, 255]), + ]); + expect(isGrayscale(data)).toBe(false); + }); +}); diff --git a/app/packages/looker/src/worker/canvas-decoder.ts b/app/packages/looker/src/worker/canvas-decoder.ts index a394554b74..c69da17500 100644 --- a/app/packages/looker/src/worker/canvas-decoder.ts +++ b/app/packages/looker/src/worker/canvas-decoder.ts @@ -1,5 +1,26 @@ import { OverlayMask } from "../numpy"; +/** + * Checks if the given pixel data is grayscale by sampling a subset of pixels. + * If the image is grayscale, the R, G, and B channels will be equal for CHECKS iteration, + * and the alpha channel will always be 255. + * + * Note: this is a very useful heuristic but still doesn't guarantee accuracy. + */ +export const isGrayscale = (data: Uint8ClampedArray, checks = 500): boolean => { + const totalPixels = data.length / 4; + const step = Math.max(1, Math.floor(totalPixels / checks)); + + for (let p = 0; p < totalPixels; p += step) { + const i = p * 4; + const [r, g, b, a] = [data[i], data[i + 1], data[i + 2], data[i + 3]]; + if (a !== 255 || r !== g || g !== b) { + return false; + } + } + return true; +}; + /** * Decodes a given image source into an OverlayMask using an OffscreenCanvas */ @@ -12,25 +33,24 @@ export const decodeWithCanvas = async (blob: ImageBitmapSource) => { const ctx = canvas.getContext("2d"); ctx.drawImage(imageBitmap, 0, 0); + imageBitmap.close(); const imageData = ctx.getImageData(0, 0, width, height); + const channels = isGrayscale(imageData.data) ? 1 : 4; - const numChannels = imageData.data.length / (width * height); - - const overlayData = { - width, - height, - data: imageData.data, - channels: numChannels, - }; - - // dispose - imageBitmap.close(); + if (channels === 1) { + // get rid of the G, B, and A channels, new buffer will be 1/4 the size + const data = new Uint8ClampedArray(width * height); + for (let i = 0; i < data.length; i++) { + data[i] = imageData.data[i * 4]; + } + imageData.data.set(data); + } return { - buffer: overlayData.data.buffer, - channels: numChannels, - arrayType: overlayData.data.constructor.name as OverlayMask["arrayType"], - shape: [overlayData.height, overlayData.width], + buffer: imageData.data.buffer, + channels, + arrayType: "Uint8ClampedArray", + shape: [height, width], } as OverlayMask; }; From e7f3eddb408c142b59233571c3a7319c6d1c0fcb Mon Sep 17 00:00:00 2001 From: topher Date: Fri, 13 Dec 2024 00:01:45 +0000 Subject: [PATCH 06/12] bump version after release branch creation --- fiftyone/constants.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fiftyone/constants.py b/fiftyone/constants.py index 18ca73f47a..fa116a2fc0 100644 --- a/fiftyone/constants.py +++ b/fiftyone/constants.py @@ -42,7 +42,7 @@ # This setting may be ``None`` if this client has no compatibility with other # versions # -COMPATIBLE_VERSIONS = ">=0.19,<1.3" +COMPATIBLE_VERSIONS = ">=0.19,<1.4" # Package metadata _META = metadata("fiftyone") diff --git a/setup.py b/setup.py index 1009d750c3..544099b830 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ from setuptools import setup, find_packages -VERSION = "1.2.0" +VERSION = "1.3.0" def get_version(): From d570937910e8136f4f965b53b27900b7dd244ac9 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 12 Dec 2024 18:11:45 -0600 Subject: [PATCH 07/12] add 1% min --- .../looker/src/worker/canvas-decoder.test.ts | 13 ++++++++++--- app/packages/looker/src/worker/canvas-decoder.ts | 8 ++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/app/packages/looker/src/worker/canvas-decoder.test.ts b/app/packages/looker/src/worker/canvas-decoder.test.ts index 57ce3de37b..427b3c6131 100644 --- a/app/packages/looker/src/worker/canvas-decoder.test.ts +++ b/app/packages/looker/src/worker/canvas-decoder.test.ts @@ -9,13 +9,11 @@ const createData = ( describe("isGrayscale", () => { it("should return true for a perfectly grayscale image", () => { - // all pixels are (100, 100, 100, 255) const data = createData(Array(100).fill([100, 100, 100, 255])); expect(isGrayscale(data)).toBe(true); }); it("should return false if alpha is not 255", () => { - // one pixel with alpha < 255 const data = createData([ [100, 100, 100, 255], [100, 100, 100, 254], @@ -25,7 +23,6 @@ describe("isGrayscale", () => { }); it("should return false if any pixel is not grayscale", () => { - // one pixel differs in g channel const data = createData([ [100, 100, 100, 255], [100, 101, 100, 255], @@ -33,4 +30,14 @@ describe("isGrayscale", () => { ]); expect(isGrayscale(data)).toBe(false); }); + + it("should detect a non-grayscale pixel placed deep enough to ensure at least 1% of pixels are checked", () => { + // large image: 100,000 pixels. 1% of 100,000 is 1,000. + // the function will check at least 1,000 pixels. + // place a non-grayscale pixel after 800 pixels. + const pixels = Array(100000).fill([50, 50, 50, 255]); + pixels[800] = [50, 51, 50, 255]; // this is within the first 1% of pixels + const data = createData(pixels); + expect(isGrayscale(data)).toBe(false); + }); }); diff --git a/app/packages/looker/src/worker/canvas-decoder.ts b/app/packages/looker/src/worker/canvas-decoder.ts index c69da17500..52d01b5d7b 100644 --- a/app/packages/looker/src/worker/canvas-decoder.ts +++ b/app/packages/looker/src/worker/canvas-decoder.ts @@ -2,13 +2,13 @@ import { OverlayMask } from "../numpy"; /** * Checks if the given pixel data is grayscale by sampling a subset of pixels. - * If the image is grayscale, the R, G, and B channels will be equal for CHECKS iteration, + * The function will check at least 500 pixels or 1% of all pixels, whichever is larger. + * If the image is grayscale, the R, G, and B channels will be equal for all sampled pixels, * and the alpha channel will always be 255. - * - * Note: this is a very useful heuristic but still doesn't guarantee accuracy. */ -export const isGrayscale = (data: Uint8ClampedArray, checks = 500): boolean => { +export const isGrayscale = (data: Uint8ClampedArray): boolean => { const totalPixels = data.length / 4; + const checks = Math.max(500, Math.floor(totalPixels * 0.01)); const step = Math.max(1, Math.floor(totalPixels / checks)); for (let p = 0; p < totalPixels; p += step) { From e48511db7b6ca0b4b18827e862c863445ca38957 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 12 Dec 2024 18:13:13 -0600 Subject: [PATCH 08/12] add clarifying comments --- app/packages/looker/src/worker/canvas-decoder.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/packages/looker/src/worker/canvas-decoder.ts b/app/packages/looker/src/worker/canvas-decoder.ts index 52d01b5d7b..390ace2a04 100644 --- a/app/packages/looker/src/worker/canvas-decoder.ts +++ b/app/packages/looker/src/worker/canvas-decoder.ts @@ -36,6 +36,8 @@ export const decodeWithCanvas = async (blob: ImageBitmapSource) => { imageBitmap.close(); const imageData = ctx.getImageData(0, 0, width, height); + + // for nongrayscale images, channel is guaranteed to be 4 (RGBA) const channels = isGrayscale(imageData.data) ? 1 : 4; if (channels === 1) { From d83c00ab0d4590c2a0541a85558a104a35cdcd35 Mon Sep 17 00:00:00 2001 From: Sashank Aryal Date: Thu, 12 Dec 2024 19:41:45 -0600 Subject: [PATCH 09/12] fix rgb mask recoloring bug --- app/packages/looker/src/worker/painter.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/packages/looker/src/worker/painter.ts b/app/packages/looker/src/worker/painter.ts index 2e9f5a3ea3..6730d90cac 100644 --- a/app/packages/looker/src/worker/painter.ts +++ b/app/packages/looker/src/worker/painter.ts @@ -278,7 +278,14 @@ export const PainterFactory = (requestColor) => ({ const isRgbMaskTargets_ = isRgbMaskTargets(maskTargets); - if (maskData.channels > 2) { + // we have an additional guard for targets length = new image buffer byte length + // because we reduce the RGBA mask into a grayscale mask in first load for + // performance reasons + // For subsequent mask updates, the maskData.buffer is already a single channel + if ( + maskData.channels === 4 && + targets.length === label.mask.image.byteLength + ) { for (let i = 0; i < overlay.length; i++) { const [r, g, b] = getRgbFromMaskData(targets, maskData.channels, i); From 305377967ab16fde74febb01562cc437320d5ff7 Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Fri, 13 Dec 2024 10:32:29 -0500 Subject: [PATCH 10/12] Sort Shuffle Stage in FfityOne App (#5270) * sort * also here --------- Co-authored-by: Justin Newberry --- fiftyone/__public__.py | 2 +- fiftyone/core/stages.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fiftyone/__public__.py b/fiftyone/__public__.py index f343bdf718..1ccd9d15a5 100644 --- a/fiftyone/__public__.py +++ b/fiftyone/__public__.py @@ -215,7 +215,6 @@ MatchLabels, MatchTags, Mongo, - Shuffle, Select, SelectBy, SelectFields, @@ -224,6 +223,7 @@ SelectGroupSlices, SelectLabels, SetField, + Shuffle, Skip, SortBy, SortBySimilarity, diff --git a/fiftyone/core/stages.py b/fiftyone/core/stages.py index eb29d5a942..a71272bf15 100644 --- a/fiftyone/core/stages.py +++ b/fiftyone/core/stages.py @@ -8628,7 +8628,6 @@ def repr_ViewExpression(self, expr, level): MatchLabels, MatchTags, Mongo, - Shuffle, Select, SelectBy, SelectFields, @@ -8637,6 +8636,7 @@ def repr_ViewExpression(self, expr, level): SelectGroupSlices, SelectLabels, SetField, + Shuffle, Skip, SortBy, SortBySimilarity, From 79b83950344bad5bf43455be50ba31c006db9a9e Mon Sep 17 00:00:00 2001 From: afoley587 <54959686+afoley587@users.noreply.github.com> Date: Fri, 13 Dec 2024 11:12:25 -0500 Subject: [PATCH 11/12] fix(ci): AS-359 Update Ubuntu24 Binaries For MongoDB (#5269) --- .github/workflows/test.yml | 2 +- package/db/setup.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e21ef73509..c85454979b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,7 @@ on: workflow_call jobs: test-app: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/package/db/setup.py b/package/db/setup.py index 63c5ac5dfe..ccf07ba9ef 100644 --- a/package/db/setup.py +++ b/package/db/setup.py @@ -124,8 +124,8 @@ "x86_64": "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu2204-7.0.4.tgz", }, "24": { - "aarch64": "https://fastdl.mongodb.org/linux/mongodb-linux-aarch64-ubuntu2204-7.0.4.tgz", - "x86_64": "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu2204-7.0.4.tgz", + "aarch64": "https://fastdl.mongodb.org/linux/mongodb-linux-aarch64-ubuntu2404-8.0.4.tgz", + "x86_64": "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu2404-8.0.4.tgz", }, }, } @@ -175,7 +175,7 @@ def _get_download(): MONGODB_BINARIES = ["mongod"] -VERSION = "1.1.7" +VERSION = "1.2.0" def get_version(): From 8da1243d81384d0e74adda97e2a64d25055c18fe Mon Sep 17 00:00:00 2001 From: topher Date: Fri, 13 Dec 2024 11:49:47 -0500 Subject: [PATCH 12/12] Sort Shuffle Stage in FfityOne App (#5270) (#5272) * sort * also here --------- Co-authored-by: Justin Newberry Co-authored-by: Justin Newberry --- fiftyone/__public__.py | 2 +- fiftyone/core/stages.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fiftyone/__public__.py b/fiftyone/__public__.py index f343bdf718..1ccd9d15a5 100644 --- a/fiftyone/__public__.py +++ b/fiftyone/__public__.py @@ -215,7 +215,6 @@ MatchLabels, MatchTags, Mongo, - Shuffle, Select, SelectBy, SelectFields, @@ -224,6 +223,7 @@ SelectGroupSlices, SelectLabels, SetField, + Shuffle, Skip, SortBy, SortBySimilarity, diff --git a/fiftyone/core/stages.py b/fiftyone/core/stages.py index eb29d5a942..a71272bf15 100644 --- a/fiftyone/core/stages.py +++ b/fiftyone/core/stages.py @@ -8628,7 +8628,6 @@ def repr_ViewExpression(self, expr, level): MatchLabels, MatchTags, Mongo, - Shuffle, Select, SelectBy, SelectFields, @@ -8637,6 +8636,7 @@ def repr_ViewExpression(self, expr, level): SelectGroupSlices, SelectLabels, SetField, + Shuffle, Skip, SortBy, SortBySimilarity,