diff --git a/app/packages/core/src/components/Modal/ImaVidLooker.tsx b/app/packages/core/src/components/Modal/ImaVidLooker.tsx index 7af9d2b911..6707202a8a 100644 --- a/app/packages/core/src/components/Modal/ImaVidLooker.tsx +++ b/app/packages/core/src/components/Modal/ImaVidLooker.tsx @@ -175,16 +175,23 @@ export const ImaVidLookerReact = React.memo( return; } + setPlayHeadState({ name: timelineName, state: "buffering" }); + imaVidLookerRef.current.frameStoreController.enqueueFetch( unprocessedBufferRange ); + imaVidLookerRef.current.frameStoreController.resumeFetch(); + return new Promise((resolve) => { const fetchMoreListener = (e: CustomEvent) => { if ( e.detail.id === imaVidLookerRef.current.frameStoreController.key ) { if (storeBufferManager.containsRange(unprocessedBufferRange)) { + // todo: change playhead state in setFrameNumberAtom and not here + // if done here, store ref to last playhead status + setPlayHeadState({ name: timelineName, state: "paused" }); resolve(); window.removeEventListener( "fetchMore", @@ -250,6 +257,7 @@ export const ImaVidLookerReact = React.memo( registerOnPauseCallback, registerOnPlayCallback, registerOnSeekCallbacks, + setPlayHeadState, subscribe, } = useCreateTimeline({ name: timelineName, @@ -325,6 +333,7 @@ export const ImaVidLookerReact = React.memo( position: "relative", display: "flex", flexDirection: "column", + overflowX: "hidden", }} >
{ } paintImageOnCanvas(image: HTMLImageElement) { - this.ctx.setTransform(1, 0, 0, 1, 0, 0); + this.ctx?.setTransform(1, 0, 0, 1, 0, 0); - this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); + this.ctx?.clearRect(0, 0, this.canvas.width, this.canvas.height); - this.ctx.drawImage(image, 0, 0); + this.ctx?.drawImage(image, 0, 0); } async skipAndTryAgain(frameNumberToDraw: number, animate: boolean) { diff --git a/app/packages/looker/src/overlays/heatmap.ts b/app/packages/looker/src/overlays/heatmap.ts index 51960ed149..1742477dde 100644 --- a/app/packages/looker/src/overlays/heatmap.ts +++ b/app/packages/looker/src/overlays/heatmap.ts @@ -10,13 +10,14 @@ import { import { ARRAY_TYPES, OverlayMask, TypedArray } from "../numpy"; import { BaseState, Coordinates } from "../state"; import { isFloatArray } from "../util"; +import { clampedIndex } from "../worker/painter"; import { BaseLabel, CONTAINS, - isShown, Overlay, PointInfo, SelectData, + isShown, } from "./base"; import { sizeBytes, strokeCanvasRect, t } from "./util"; @@ -204,12 +205,13 @@ export default class HeatmapOverlay } if (state.options.coloring.by === "value") { - const index = Math.round( - (Math.max(value - start, 0) / (stop - start)) * - (state.options.coloring.scale.length - 1) + const index = clampedIndex( + value, + start, + stop, + state.options.coloring.scale.length ); - - return get32BitColor(state.options.coloring.scale[index]); + return index < 0 ? 0 : get32BitColor(state.options.coloring.scale[index]); } const color = getColor( @@ -219,9 +221,9 @@ export default class HeatmapOverlay ); const max = Math.max(Math.abs(start), Math.abs(stop)); - value = Math.min(max, Math.abs(value)) / max; + const result = Math.min(max, Math.abs(value)) / max; - return get32BitColor(color, value / max); + return get32BitColor(color, result / max); } private getTarget(state: Readonly): number { diff --git a/app/packages/looker/src/worker/painter.test.ts b/app/packages/looker/src/worker/painter.test.ts index e9043c4a14..f4353857e7 100644 --- a/app/packages/looker/src/worker/painter.test.ts +++ b/app/packages/looker/src/worker/painter.test.ts @@ -63,3 +63,13 @@ describe("filter resolves correctly", () => { ).toBeUndefined(); }); }); + +describe("heatmap utils", () => { + it("clamps for heatmaps", async () => { + // A value below a heatmap range returns -1 + expect(painter.clampedIndex(1, 2, 3, 4)).toBe(-1); + + // A value above a heatmap range return the max + expect(painter.clampedIndex(4, 2, 3, 4)).toBe(3); + }); +}); diff --git a/app/packages/looker/src/worker/painter.ts b/app/packages/looker/src/worker/painter.ts index 9b3be37354..66df8da0a1 100644 --- a/app/packages/looker/src/worker/painter.ts +++ b/app/packages/looker/src/worker/painter.ts @@ -206,23 +206,28 @@ export const PainterFactory = (requestColor) => ({ } // 0 is background image - if (value !== 0) { - let r; - if (coloring.by === COLOR_BY.FIELD) { - color = - fieldSetting?.fieldColor ?? - (await requestColor(coloring.pool, coloring.seed, field)); - - r = get32BitColor(color, Math.min(max, Math.abs(value)) / max); - } else { - const index = Math.round( - (Math.max(value - start, 0) / (stop - start)) * (scale.length - 1) - ); - r = get32BitColor(scale[index]); + if (value === 0) { + continue; + } + let r: number; + if (coloring.by === COLOR_BY.FIELD) { + color = + fieldSetting?.fieldColor ?? + (await requestColor(coloring.pool, coloring.seed, field)); + + r = get32BitColor(color, Math.min(max, Math.abs(value)) / max); + } else { + const index = clampedIndex(value, start, stop, scale.length); + + if (index < 0) { + // values less than range start are background + continue; } - overlay[i] = r; + r = get32BitColor(scale[index]); } + + overlay[i] = r; } }, Segmentation: async ( @@ -386,8 +391,23 @@ export const convertToHex = (color: string) => const convertMaskColorsToObject = (array: MaskColorInput[]) => { const result = {}; if (!array) return {}; - array.forEach((item) => { + for (const item of array) { result[item.intTarget.toString()] = item.color; - }); + } return result; }; + +export const clampedIndex = ( + value: number, + start: number, + stop: number, + length: number +) => { + if (value < start) { + return -1; + } + const clamped = Math.min(value, stop); + return Math.round( + (Math.max(clamped - start, 0) / (stop - start)) * (length - 1) + ); +}; diff --git a/app/packages/playback/src/lib/state.ts b/app/packages/playback/src/lib/state.ts index 66702627dc..ba410c67f3 100644 --- a/app/packages/playback/src/lib/state.ts +++ b/app/packages/playback/src/lib/state.ts @@ -13,6 +13,7 @@ import { } from "./constants"; export type PlayheadState = + | "buffering" | "playing" | "paused" | "waitingToPlay" diff --git a/app/packages/playback/src/lib/use-create-timeline.ts b/app/packages/playback/src/lib/use-create-timeline.ts index b70719b009..a19694b65d 100644 --- a/app/packages/playback/src/lib/use-create-timeline.ts +++ b/app/packages/playback/src/lib/use-create-timeline.ts @@ -157,6 +157,10 @@ export const useCreateTimeline = ( return; } + if (playHeadStateRef.current === "buffering") { + return; + } + setPlayHeadState({ name: timelineName, state: "playing" }); if (onPlayListenerRef.current) { onPlayListenerRef.current(); @@ -364,6 +368,10 @@ export const useCreateTimeline = ( const key = e.key.toLowerCase(); if (key === " ") { + if (playHeadState === "buffering") { + return; + } + if (playHeadState === "paused") { play(); } else { @@ -448,6 +456,10 @@ export const useCreateTimeline = ( * Re-render all subscribers of the timeline with current frame number. */ refresh, + /** + * Set the playhead state of the timeline. + */ + setPlayHeadState, /** * Subscribe to the timeline. */ diff --git a/app/packages/playback/src/views/svgs/buffering.svg b/app/packages/playback/src/views/svgs/buffering.svg index a894767a0d..bbaa39ff17 100644 --- a/app/packages/playback/src/views/svgs/buffering.svg +++ b/app/packages/playback/src/views/svgs/buffering.svg @@ -1,3 +1,8 @@ + + - + + + \ No newline at end of file diff --git a/docs/scripts/make_model_zoo_docs.py b/docs/scripts/make_model_zoo_docs.py index 637819b1a6..21f1e77424 100644 --- a/docs/scripts/make_model_zoo_docs.py +++ b/docs/scripts/make_model_zoo_docs.py @@ -90,13 +90,9 @@ import fiftyone.zoo as foz {% if 'segment-anything' in name and 'video' in name %} from fiftyone import ViewField as F -{% endif %} - -{% if 'med-sam' in name %} +{% elif 'med-sam' in name %} from fiftyone import ViewField as F from fiftyone.utils.huggingface import load_from_hub - - dataset = load_from_hub("Voxel51/BTCV-CT-as-video-MedSAM2-dataset")[:2] {% endif %} {% if 'imagenet' in name %} @@ -117,6 +113,7 @@ .save() ) {% elif 'med-sam' in name %} + dataset = load_from_hub("Voxel51/BTCV-CT-as-video-MedSAM2-dataset")[:2] # Retaining detections from a single frame in the middle # Note that SAM2 only propagates segmentation masks forward in a video @@ -126,7 +123,6 @@ .set_field("frames.gt_detections", None) .save() ) - {% else %} dataset = foz.load_zoo_dataset( "coco-2017", diff --git a/docs/source/deprecation.rst b/docs/source/deprecation.rst index fa8c7f1315..277b102b3b 100644 --- a/docs/source/deprecation.rst +++ b/docs/source/deprecation.rst @@ -5,19 +5,23 @@ FiftyOne Deprecation Notices .. default-role:: code +.. _deprecation-fiftyone-desktop: + FiftyOne Desktop ---------------- -*Support ended with 0.25.0* +*Support ended with FiftyOne 0.25.0* -A compatible `fiftyone-desktop https://pypi.org/project/fiftyone-desktop/`_ +A compatible `fiftyone-desktop `_ package is no longer available as of `fiftyone==0.25.0`. Chromium-based browsers, Firefox, or a :ref:`notebook ` environment are recommended for the best FiftyOne experience. +.. _deprecation-python-3.8: + Python 3.8 ---------- -*Support Ended October 2024* +*Support ended October 1, 2024* `Python 3.8 `_ transitions to `end-of-life` effective October of 2024. FiftyOne releases after diff --git a/docs/source/images/datasets/quickstart-video-summary-fields.gif b/docs/source/images/datasets/quickstart-video-summary-fields.gif new file mode 100644 index 0000000000..cd424cd996 Binary files /dev/null and b/docs/source/images/datasets/quickstart-video-summary-fields.gif differ diff --git a/docs/source/index.rst b/docs/source/index.rst index aefd20e103..1990c62927 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -181,6 +181,11 @@ learn how: :image_src: https://voxel51.com/images/integrations/v7-128.png :image_title: V7 +.. customimagelink:: + :image_link: https://github.com/segments-ai/segments-voxel51-plugin + :image_src: https://voxel51.com/images/integrations/segments-128.png + :image_title: Segments + .. customimagelink:: :image_link: integrations/labelbox.html :image_src: https://voxel51.com/images/integrations/labelbox-128.png diff --git a/docs/source/plugins/developing_plugins.rst b/docs/source/plugins/developing_plugins.rst index 6ac02fc7f9..68b9a1c12a 100644 --- a/docs/source/plugins/developing_plugins.rst +++ b/docs/source/plugins/developing_plugins.rst @@ -1774,12 +1774,11 @@ in the App. Panels can be defined in either Python or JS, and FiftyOne comes with a number of :ref:`builtin panels ` for common tasks. -Depending on the ``surfaces`` panel config, panels can be scoped to either -the grid or the modal. You can open these panels from the "+" menu, which -is available in both the grid and modal views. Whereas grid panels enable -extensibility at the macro level, allowing you to work with entire datasets, -modal panels provide extensibility at the micro level, focusing on individual -samples and scenarios. +Panels can be scoped to the App's grid view or modal view via their +:ref:`config `. Grid panels enable extensibility at the macro +level, allowing you to work with entire datasets or views, while modal panels +provide extensibility at the micro level, focusing on individual samples and +scenarios. Panels, like :ref:`operators `, can make use of the :mod:`fiftyone.operators.types` module and the @@ -1838,10 +1837,9 @@ subsequent sections. # Whether to allow multiple instances of the panel to be opened allow_multiple=False, - # Whether the panel should be available in the grid view - # modal view, or both + # Whether the panel should be available in the grid, modal, or both # Possible values: "grid", "modal", "grid modal" - surfaces="grid modal" # default = "grid" + surfaces="grid", # default = "grid" # Markdown-formatted text that describes the panel. This is # rendererd in a tooltip when the help icon in the panel @@ -2138,7 +2136,7 @@ Panel config Every panel must define a :meth:`config ` property that -defines its name, display name, and other optional metadata about its +defines its name, display name, surfaces, and other optional metadata about its behavior: .. code-block:: python @@ -2162,8 +2160,26 @@ behavior: # Whether to allow multiple instances of the panel to be opened allow_multiple=False, + + # Whether the panel should be available in the grid, modal, or both + # Possible values: "grid", "modal", "grid modal" + surfaces="grid", # default = "grid" + + # Markdown-formatted text that describes the panel. This is + # rendererd in a tooltip when the help icon in the panel + # title is hovered over + help_markdown="A description of the panel", ) +The ``surfaces`` key defines the panel's scope: + +- Grid panels can be accessed from the ``+`` button in the App's + :ref:`grid view `, which allows you to build macro + experiences that work with entire datasets or views +- Modal panels can be accessed from the ``+`` button in the App's + :ref:`modal view `, which allows you to build interactions + that focus on individual samples and scenarios + .. _panel-execution-context: Execution context diff --git a/docs/source/release-notes.rst b/docs/source/release-notes.rst index 1ff8649d45..ec04235961 100644 --- a/docs/source/release-notes.rst +++ b/docs/source/release-notes.rst @@ -3,14 +3,125 @@ FiftyOne Release Notes .. default-role:: code +FiftyOne Teams 2.1.0 +-------------------- +*Released October 1, 2024* + +Includes all updates from :ref:`FiftyOne 1.0.0 `, plus: + +- Super admins can now migrate their deployments to + :ref:`Internal Mode ` via the + :ref:`Super Admin UI ` +- Added support for sending user invitations in + :ref:`Internal Mode ` +- Optimized performance of the :ref:`dataset page ` +- Fixed a BSON serialization bug that could cause errors when cloning or + exporting certain dataset views from the Teams UI + +.. _release-notes-v1.0.0: + +FiftyOne 1.0.0 +-------------- +*Released October 1, 2024* + +What's New + +- The `FiftyOne Brain `_ is now + fully open source. Contributions are welcome! +- Added :ref:`Modal Panels `, bringing the ability to develop and + use panels in the App's sample modal + `#4625 `_ +- All datasets now have :ref:`automatically populated ` + `created_at` and `last_modified_at` fields on their samples and frames + `#4597 `_ +- Added support for loading + :ref:`remotely-sourced zoo datasets ` whose + download/preparation instructions are stored in GitHub or public URLs + `#4752 `_ +- Added support for loading + :ref:`remotely-sourced zoo models ` whose definitions are + stored in GitHub or public URLs + `#4786 `_ +- Added `Med-SAM2 `_ to the + :ref:`model zoo `! + `#4733 `_, + `#4828 `_ + +App + +- Added dozens of :ref:`builtin operators ` for performing + common operations directly from the App + `#4830 `_ +- Label overlays in the grid are now scaled proportionally to grid zoom + `#4747 `_ +- Improved support for visualizing and filtering |DynamicEmbeddedDocument| list + fields + `#4833 `_ +- Added a new timeline API for synchronizing playback of multiple modal panels + `#4772 `_ +- Improved UI, documentation, and robustness when working with + :ref:`custom color schemes ` + `#4763 `_ +- Fixed a bug where the active group slice was not being persisted when + navigating between groups in the modal + `#4836 `_ +- Fixed a bug when selecting samples in grouped datasets in the modal + `#4789 `_ +- Fixed :ref:`heatmaps ` rendering for values outside of the `range` + attribute `#4865 `_ + +Core + +- Added support for creating :ref:`summary fields ` to optimize + queries on large datasets with many objects + `#4765 `_ +- Dataset fields now have automatically populated `created_at` attributes + `#4730 `_ +- Upgraded the + :meth:`delete_samples() ` + and :meth:`clear_frames() ` + methods to support bulk deletions of 100k+ samples/frames + `#4787 `_ +- The :meth:`default_sidebar_groups() ` + method now correctly handles datetime fields + `#4815 `_ +- Fixed an off-by-one error when converting semantic segmentations to/from + instance segmentations + `#4826 `_ +- Protect against infinitely growing content size batchers + `#4806 `_ +- Removed the deprecated `remove_sample()` and `remove_samples()` methods from + the |Dataset| class + `#4832 `_ +- Deprecated :ref:`Python 3.8 support ` + +Plugins + +- Added + :meth:`ctx.group_slice ` + to the operator execution context + `#4850 `_ +- Added + :meth:`set_group_slice() ` + to the operator execution context + `#4844 `_ +- Improved styling for :class:`GridView ` + components + `#4764 `_ +- A loading error is now displayed in the actions row when operators with + :ref:`placements ` fail to load + `#4714 `_ +- Ensure the App loads when plugins fail to load + `#4769 `_ + +.. _release-notes-v0.25.2: FiftyOne 0.25.2 --------------- *Released September 19, 2024* -* Require `pymongo<4.9` to fix database connections -* Require `pydicom<3` for :ref:`DICOM datasets ` - +- Require `pymongo<4.9` to fix database connections +- Require `pydicom<3` for :ref:`DICOM datasets ` FiftyOne Teams 2.0.1 -------------------- diff --git a/docs/source/user_guide/using_datasets.rst b/docs/source/user_guide/using_datasets.rst index 33d943b1b8..75cdf7fd93 100644 --- a/docs/source/user_guide/using_datasets.rst +++ b/docs/source/user_guide/using_datasets.rst @@ -1635,16 +1635,14 @@ editable at any time: Summary fields -------------- -Summary fields allow you to efficiently perform queries where directly querying -the underlying field is prohibitively slow due to the number of objects/frames -in the field. +Summary fields allow you to efficiently perform queries on large datasets where +directly querying the underlying field is prohibitively slow due to the number +of objects/frames in the field. -For example, as we'll see below, summary fields are useful for retrieving -samples in a video dataset that contain specific values of interest in at least -one frame. - -Use :meth:`create_summary_field() ` -to create a summary field for a given input field path: +For example, suppose you're working on a +:ref:`video dataset ` with frame-level objects, and you're +interested in finding videos that contain specific classes of interest, eg +`person`, in at least one frame: .. code-block:: python :linenos: @@ -1656,6 +1654,25 @@ to create a summary field for a given input field path: dataset = foz.load_zoo_dataset("quickstart-video") dataset.set_field("frames.detections.detections.confidence", F.rand()).save() + session = fo.launch_app(dataset) + +.. image:: /images/datasets/quickstart-video.gif + :alt: quickstart-video + :align: center + +One approach is to directly query the frame-level field (`frames.detections` +in this case) in the App's sidebar. However, when the dataset is large, such +queries are inefficient, as they cannot +:ref:`leverage indexes ` and thus require full +collection scans over all frames to retrieve the relevant samples. + +A more efficient approach is to first use +:meth:`create_summary_field() ` +to summarize the relevant input field path(s): + +.. code-block:: python + :linenos: + # Generate a summary field for object labels field_name = dataset.create_summary_field("frames.detections.detections.label") @@ -1666,15 +1683,6 @@ to create a summary field for a given input field path: # Generate a summary field for [min, max] confidences dataset.create_summary_field("frames.detections.detections.confidence") -.. note:: - - Summary fields are :ref:`read-only `, as they are - implicitly derived from the contents of their source field and are not - intended to be directly modified. - - They are also :ref:`indexed ` by default, so - filtering them :ref:`in the App ` is performant. - Summary fields can be generated for sample-level and frame-level fields, and the input fields can be either categorical or numeric: @@ -1753,6 +1761,28 @@ the input fields can be either categorical or numeric: ] """ +As the above examples illustrate, summary fields allow you to encode various +types of information at the sample-level that you can directly query to find +samples that contain specific values. + +Moreover, summary fields are :ref:`indexed ` by default +and the App can natively leverage these indexes to provide performant +filtering: + +.. image:: /images/datasets/quickstart-video-summary-fields.gif + :alt: quickstart-video-summary-fields + :align: center + +.. note:: + + Summary fields are automatically added to a `summaries` + :ref:`sidebar group ` in the App for + easy access and organization. + + They are also :ref:`read-only ` by default, as they are + implicitly derived from the contents of their source field and are not + intended to be directly modified. + You can use :meth:`list_summary_fields() ` to list the names of the summary fields on your dataset: diff --git a/fiftyone/operators/builtin.py b/fiftyone/operators/builtin.py index bffe9d6565..12d4ee76bb 100644 --- a/fiftyone/operators/builtin.py +++ b/fiftyone/operators/builtin.py @@ -1940,8 +1940,9 @@ def resolve_input(self, ctx): ) # @todo infer this automatically from current App spaces - spaces_prop = inputs.str( + spaces_prop = inputs.oneof( "spaces", + [types.String(), types.Object()], default=None, required=True, label="Spaces", @@ -1955,7 +1956,7 @@ def resolve_input(self, ctx): spaces = ctx.params.get("spaces", None) if spaces is not None: try: - fo.Space.from_json(spaces) + _parse_spaces(spaces) except: spaces_prop.invalid = True spaces_prop.error_message = "Invalid workspace definition" @@ -1978,10 +1979,7 @@ def execute(self, ctx): color = ctx.params.get("color", None) spaces = ctx.params.get("spaces", None) - if isinstance(spaces, dict): - spaces = fo.Space.from_dict(spaces) - else: - spaces = fo.Space.from_json(spaces) + spaces = _parse_spaces(spaces) ctx.dataset.save_workspace( name, @@ -2294,6 +2292,12 @@ def _get_non_default_frame_fields(dataset): return schema +def _parse_spaces(spaces): + if isinstance(spaces, dict): + return fo.Space.from_dict(spaces) + return fo.Space.from_json(spaces) + + BUILTIN_OPERATORS = [ EditFieldInfo(_builtin=True), CloneSelectedSamples(_builtin=True), diff --git a/package/db/setup.py b/package/db/setup.py index 8297b720ad..a0355f136d 100644 --- a/package/db/setup.py +++ b/package/db/setup.py @@ -65,6 +65,12 @@ "x86_64": "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu2204-6.0.5.tgz", }, }, + "fedora": { + "4": { + "aarch64": "https://fastdl.mongodb.org/linux/mongodb-linux-aarch64-rhel90-7.0.2.tgz", + "x86_64": "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel90-7.0.2.tgz", + }, + }, "pop": { "18": { "aarch64": "https://fastdl.mongodb.org/linux/mongodb-linux-aarch64-ubuntu1804-5.0.4.tgz",