From c4cecc68a7fad656c1dad19587c808ddad2bb068 Mon Sep 17 00:00:00 2001
From: Paul Elliott <paul.elliott@kitware.com>
Date: Fri, 1 Nov 2024 12:30:47 -0400
Subject: [PATCH] feat(ImageDetection): use component from trame-annotations

---
 pyproject.toml                                |   1 +
 src/nrtk_explorer/app/ui/image_list.py        |  13 +-
 src/nrtk_explorer/widgets/nrtk_explorer.py    |  19 --
 .../src/components/ImageDetection.vue         | 278 ------------------
 vue-components/src/components/index.js        |   2 -
 5 files changed, 8 insertions(+), 305 deletions(-)
 delete mode 100644 vue-components/src/components/ImageDetection.vue

diff --git a/pyproject.toml b/pyproject.toml
index 4c70deed..21f17e9f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,6 +42,7 @@ dependencies = [
     "transformers",
     "umap-learn",
     "nrtk[headless]>=0.12.0",
+    "trame-annotations",
 ]
 
 [project.optional-dependencies]
diff --git a/src/nrtk_explorer/app/ui/image_list.py b/src/nrtk_explorer/app/ui/image_list.py
index e11dac7d..5efdb2ab 100644
--- a/src/nrtk_explorer/app/ui/image_list.py
+++ b/src/nrtk_explorer/app/ui/image_list.py
@@ -1,10 +1,11 @@
 from pathlib import Path
 from trame.widgets import html, quasar, client
 from trame.decorators import TrameApp, change
+from trame_annotations.widgets.annotations import ImageDetection
 from nrtk_explorer.app.trame_utils import change_checker
-from nrtk_explorer.widgets.nrtk_explorer import ImageDetection
 from nrtk_explorer.app.images.image_ids import get_image_state_keys
 
+
 CSS_FILE = Path(__file__).with_name("image_list.css")
 
 COLUMNS = [
@@ -90,7 +91,7 @@ def __init__(
         categories=None,
         selected=None,
         hover=None,
-        containerSelector=None,
+        container_selector=None,
         **kwargs,
     ):
         super().__init__(
@@ -105,7 +106,7 @@ def __init__(
                 categories=categories,
                 selected=selected,
                 hover=hover,
-                containerSelector=containerSelector,
+                container_selector=container_selector,
             )
             quasar.QInnerLoading(
                 showing=(f"!{src[0]} || (show_annotations_on_images && !{annotations[0]}.value)",)
@@ -247,7 +248,7 @@ def __init__(self, on_scroll, on_hover, **kwargs):
                             categories=("annotation_categories",),
                             selected=("(props.row.original == hovered_id)",),
                             hover=(on_hover, "[$event]"),
-                            containerSelector="#image-list .q-table__middle",
+                            container_selector="#image-list .q-table__middle",
                         )
                 with html.Template(
                     v_slot_body_cell_original=True,
@@ -264,7 +265,7 @@ def __init__(self, on_scroll, on_hover, **kwargs):
                             categories=("annotation_categories",),
                             selected=("(props.row.original == hovered_id)",),
                             hover=(on_hover, "[$event]"),
-                            containerSelector="#image-list .q-table__middle",
+                            container_selector="#image-list .q-table__middle",
                         )
                 with html.Template(
                     v_slot_body_cell_transformed=True,
@@ -284,7 +285,7 @@ def __init__(self, on_scroll, on_hover, **kwargs):
                             categories=("annotation_categories",),
                             selected=("(props.row.transformed == hovered_id)",),
                             hover=(on_hover, "[$event]"),
-                            containerSelector="#image-list .q-table__middle",
+                            container_selector="#image-list .q-table__middle",
                         )
                 # Grid Mode template for each row/grid-item
                 with html.Template(
diff --git a/src/nrtk_explorer/widgets/nrtk_explorer.py b/src/nrtk_explorer/widgets/nrtk_explorer.py
index 1ce9b7a2..6df5a65f 100644
--- a/src/nrtk_explorer/widgets/nrtk_explorer.py
+++ b/src/nrtk_explorer/widgets/nrtk_explorer.py
@@ -9,25 +9,6 @@ def __init__(self, _elem_name, children=None, **kwargs):
             self.server.enable_module(module)
 
 
-class ImageDetection(HtmlElement):
-    def __init__(self, **kwargs):
-        super().__init__(
-            "image-detection",
-            **kwargs,
-        )
-        self._attr_names += [
-            "identifier",
-            "src",
-            "annotations",
-            "categories",
-            "selected",
-            "containerSelector",
-        ]
-        self._event_names += [
-            "hover",
-        ]
-
-
 class ScatterPlot(HtmlElement):
     def __init__(self, **kwargs):
         super().__init__(
diff --git a/vue-components/src/components/ImageDetection.vue b/vue-components/src/components/ImageDetection.vue
deleted file mode 100644
index d784f86b..00000000
--- a/vue-components/src/components/ImageDetection.vue
+++ /dev/null
@@ -1,278 +0,0 @@
-<script setup lang="ts">
-import { ref, watchEffect, computed, onMounted, type MaybeRef, unref } from 'vue'
-
-import { Quadtree, Rectangle } from '@timohausmann/quadtree-ts'
-
-import type { Annotation, Category, Vector3 } from '../types'
-
-const CATEGORY_COLORS: Vector3<number>[] = [
-  [255, 0, 0],
-  [0, 255, 0],
-  [0, 0, 255],
-  [255, 255, 0],
-  [255, 0, 255],
-  [0, 255, 255]
-]
-
-const TOOLTIP_OFFSET = [8, 8]
-const TOOLTIP_PADDING = 12 // fudge to keep tooltip from clipping/overflowing. In pixels
-
-let annotationsTree: Quadtree<Rectangle<number>> | undefined = undefined
-
-function doRectanglesOverlap(recA: Rectangle<any>, recB: Rectangle<any>) {
-  const noHOverlap = recB.x >= recA.x + recA.width || recA.x >= recB.x + recB.width
-
-  if (noHOverlap) {
-    return false
-  }
-
-  const noVOverlap = recB.y >= recA.y + recA.height || recA.y >= recB.y + recB.height
-
-  return !noVOverlap
-}
-
-const props = defineProps<{
-  identifier: string
-  src: MaybeRef<string>
-  annotations?: MaybeRef<Annotation[]>
-  categories: { [key: number]: Category }
-  selected: boolean
-  containerSelector?: string
-}>()
-
-const visibleCanvas = ref<HTMLCanvasElement>()
-const visibleCtx = computed(() => visibleCanvas.value?.getContext('2d', { alpha: true }))
-const pickingCanvas = ref<HTMLCanvasElement>()
-const pickingCtx = computed(() =>
-  pickingCanvas.value?.getContext('2d', { willReadFrequently: true })
-)
-const labelContainer = ref<HTMLDivElement>()
-
-const imageSize = ref({ width: 0, height: 0 })
-const img = ref<HTMLImageElement>()
-const onImageLoad = () => {
-  imageSize.value = { width: img.value?.naturalWidth ?? 0, height: img.value?.naturalHeight ?? 0 }
-}
-
-const annotations = computed(() => unref(props.annotations) ?? [])
-
-const annotationsWithColor = computed(() => {
-  return annotations.value.map((annotation) => {
-    const mutex = annotation.category_id ?? 0
-    const color = CATEGORY_COLORS[mutex % CATEGORY_COLORS.length]
-    return { ...annotation, color }
-  })
-})
-
-// draw visible annotations
-watchEffect(() => {
-  if (!visibleCanvas.value || !visibleCtx.value) {
-    return
-  }
-  const canvas = visibleCanvas.value
-  const ctx = visibleCtx.value
-
-  canvas.width = imageSize.value.width
-  canvas.height = imageSize.value.height
-  ctx.clearRect(0, 0, canvas.width, canvas.height)
-
-  const opacity = 0.5
-  annotationsWithColor.value.forEach(({ color, bbox }) => {
-    ctx.fillStyle = `rgba(${[...color, opacity].join(',')})`
-    ctx.fillRect(bbox[0], bbox[1], bbox[2], bbox[3])
-  })
-})
-
-// draw picking annotations
-watchEffect(() => {
-  if (!pickingCtx.value || !pickingCanvas.value) {
-    return
-  }
-  const canvas = pickingCanvas.value
-  const ctx = pickingCtx.value
-
-  canvas.width = imageSize.value.width
-  canvas.height = imageSize.value.height
-  ctx.clearRect(0, 0, canvas.width, canvas.height)
-
-  annotationsTree = new Quadtree({
-    width: canvas.width,
-    height: canvas.height,
-    maxLevels: 8,
-    maxObjects: 10
-  })
-
-  annotations.value.forEach((annotation, i) => {
-    const treeNode = new Rectangle<number>({
-      x: annotation.bbox[0],
-      y: annotation.bbox[1],
-      width: annotation.bbox[2],
-      height: annotation.bbox[3],
-      data: i
-    })
-    annotationsTree?.insert(treeNode)
-    ctx.fillStyle = `rgb(255, 0, 0)`
-    ctx.fillRect(annotation.bbox[0], annotation.bbox[1], annotation.bbox[2], annotation.bbox[3])
-  })
-})
-
-interface HoverEvent {
-  id: string
-}
-
-type Events = {
-  hover: [HoverEvent]
-}
-
-const emit = defineEmits<Events>()
-
-function hideLabel() {
-  if (labelContainer.value) labelContainer.value.style.visibility = 'hidden'
-}
-
-onMounted(hideLabel)
-
-function mouseEnter() {
-  emit('hover', { id: props.identifier })
-}
-function mouseLeave() {
-  emit('hover', { id: '' })
-  hideLabel()
-}
-
-function displayToPixel(x: number, y: number, canvas: HTMLCanvasElement): [number, number] {
-  const canvasBounds = canvas.getBoundingClientRect()
-
-  const pixelX = (canvas.width * (x - canvasBounds.left)) / canvasBounds.width
-  const pixelY = (canvas.height * (y - canvasBounds.top)) / canvasBounds.height
-
-  return [pixelX, pixelY]
-}
-
-const mounted = ref(false)
-onMounted(() => {
-  mounted.value = true
-})
-const container = computed(() => {
-  if (!mounted.value || !props.containerSelector) return null
-  return document.querySelector(props.containerSelector)
-})
-
-function mouseMove(e: MouseEvent) {
-  if (
-    !pickingCanvas.value ||
-    pickingCanvas.value.width === 0 ||
-    !labelContainer.value ||
-    !annotationsTree ||
-    !props.categories ||
-    !props.annotations
-  ) {
-    return
-  }
-  const ctx = pickingCtx.value
-  if (!ctx) {
-    return
-  }
-
-  const [pixelX, pixelY] = displayToPixel(e.clientX, e.clientY, pickingCanvas.value)
-  const pixelValue = ctx.getImageData(pixelX, pixelY, 1, 1).data[0]
-  const pickedSomething = pixelValue > 0
-
-  if (!pickedSomething) {
-    labelContainer.value.style.visibility = 'hidden'
-    return
-  }
-
-  labelContainer.value.style.visibility = 'visible'
-
-  const pixelRectangle = new Rectangle<number>({ x: pixelX, y: pixelY, width: 2, height: 2 })
-  const hits = annotationsTree
-    .retrieve(pixelRectangle)
-    .filter((rect: any) => doRectanglesOverlap(rect, pixelRectangle))
-    .filter((hit) => hit.data != undefined)
-    .map((hit) => {
-      const annotation = annotationsWithColor.value[hit.data!]
-      const name = props.categories[annotation.category_id]?.name ?? annotation.label
-      const color = annotation.color
-      const category = document.createElement('li')
-      category.style.textShadow = `rgba(${color.join(',')},0.6) 1px 1px 3px`
-      const annotationId = annotation.id ? ` : ${annotation.id}` : ''
-      category.textContent = `${name}${annotationId}`
-      return category
-    })
-
-  labelContainer.value.replaceChildren(...hits)
-
-  // Position the tooltip
-  const [x, y] = [e.offsetX, e.offsetY]
-  let posX = x + TOOLTIP_OFFSET[0]
-  let posY = y + TOOLTIP_OFFSET[1]
-
-  const tooltipRect = labelContainer.value.getBoundingClientRect()
-  const parentRect = pickingCanvas.value.getBoundingClientRect()
-  const containerRect = container.value?.getBoundingClientRect() ?? {
-    left: 0,
-    top: 0,
-    width: window.innerWidth,
-    height: window.innerHeight
-  }
-
-  const toolTipInContainer = {
-    left: parentRect.left + posX - containerRect.left,
-    top: parentRect.top + posY - containerRect.top,
-    width: tooltipRect.width + TOOLTIP_PADDING,
-    height: tooltipRect.height + TOOLTIP_PADDING
-  }
-
-  // if text goes off the edge, move up and/or left
-  if (toolTipInContainer.left + toolTipInContainer.width > containerRect.width) {
-    posX = x - tooltipRect.width - TOOLTIP_OFFSET[0]
-  }
-  if (toolTipInContainer.top + toolTipInContainer.height > containerRect.height) {
-    posY = y - tooltipRect.height - TOOLTIP_OFFSET[1]
-  }
-
-  labelContainer.value.style.left = `${posX}px`
-  labelContainer.value.style.top = `${posY}px`
-}
-
-const borderSize = computed(() => (props.selected ? '4' : '0'))
-
-const src = computed(() => unref(props.src))
-</script>
-
-<template>
-  <div style="position: relative">
-    <img
-      :src="src"
-      ref="img"
-      @load="onImageLoad"
-      :style="{ outlineWidth: borderSize + 'px' }"
-      style="width: 100%; outline-style: dotted; outline-color: red"
-    />
-    <canvas ref="visibleCanvas" style="width: 100%; position: absolute; left: 0; top: 0"></canvas>
-    <canvas
-      ref="pickingCanvas"
-      style="opacity: 0; width: 100%; position: absolute; left: 0; top: 0"
-      @mouseenter="mouseEnter"
-      @mousemove="mouseMove"
-      @mouseleave="mouseLeave"
-    ></canvas>
-    <ul
-      ref="labelContainer"
-      style="
-        position: absolute;
-        z-index: 10;
-        padding: 0.4rem;
-        white-space: pre;
-        font-size: small;
-        border-radius: 0.2rem;
-        border-color: rgba(127, 127, 127, 0.75);
-        border-style: solid;
-        border-width: thin;
-        background-color: #efefef;
-        list-style-type: none;
-      "
-    ></ul>
-  </div>
-</template>
diff --git a/vue-components/src/components/index.js b/vue-components/src/components/index.js
index dc3eca22..5f1bbba1 100644
--- a/vue-components/src/components/index.js
+++ b/vue-components/src/components/index.js
@@ -1,11 +1,9 @@
-import ImageDetection from './ImageDetection.vue'
 import ScatterPlot from './ScatterPlot.vue'
 import ParamsWidget from './ParamsWidget.vue'
 import FilterOptionsWidget from './FilterOptionsWidget.vue'
 import FilterOperatorWidget from './FilterOperatorWidget.vue'
 
 export default {
-  imageDetection: ImageDetection,
   scatterPlot: ScatterPlot,
   paramsWidget: ParamsWidget,
   filterOptionsWidget: FilterOptionsWidget,