From a347cb5a42bf0c8568476d3ebc3edf25ea1f3eed Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 4 Aug 2024 08:05:26 -0500 Subject: [PATCH 001/169] Fix large tablet recording view layout (#12753) --- web/src/views/events/RecordingView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/views/events/RecordingView.tsx b/web/src/views/events/RecordingView.tsx index ce416b873b..c1b30b98eb 100644 --- a/web/src/views/events/RecordingView.tsx +++ b/web/src/views/events/RecordingView.tsx @@ -507,7 +507,7 @@ export function RecordingView({ "pt-2 portrait:w-full", mainCameraAspect == "wide" ? "aspect-wide landscape:w-full" - : "aspect-video landscape:h-[94%]", + : "aspect-video landscape:h-[94%] landscape:xl:h-[65%]", ), )} style={{ From 4a867ddd56d9dffc12a83fcba487f9bbeaecfa05 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 4 Aug 2024 08:06:11 -0500 Subject: [PATCH 002/169] Use radix css var to limit desktop menu height (#12743) --- web/src/components/menu/GeneralSettings.tsx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/web/src/components/menu/GeneralSettings.tsx b/web/src/components/menu/GeneralSettings.tsx index baab171cc7..34637d57ef 100644 --- a/web/src/components/menu/GeneralSettings.tsx +++ b/web/src/components/menu/GeneralSettings.tsx @@ -139,8 +139,18 @@ export default function GeneralSettings({ className }: GeneralSettingsProps) {
From 93b81756c6f26fc5b4916cafbefe7f117e7a066b Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 5 Aug 2024 08:01:42 -0500 Subject: [PATCH 003/169] Only use dense property on phones for motion review timeline (#12768) --- web/src/views/events/EventView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/views/events/EventView.tsx b/web/src/views/events/EventView.tsx index a6c5e6bc8f..8b4569be0f 100644 --- a/web/src/views/events/EventView.tsx +++ b/web/src/views/events/EventView.tsx @@ -1057,7 +1057,7 @@ function MotionReview({ setScrubbing(scrubbing); }} - dense={isMobile} + dense={isMobileOnly} /> ) : ( From 5069072a8420ab9a0d872eeb3e7499eb5456e2e0 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 5 Aug 2024 07:20:21 -0600 Subject: [PATCH 004/169] Fix iOS export buttons (#12755) * Fix iOS export buttons * Use layering instead of z index --- web/src/components/card/ExportCard.tsx | 41 +++++++++++++------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/web/src/components/card/ExportCard.tsx b/web/src/components/card/ExportCard.tsx index a2c92b5fbe..d39cbbedab 100644 --- a/web/src/components/card/ExportCard.tsx +++ b/web/src/components/card/ExportCard.tsx @@ -124,13 +124,27 @@ export default function ExportCard({ onMouseLeave={isDesktop ? () => setHovered(false) : undefined} onClick={isDesktop ? undefined : () => setHovered(!hovered)} > - {hovered && ( + {exportedRecording.in_progress ? ( + + ) : ( <> -
+ {exportedRecording.thumb_path.length > 0 ? ( + setLoading(false)} + /> + ) : ( +
+ )} + + )} + {hovered && ( +
+
{!exportedRecording.in_progress && ( @@ -167,7 +181,7 @@ export default function ExportCard({ {!exportedRecording.in_progress && ( )} - - )} - {exportedRecording.in_progress ? ( - - ) : ( - <> - {exportedRecording.thumb_path.length > 0 ? ( - setLoading(false)} - /> - ) : ( -
- )} - +
)} {loading && ( )} -
+
{exportedRecording.name.replaceAll("_", " ")}
From f8f7b74792cc96b344a5abb53dd61b6936f3928f Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 6 Aug 2024 06:40:20 -0600 Subject: [PATCH 005/169] Update version --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0cfd9b282a..1c1d19a0c2 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ default_target: local COMMIT_HASH := $(shell git log -1 --pretty=format:"%h"|tail -1) -VERSION = 0.14.0 +VERSION = 0.14.1 IMAGE_REPO ?= ghcr.io/blakeblackshear/frigate GITHUB_REF_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) CURRENT_UID := $(shell id -u) From 43d2986208982d2219409742986d27fc4687d479 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 6 Aug 2024 09:08:14 -0600 Subject: [PATCH 006/169] Handle case where sub label was null (#12785) --- web/src/components/player/LivePlayer.tsx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/web/src/components/player/LivePlayer.tsx b/web/src/components/player/LivePlayer.tsx index 88730b8fb1..67057a2784 100644 --- a/web/src/components/player/LivePlayer.tsx +++ b/web/src/components/player/LivePlayer.tsx @@ -266,10 +266,7 @@ export default function LivePlayer({ ), ]), ] - .filter( - (label) => - label !== undefined && !label.includes("-verified"), - ) + .filter((label) => label?.includes("-verified") == false) .map((label) => capitalizeFirstLetter(label)) .sort() .join(", ") From 8212b66ee0f19e1e393b784884abdff6d644b889 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 6 Aug 2024 09:08:43 -0600 Subject: [PATCH 007/169] Use camera status to get state of camera config (#12787) * Use camera status to get state of camera config * Fix spelling --- frigate/comms/dispatcher.py | 15 ++++++++++++- web/src/api/ws.tsx | 44 ++++++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py index db6c44c110..26922f284d 100644 --- a/frigate/comms/dispatcher.py +++ b/frigate/comms/dispatcher.py @@ -129,7 +129,20 @@ def _receive(self, topic: str, payload: str) -> Optional[Any]: elif topic == UPDATE_CAMERA_ACTIVITY: self.camera_activity = payload elif topic == "onConnect": - self.publish("camera_activity", json.dumps(self.camera_activity)) + camera_status = self.camera_activity.copy() + + for camera in camera_status.keys(): + camera_status[camera]["config"] = { + "detect": self.config.cameras[camera].detect.enabled, + "snapshots": self.config.cameras[camera].snapshots.enabled, + "record": self.config.cameras[camera].record.enabled, + "audio": self.config.cameras[camera].audio.enabled, + "autotracking": self.config.cameras[ + camera + ].onvif.autotracking.enabled, + } + + self.publish("camera_activity", json.dumps(camera_status)) else: self.publish(topic, payload, retain=False) diff --git a/web/src/api/ws.tsx b/web/src/api/ws.tsx index afcbaa0c04..94f381ada4 100644 --- a/web/src/api/ws.tsx +++ b/web/src/api/ws.tsx @@ -1,7 +1,6 @@ import { baseUrl } from "./baseUrl"; import { useCallback, useEffect, useState } from "react"; import useWebSocket, { ReadyState } from "react-use-websocket"; -import { FrigateConfig } from "@/types/frigateConfig"; import { FrigateCameraState, FrigateEvent, @@ -9,7 +8,6 @@ import { ToggleableSetting, } from "@/types/ws"; import { FrigateStats } from "@/types/stats"; -import useSWR from "swr"; import { createContainer } from "react-tracked"; import useDeepMemo from "@/hooks/use-deep-memo"; @@ -26,40 +24,50 @@ type WsState = { type useValueReturn = [WsState, (update: Update) => void]; function useValue(): useValueReturn { - // basic config - const { data: config } = useSWR("config", { - revalidateOnFocus: false, - }); const wsUrl = `${baseUrl.replace(/^http/, "ws")}ws`; // main state + + const [hasCameraState, setHasCameraState] = useState(false); const [wsState, setWsState] = useState({}); useEffect(() => { - if (!config) { + if (hasCameraState) { + return; + } + + const activityValue: string = wsState["camera_activity"] as string; + + if (!activityValue) { + return; + } + + const cameraActivity: { [key: string]: object } = JSON.parse(activityValue); + + if (!cameraActivity) { return; } const cameraStates: WsState = {}; - Object.keys(config.cameras).forEach((camera) => { - const { name, record, detect, snapshots, audio, onvif } = - config.cameras[camera]; - cameraStates[`${name}/recordings/state`] = record.enabled ? "ON" : "OFF"; - cameraStates[`${name}/detect/state`] = detect.enabled ? "ON" : "OFF"; - cameraStates[`${name}/snapshots/state`] = snapshots.enabled - ? "ON" - : "OFF"; - cameraStates[`${name}/audio/state`] = audio.enabled ? "ON" : "OFF"; - cameraStates[`${name}/ptz_autotracker/state`] = onvif.autotracking.enabled + Object.entries(cameraActivity).forEach(([name, state]) => { + const { record, detect, snapshots, audio, autotracking } = + // @ts-expect-error we know this is correct + state["config"]; + cameraStates[`${name}/recordings/state`] = record ? "ON" : "OFF"; + cameraStates[`${name}/detect/state`] = detect ? "ON" : "OFF"; + cameraStates[`${name}/snapshots/state`] = snapshots ? "ON" : "OFF"; + cameraStates[`${name}/audio/state`] = audio ? "ON" : "OFF"; + cameraStates[`${name}/ptz_autotracker/state`] = autotracking ? "ON" : "OFF"; }); setWsState({ ...wsState, ...cameraStates }); + setHasCameraState(true); // we only want this to run initially when the config is loaded // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); + }, [wsState]); // ws handler const { sendJsonMessage, readyState } = useWebSocket(wsUrl, { From 54e1bd9eeb1dd5419c4de30bb7972cdc8aafe5bf Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Tue, 6 Aug 2024 10:41:11 -0500 Subject: [PATCH 008/169] Ensure review cameras are sorted by config ui order if specified (#12789) --- web/src/components/filter/ReviewFilterGroup.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/web/src/components/filter/ReviewFilterGroup.tsx b/web/src/components/filter/ReviewFilterGroup.tsx index 5ee0ac9bb5..d01ea384f5 100644 --- a/web/src/components/filter/ReviewFilterGroup.tsx +++ b/web/src/components/filter/ReviewFilterGroup.tsx @@ -136,7 +136,11 @@ export default function ReviewFilterGroup({ const filterValues = useMemo( () => ({ - cameras: Object.keys(config?.cameras || {}), + cameras: Object.keys(config?.cameras ?? {}).sort( + (a, b) => + (config?.cameras[a]?.ui?.order ?? 0) - + (config?.cameras[b]?.ui?.order ?? 0), + ), labels: Object.values(allLabels || {}), zones: Object.values(allZones || {}), }), From 9c2974438d6c78d4ab6b2bb0988225bf4da6e1bd Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 6 Aug 2024 14:15:00 -0600 Subject: [PATCH 009/169] Handle case where user stops scrubbing but remains hovering (#12794) * Handle case where user stops scrubbing but remains hovering * Add type --- .../player/PreviewThumbnailPlayer.tsx | 148 +++++++++++------- web/src/types/timeline.ts | 2 + 2 files changed, 94 insertions(+), 56 deletions(-) diff --git a/web/src/components/player/PreviewThumbnailPlayer.tsx b/web/src/components/player/PreviewThumbnailPlayer.tsx index 20eefe3614..d2192e85df 100644 --- a/web/src/components/player/PreviewThumbnailPlayer.tsx +++ b/web/src/components/player/PreviewThumbnailPlayer.tsx @@ -21,7 +21,7 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"; import ImageLoadingIndicator from "../indicators/ImageLoadingIndicator"; import useContextMenu from "@/hooks/use-contextmenu"; import ActivityIndicator from "../indicators/activity-indicator"; -import { TimeRange } from "@/types/timeline"; +import { TimelineScrubMode, TimeRange } from "@/types/timeline"; import { NoThumbSlider } from "../ui/slider"; import { PREVIEW_FPS, PREVIEW_PADDING } from "@/types/preview"; import { capitalizeFirstLetter } from "@/utils/stringUtil"; @@ -414,7 +414,7 @@ export function VideoPreview({ if (isSafari || (isFirefox && isMobile)) { playerRef.current.pause(); - setManualPlayback(true); + setPlaybackMode("compat"); } else { playerRef.current.currentTime = playerStartTime; playerRef.current.playbackRate = PREVIEW_FPS; @@ -453,9 +453,9 @@ export function VideoPreview({ setReviewed(); if (loop && playerRef.current) { - if (manualPlayback) { - setManualPlayback(false); - setTimeout(() => setManualPlayback(true), 100); + if (playbackMode != "auto") { + setPlaybackMode("auto"); + setTimeout(() => setPlaybackMode("compat"), 100); } playerRef.current.currentTime = playerStartTime; @@ -472,7 +472,7 @@ export function VideoPreview({ playerRef.current?.pause(); } - setManualPlayback(false); + setPlaybackMode("auto"); setProgress(100.0); } else { setProgress(playerPercent); @@ -486,9 +486,10 @@ export function VideoPreview({ // safari is incapable of playing at a speed > 2x // so manual seeking is required on iOS - const [manualPlayback, setManualPlayback] = useState(false); + const [playbackMode, setPlaybackMode] = useState("auto"); + useEffect(() => { - if (!manualPlayback || !playerRef.current) { + if (playbackMode != "compat" || !playerRef.current) { return; } @@ -503,10 +504,14 @@ export function VideoPreview({ // we know that these deps are correct // eslint-disable-next-line react-hooks/exhaustive-deps - }, [manualPlayback, playerRef]); + }, [playbackMode, playerRef]); // user interaction + useEffect(() => { + setIgnoreClick(playbackMode != "auto" && playbackMode != "compat"); + }, [playbackMode, setIgnoreClick]); + const onManualSeek = useCallback( (values: number[]) => { const value = values[0]; @@ -515,14 +520,8 @@ export function VideoPreview({ return; } - if (manualPlayback) { - setManualPlayback(false); - setIgnoreClick(true); - } - if (playerRef.current.paused == false) { playerRef.current.pause(); - setIgnoreClick(true); } if (setReviewed) { @@ -536,27 +535,21 @@ export function VideoPreview({ // we know that these deps are correct // eslint-disable-next-line react-hooks/exhaustive-deps - [ - manualPlayback, - playerDuration, - playerRef, - playerStartTime, - setIgnoreClick, - ], + [playerDuration, playerRef, playerStartTime, setIgnoreClick], ); const onStopManualSeek = useCallback(() => { setTimeout(() => { - setIgnoreClick(false); setHoverTimeout(undefined); if (isSafari || (isFirefox && isMobile)) { - setManualPlayback(true); + setPlaybackMode("compat"); } else { + setPlaybackMode("auto"); playerRef.current?.play(); } }, 500); - }, [playerRef, setIgnoreClick]); + }, [playerRef]); const onProgressHover = useCallback( (event: React.MouseEvent) => { @@ -572,10 +565,8 @@ export function VideoPreview({ if (hoverTimeout) { clearTimeout(hoverTimeout); } - - setHoverTimeout(setTimeout(() => onStopManualSeek(), 500)); }, - [sliderRef, hoverTimeout, onManualSeek, onStopManualSeek, setHoverTimeout], + [sliderRef, hoverTimeout, onManualSeek], ); return ( @@ -597,14 +588,37 @@ export function VideoPreview({ {showProgress && ( { + setPlaybackMode("drag"); + onManualSeek(event); + }} onValueCommit={onStopManualSeek} min={0} step={1} max={100} - onMouseMove={isMobile ? undefined : onProgressHover} + onMouseMove={ + isMobile + ? undefined + : (event) => { + if (playbackMode != "drag") { + setPlaybackMode("hover"); + onProgressHover(event); + } + } + } + onMouseLeave={ + isMobile + ? undefined + : () => { + if (!sliderRef.current) { + return; + } + + setHoverTimeout(setTimeout(() => onStopManualSeek(), 500)); + } + } /> )}
@@ -642,7 +656,8 @@ export function InProgressPreview({ }/frames`, { revalidateOnFocus: false }, ); - const [manualFrame, setManualFrame] = useState(false); + + const [playbackMode, setPlaybackMode] = useState("auto"); const [hoverTimeout, setHoverTimeout] = useState(); const [key, setKey] = useState(0); @@ -655,7 +670,7 @@ export function InProgressPreview({ onTimeUpdate(review.start_time - PREVIEW_PADDING + key); } - if (manualFrame) { + if (playbackMode != "auto") { return; } @@ -692,19 +707,18 @@ export function InProgressPreview({ // we know that these deps are correct // eslint-disable-next-line react-hooks/exhaustive-deps - }, [key, manualFrame, previewFrames]); + }, [key, playbackMode, previewFrames]); // user interaction + useEffect(() => { + setIgnoreClick(playbackMode != "auto"); + }, [playbackMode, setIgnoreClick]); + const onManualSeek = useCallback( (values: number[]) => { const value = values[0]; - if (!manualFrame) { - setManualFrame(true); - setIgnoreClick(true); - } - if (!review.has_been_reviewed) { setReviewed(review.id); } @@ -714,19 +728,18 @@ export function InProgressPreview({ // we know that these deps are correct // eslint-disable-next-line react-hooks/exhaustive-deps - [manualFrame, setIgnoreClick, setManualFrame, setKey], + [setIgnoreClick, setKey], ); const onStopManualSeek = useCallback( (values: number[]) => { const value = values[0]; setTimeout(() => { - setIgnoreClick(false); - setManualFrame(false); + setPlaybackMode("auto"); setKey(value - 1); }, 500); }, - [setManualFrame, setIgnoreClick], + [setPlaybackMode], ); const onProgressHover = useCallback( @@ -744,17 +757,8 @@ export function InProgressPreview({ if (hoverTimeout) { clearTimeout(hoverTimeout); } - - setHoverTimeout(setTimeout(() => onStopManualSeek(progress), 500)); }, - [ - sliderRef, - hoverTimeout, - previewFrames, - onManualSeek, - onStopManualSeek, - setHoverTimeout, - ], + [sliderRef, hoverTimeout, previewFrames, onManualSeek], ); if (!previewFrames || previewFrames.length == 0) { @@ -776,14 +780,46 @@ export function InProgressPreview({ {showProgress && ( { + setPlaybackMode("drag"); + onManualSeek(event); + }} onValueCommit={onStopManualSeek} min={0} step={1} max={previewFrames.length - 1} - onMouseMove={isMobile ? undefined : onProgressHover} + onMouseMove={ + isMobile + ? undefined + : (event) => { + if (playbackMode != "drag") { + setPlaybackMode("hover"); + onProgressHover(event); + } + } + } + onMouseLeave={ + isMobile + ? undefined + : (event) => { + if (!sliderRef.current || !previewFrames) { + return; + } + + const rect = sliderRef.current.getBoundingClientRect(); + const positionX = event.clientX - rect.left; + const width = sliderRef.current.clientWidth; + const progress = [ + Math.round((positionX / width) * previewFrames.length), + ]; + + setHoverTimeout( + setTimeout(() => onStopManualSeek(progress), 500), + ); + } + } /> )}
diff --git a/web/src/types/timeline.ts b/web/src/types/timeline.ts index b4e02304c1..b7945314d9 100644 --- a/web/src/types/timeline.ts +++ b/web/src/types/timeline.ts @@ -26,3 +26,5 @@ export type Timeline = { export type TimeRange = { before: number; after: number }; export type TimelineType = "timeline" | "events"; + +export type TimelineScrubMode = "auto" | "drag" | "hover" | "compat"; From e563692fa2810c6d9ebf11a247296a197eea927a Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 6 Aug 2024 16:11:20 -0600 Subject: [PATCH 010/169] Add camera name to audio debug line (#12799) * Add camera name to audio debug line * Formatting --- frigate/events/audio.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frigate/events/audio.py b/frigate/events/audio.py index fca16f364a..471d403b81 100644 --- a/frigate/events/audio.py +++ b/frigate/events/audio.py @@ -209,7 +209,9 @@ def detect_audio(self, audio) -> None: audio_detections = [] for label, score, _ in model_detections: - logger.debug(f"Heard {label} with a score of {score}") + logger.debug( + f"{self.config.name} heard {label} with a score of {score}" + ) if label not in self.config.audio.listen: continue From 57503cc318fddb3165bacb2de1f1dea3fbf87060 Mon Sep 17 00:00:00 2001 From: Marc Altmann <40744649+MarcA711@users.noreply.github.com> Date: Wed, 7 Aug 2024 15:28:12 +0200 Subject: [PATCH 011/169] fix default model for rknn detector (#12807) --- frigate/detectors/plugins/rknn.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frigate/detectors/plugins/rknn.py b/frigate/detectors/plugins/rknn.py index af22ca3588..7606313b5d 100644 --- a/frigate/detectors/plugins/rknn.py +++ b/frigate/detectors/plugins/rknn.py @@ -23,7 +23,6 @@ class RknnDetectorConfig(BaseDetectorConfig): type: Literal[DETECTOR_KEY] num_cores: int = Field(default=0, ge=0, le=3, title="Number of NPU cores to use.") - purge_model_cache: bool = Field(default=True) class Rknn(DetectionApi): @@ -36,7 +35,9 @@ def __init__(self, config: RknnDetectorConfig): core_mask = 2**config.num_cores - 1 soc = self.get_soc() - model_props = self.parse_model_input(config.model.path, soc) + model_path = config.model.path or "deci-fp16-yolonas_s" + + model_props = self.parse_model_input(model_path, soc) if model_props["preset"]: config.model.model_type = model_props["model_type"] From 9f43d10ba71084b72c7fd350c1be875d73191613 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Wed, 7 Aug 2024 13:31:39 -0500 Subject: [PATCH 012/169] Ensure review card icon color for event view is visible in light mode (#12812) --- web/src/components/card/ReviewCard.tsx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/web/src/components/card/ReviewCard.tsx b/web/src/components/card/ReviewCard.tsx index fe221e1121..64f52bd1b0 100644 --- a/web/src/components/card/ReviewCard.tsx +++ b/web/src/components/card/ReviewCard.tsx @@ -130,10 +130,16 @@ export default function ReviewCard({
{event.data.objects.map((object) => { - return getIconForLabel(object, "size-3 text-white"); + return getIconForLabel( + object, + "size-3 text-primary dark:text-white", + ); })} {event.data.audio.map((audio) => { - return getIconForLabel(audio, "size-3 text-white"); + return getIconForLabel( + audio, + "size-3 text-primary dark:text-white", + ); })}
{formattedDate}
From 33e04fe61fc75472f55563766ee020c7f376de1b Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 9 Aug 2024 07:46:18 -0500 Subject: [PATCH 013/169] Add right click to delete points in desktop mask/zone editor (#12744) --- web/src/components/settings/PolygonCanvas.tsx | 26 +++++++++++++++++++ .../settings/PolygonEditControls.tsx | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/web/src/components/settings/PolygonCanvas.tsx b/web/src/components/settings/PolygonCanvas.tsx index 6121293e5e..e6851b63c9 100644 --- a/web/src/components/settings/PolygonCanvas.tsx +++ b/web/src/components/settings/PolygonCanvas.tsx @@ -114,6 +114,29 @@ export function PolygonCanvas({ const mousePos = stage.getPointerPosition() ?? { x: 0, y: 0 }; const intersection = stage.getIntersection(mousePos); + // right click on desktops to delete a point + if ( + e.evt instanceof MouseEvent && + e.evt.button === 2 && + intersection?.getClassName() == "Circle" + ) { + const pointIndex = parseInt(intersection.name()?.split("-")[1]); + if (!isNaN(pointIndex)) { + const updatedPoints = activePolygon.points.filter( + (_, index) => index !== pointIndex, + ); + updatedPolygons[activePolygonIndex] = { + ...activePolygon, + points: updatedPoints, + pointsOrder: activePolygon.pointsOrder?.filter( + (_, index) => index !== pointIndex, + ), + }; + setPolygons(updatedPolygons); + } + return; + } + if ( activePolygon.points.length >= 3 && intersection?.getClassName() == "Circle" && @@ -236,6 +259,9 @@ export function PolygonCanvas({ onMouseDown={handleMouseDown} onTouchStart={handleMouseDown} onMouseOver={handleStageMouseOver} + onContextMenu={(e) => { + e.evt.preventDefault(); + }} > - Undo + Remove last point From 6d9590b4ec63c6649b05914ac0c08ae0853f2988 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 9 Aug 2024 07:46:39 -0500 Subject: [PATCH 014/169] Persist live view muted/unmuted for session only (#12727) * Persist live view muted/unmuted for session only * consistent naming --- web/src/hooks/use-session-persistence.ts | 39 ++++++++++++++++++++++++ web/src/views/live/LiveCameraView.tsx | 5 +-- 2 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 web/src/hooks/use-session-persistence.ts diff --git a/web/src/hooks/use-session-persistence.ts b/web/src/hooks/use-session-persistence.ts new file mode 100644 index 0000000000..3662c799cd --- /dev/null +++ b/web/src/hooks/use-session-persistence.ts @@ -0,0 +1,39 @@ +import { useCallback, useState } from "react"; + +type useSessionPersistenceReturn = [ + value: S | undefined, + setValue: (value: S | undefined) => void, +]; + +export function useSessionPersistence( + key: string, + defaultValue: S | undefined = undefined, +): useSessionPersistenceReturn { + const [storedValue, setStoredValue] = useState(() => { + try { + const value = window.sessionStorage.getItem(key); + + if (value) { + return JSON.parse(value); + } else { + window.sessionStorage.setItem(key, JSON.stringify(defaultValue)); + return defaultValue; + } + } catch (err) { + return defaultValue; + } + }); + + const setValue = useCallback( + (newValue: S | undefined) => { + try { + window.sessionStorage.setItem(key, JSON.stringify(newValue)); + // eslint-disable-next-line no-empty + } catch (err) {} + setStoredValue(newValue); + }, + [key], + ); + + return [storedValue, setValue]; +} diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx index 033b0d71cc..cd7190e86e 100644 --- a/web/src/views/live/LiveCameraView.tsx +++ b/web/src/views/live/LiveCameraView.tsx @@ -78,6 +78,7 @@ import { useNavigate } from "react-router-dom"; import { TransformWrapper, TransformComponent } from "react-zoom-pan-pinch"; import useSWR from "swr"; import { cn } from "@/lib/utils"; +import { useSessionPersistence } from "@/hooks/use-session-persistence"; type LiveCameraViewProps = { config?: FrigateConfig; @@ -194,7 +195,7 @@ export default function LiveCameraView({ // playback state - const [audio, setAudio] = useState(false); + const [audio, setAudio] = useSessionPersistence("liveAudio", false); const [mic, setMic] = useState(false); const [webRTC, setWebRTC] = useState(false); const [pip, setPip] = useState(false); @@ -404,7 +405,7 @@ export default function LiveCameraView({ className="p-2 md:p-0" variant={fullscreen ? "overlay" : "primary"} Icon={audio ? GiSpeaker : GiSpeakerOff} - isActive={audio} + isActive={audio ?? false} title={`${audio ? "Disable" : "Enable"} Camera Audio`} onClick={() => setAudio(!audio)} /> From c84511de164b4791194f719d964ef51bcd61bc1e Mon Sep 17 00:00:00 2001 From: "Soren L. Hansen" Date: Fri, 9 Aug 2024 06:26:26 -0700 Subject: [PATCH 015/169] Fix auth when serving Frigate at a subpath (#12815) Ensure axios.defaults.baseURL is set when accessing login form. Drop `/api` prefix in login form's `axios.post` call, since `/api` is part of the baseURL. Redirect to subpath on succesful authentication. Prepend subpath to default logout url. Fixes #12814 --- web/src/components/auth/AuthForm.tsx | 5 +++-- web/src/components/menu/AccountSettings.tsx | 3 ++- web/src/login.tsx | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/web/src/components/auth/AuthForm.tsx b/web/src/components/auth/AuthForm.tsx index 8050857748..f3a4358282 100644 --- a/web/src/components/auth/AuthForm.tsx +++ b/web/src/components/auth/AuthForm.tsx @@ -2,6 +2,7 @@ import * as React from "react"; +import { baseUrl } from "../../api/baseUrl"; import { cn } from "@/lib/utils"; import { Input } from "@/components/ui/input"; import { Button } from "@/components/ui/button"; @@ -43,7 +44,7 @@ export function UserAuthForm({ className, ...props }: UserAuthFormProps) { setIsLoading(true); try { await axios.post( - "/api/login", + "/login", { user: values.user, password: values.password, @@ -54,7 +55,7 @@ export function UserAuthForm({ className, ...props }: UserAuthFormProps) { }, }, ); - window.location.href = "/"; + window.location.href = baseUrl; } catch (error) { if (axios.isAxiosError(error)) { const err = error as AxiosError; diff --git a/web/src/components/menu/AccountSettings.tsx b/web/src/components/menu/AccountSettings.tsx index 20c852e65a..1b7470b9b5 100644 --- a/web/src/components/menu/AccountSettings.tsx +++ b/web/src/components/menu/AccountSettings.tsx @@ -3,6 +3,7 @@ import { TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip"; +import { baseUrl } from "../../api/baseUrl"; import { cn } from "@/lib/utils"; import { TooltipPortal } from "@radix-ui/react-tooltip"; import { isDesktop } from "react-device-detect"; @@ -26,7 +27,7 @@ type AccountSettingsProps = { export default function AccountSettings({ className }: AccountSettingsProps) { const { data: profile } = useSWR("profile"); const { data: config } = useSWR("config"); - const logoutUrl = config?.proxy?.logout_url || "/api/logout"; + const logoutUrl = config?.proxy?.logout_url || `${baseUrl}api/logout`; const Container = isDesktop ? DropdownMenu : Drawer; const Trigger = isDesktop ? DropdownMenuTrigger : DrawerTrigger; diff --git a/web/src/login.tsx b/web/src/login.tsx index cea5e3e423..4906b58c68 100644 --- a/web/src/login.tsx +++ b/web/src/login.tsx @@ -1,6 +1,7 @@ import React from "react"; import ReactDOM from "react-dom/client"; import LoginPage from "@/pages/LoginPage.tsx"; +import "@/api"; import "./index.css"; ReactDOM.createRoot(document.getElementById("root")!).render( From 70618e93b7737922a7ae5e7b98636f2226df9dbd Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 9 Aug 2024 07:29:35 -0600 Subject: [PATCH 016/169] Add button to mark review item as reviewed in filmstrip (#12878) * Add button to mark review item as reviewd in filmstrip * Add tooltip --- web/src/components/card/AnimatedEventCard.tsx | 26 ++++++++++++++++++- web/src/views/live/LiveDashboardView.tsx | 1 + 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/web/src/components/card/AnimatedEventCard.tsx b/web/src/components/card/AnimatedEventCard.tsx index 552aa9f26b..85d6eb6d4b 100644 --- a/web/src/components/card/AnimatedEventCard.tsx +++ b/web/src/components/card/AnimatedEventCard.tsx @@ -12,17 +12,21 @@ import { isCurrentHour } from "@/utils/dateUtil"; import { useCameraPreviews } from "@/hooks/use-camera-previews"; import { baseUrl } from "@/api/baseUrl"; import { useApiHost } from "@/api"; -import { isSafari } from "react-device-detect"; +import { isDesktop, isSafari } from "react-device-detect"; import { usePersistence } from "@/hooks/use-persistence"; import { Skeleton } from "../ui/skeleton"; +import { Button } from "../ui/button"; +import { FaCircleCheck } from "react-icons/fa6"; type AnimatedEventCardProps = { event: ReviewSegment; selectedGroup?: string; + updateEvents: () => void; }; export function AnimatedEventCard({ event, selectedGroup, + updateEvents, }: AnimatedEventCardProps) { const { data: config } = useSWR("config"); const apiHost = useApiHost(); @@ -59,6 +63,7 @@ export function AnimatedEventCard({ }, [visibilityListener]); const [isLoaded, setIsLoaded] = useState(false); + const [isHovered, setIsHovered] = useState(false); // interaction @@ -102,7 +107,26 @@ export function AnimatedEventCard({ style={{ aspectRatio: aspectRatio, }} + onMouseEnter={isDesktop ? () => setIsHovered(true) : undefined} + onMouseLeave={isDesktop ? () => setIsHovered(false) : undefined} > + {isHovered && ( + + + + + Mark as Reviewed + + )}
); })} From e9e86cc5afacb4fe66c4fdd6c8077e37f0080aa3 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 9 Aug 2024 15:59:55 -0600 Subject: [PATCH 017/169] Fix use experimental migrator (#12906) --- frigate/util/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frigate/util/config.py b/frigate/util/config.py index acb7a9cb93..e7744e56d4 100644 --- a/frigate/util/config.py +++ b/frigate/util/config.py @@ -90,7 +90,7 @@ def migrate_014(config: dict[str, dict[str, any]]) -> dict[str, dict[str, any]]: # Remove UI fields if new_config.get("ui"): if new_config["ui"].get("use_experimental"): - del new_config["ui"]["experimental"] + del new_config["ui"]["use_experimental"] if new_config["ui"].get("live_mode"): del new_config["ui"]["live_mode"] From 78d67484e14ba9c581bdad4e3c5313aac378215f Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 9 Aug 2024 16:12:07 -0600 Subject: [PATCH 018/169] Web deps (#12908) * Update web compnent deps * Update other web deps --- web/package-lock.json | 424 +++++++++++++++++++++--------------------- web/package.json | 32 ++-- 2 files changed, 232 insertions(+), 224 deletions(-) diff --git a/web/package-lock.json b/web/package-lock.json index b49d6ad836..6b8a0d2a41 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -8,7 +8,7 @@ "name": "web-new", "version": "0.0.0", "dependencies": { - "@cycjimmy/jsmpeg-player": "^6.0.5", + "@cycjimmy/jsmpeg-player": "^6.1.1", "@hookform/resolvers": "^3.9.0", "@radix-ui/react-alert-dialog": "^1.1.1", "@radix-ui/react-aspect-ratio": "^1.1.0", @@ -30,19 +30,19 @@ "@radix-ui/react-toggle": "^1.1.0", "@radix-ui/react-toggle-group": "^1.1.0", "@radix-ui/react-tooltip": "^1.1.2", - "apexcharts": "^3.50.0", - "axios": "^1.7.2", + "apexcharts": "^3.52.0", + "axios": "^1.7.3", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", "copy-to-clipboard": "^3.3.3", "date-fns": "^3.6.0", - "hls.js": "^1.5.13", + "hls.js": "^1.5.14", "idb-keyval": "^6.2.1", "immer": "^10.1.1", - "konva": "^9.3.13", + "konva": "^9.3.14", "lodash": "^4.17.21", "lucide-react": "^0.407.0", - "monaco-yaml": "^5.1.1", + "monaco-yaml": "^5.2.2", "next-themes": "^0.3.0", "nosleep.js": "^0.12.0", "react": "^18.3.1", @@ -54,7 +54,7 @@ "react-hook-form": "^7.52.1", "react-icons": "^5.2.1", "react-konva": "^18.2.10", - "react-router-dom": "^6.24.1", + "react-router-dom": "^6.26.0", "react-swipeable": "^7.0.1", "react-tracked": "^2.0.0", "react-transition-group": "^4.4.5", @@ -76,7 +76,7 @@ "devDependencies": { "@tailwindcss/forms": "^0.5.7", "@testing-library/jest-dom": "^6.4.6", - "@types/lodash": "^4.17.6", + "@types/lodash": "^4.17.7", "@types/node": "^20.14.10", "@types/react": "^18.3.2", "@types/react-dom": "^18.3.0", @@ -87,8 +87,8 @@ "@typescript-eslint/eslint-plugin": "^7.5.0", "@typescript-eslint/parser": "^7.5.0", "@vitejs/plugin-react-swc": "^3.6.0", - "@vitest/coverage-v8": "^2.0.2", - "autoprefixer": "^10.4.19", + "@vitest/coverage-v8": "^2.0.5", + "autoprefixer": "^10.4.20", "eslint": "^8.57.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-jest": "^28.2.0", @@ -98,15 +98,15 @@ "eslint-plugin-vitest-globals": "^1.5.0", "fake-indexeddb": "^6.0.0", "jest-websocket-mock": "^2.5.0", - "jsdom": "^24.0.0", - "msw": "^2.3.0", + "jsdom": "^24.1.1", + "msw": "^2.3.5", "postcss": "^8.4.39", "prettier": "^3.3.2", "prettier-plugin-tailwindcss": "^0.6.5", - "tailwindcss": "^3.4.3", - "typescript": "^5.5.3", - "vite": "^5.3.3", - "vitest": "^2.0.2" + "tailwindcss": "^3.4.9", + "typescript": "^5.5.4", + "vite": "^5.4.0", + "vitest": "^2.0.5" } }, "node_modules/@aashutoshrathi/word-wrap": { @@ -233,10 +233,22 @@ "statuses": "^2.0.1" } }, + "node_modules/@bundled-es-modules/tough-cookie": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/@bundled-es-modules/tough-cookie/-/tough-cookie-0.1.6.tgz", + "integrity": "sha512-dvMHbL464C0zI+Yqxbz6kZ5TOEp7GLW+pry/RWndAR8MJQAXZ2rPmIs8tziTZjeIyhSNZgZbCePtfSbdWqStJw==", + "dev": true, + "license": "ISC", + "dependencies": { + "@types/tough-cookie": "^4.0.5", + "tough-cookie": "^4.1.4" + } + }, "node_modules/@cycjimmy/jsmpeg-player": { - "version": "6.0.5", - "resolved": "https://registry.npmjs.org/@cycjimmy/jsmpeg-player/-/jsmpeg-player-6.0.5.tgz", - "integrity": "sha512-bVNHQ7VN9ecKT5AI/6RC7zpW/y4ca68a9txeR5Wiin+jKpUn/7buMe+5NPub89A8NNeNnKPQfrD2+c76ch36mA==" + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@cycjimmy/jsmpeg-player/-/jsmpeg-player-6.1.1.tgz", + "integrity": "sha512-YqT7U/3LxGu+6ikd+GGPe3rA2o6P4xrBHsWi/WRqv4n58h91fWDxS/3smneod+u6H2RnWlmXvZqx960dQ9T9gQ==", + "license": "MIT" }, "node_modules/@esbuild/aix-ppc64": { "version": "0.21.5", @@ -986,15 +998,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@mswjs/cookies": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@mswjs/cookies/-/cookies-1.1.0.tgz", - "integrity": "sha512-0ZcCVQxifZmhwNBoQIrystCb+2sWBY2Zw8lpfJBPCHGCA/HWqehITeCRVIv4VMy8MPlaHo2w2pTHFV2pFfqKPw==", - "dev": true, - "engines": { - "node": ">=18" - } - }, "node_modules/@mswjs/interceptors": { "version": "0.29.1", "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.29.1.tgz", @@ -2200,9 +2203,9 @@ "license": "MIT" }, "node_modules/@remix-run/router": { - "version": "1.17.1", - "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.17.1.tgz", - "integrity": "sha512-mCOMec4BKd6BRGBZeSnGiIgwsbLGp3yhVqAD8H+PxiRNEHgDpZb8J1TnrSDlg97t0ySKMQJTHCWBCmBpSmkF6Q==", + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.19.0.tgz", + "integrity": "sha512-zDICCLKEwbVYTS6TjYaWtHXxkdoUvD/QXvyVZjGCsWz5vyH7aFeONlPffPdW+Y/t6KT0MgXb2Mfjun9YpWN1dA==", "license": "MIT", "engines": { "node": ">=14.0.0" @@ -2692,15 +2695,10 @@ "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", "dev": true }, - "node_modules/@types/json-schema": { - "version": "7.0.15", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", - "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==" - }, "node_modules/@types/lodash": { - "version": "4.17.6", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.6.tgz", - "integrity": "sha512-OpXEVoCKSS3lQqjx9GGGOapBeuW5eUboYHRlHP9urXPX25IKZ6AnP5ZRxtVf63iieUbsHxLn8NQ5Nlftc6yzAA==", + "version": "4.17.7", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.7.tgz", + "integrity": "sha512-8wTvZawATi/lsmNu10/j2hk1KEP0IvjubqPE3cu1Xz7xfXXt5oCq3SNUz4fMIP4XGF9Ky+Ue2tBA3hcS7LSBlA==", "dev": true, "license": "MIT" }, @@ -2795,6 +2793,13 @@ "integrity": "sha512-QIvDlGAKyF3YJbT3QZnfC+RIvV5noyDbi+ZJ5rkaSRqxCGrYJefgXm3leZAjtoQOutZe1hCXbAg+p89/Vj4HlQ==", "dev": true }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/wrap-ansi": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/wrap-ansi/-/wrap-ansi-3.0.0.tgz", @@ -3041,9 +3046,9 @@ } }, "node_modules/@vitest/coverage-v8": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-2.0.2.tgz", - "integrity": "sha512-iA8eb4PMid3bMc++gfQSTvYE1QL//fC8pz+rKsTUDBFjdDiy/gH45hvpqyDu5K7FHhvgG0GNNCJzTMMSFKhoxg==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-2.0.5.tgz", + "integrity": "sha512-qeFcySCg5FLO2bHHSa0tAZAOnAUbp4L6/A5JDuj9+bt53JREl8hpLjLHEWF0e/gWc8INVpJaqA7+Ene2rclpZg==", "dev": true, "license": "MIT", "dependencies": { @@ -3057,7 +3062,6 @@ "magic-string": "^0.30.10", "magicast": "^0.3.4", "std-env": "^3.7.0", - "strip-literal": "^2.1.0", "test-exclude": "^7.0.1", "tinyrainbow": "^1.2.0" }, @@ -3065,18 +3069,18 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "vitest": "2.0.2" + "vitest": "2.0.5" } }, "node_modules/@vitest/expect": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.0.2.tgz", - "integrity": "sha512-nKAvxBYqcDugYZ4nJvnm5OR8eDJdgWjk4XM9owQKUjzW70q0icGV2HVnQOyYsp906xJaBDUXw0+9EHw2T8e0mQ==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.0.5.tgz", + "integrity": "sha512-yHZtwuP7JZivj65Gxoi8upUN2OzHTi3zVfjwdpu2WrvCZPLwsJ2Ey5ILIPccoW23dd/zQBlJ4/dhi7DWNyXCpA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "2.0.2", - "@vitest/utils": "2.0.2", + "@vitest/spy": "2.0.5", + "@vitest/utils": "2.0.5", "chai": "^5.1.1", "tinyrainbow": "^1.2.0" }, @@ -3085,9 +3089,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.0.2.tgz", - "integrity": "sha512-SBCyOXfGVvddRd9r2PwoVR0fonQjh9BMIcBMlSzbcNwFfGr6ZhOhvBzurjvi2F4ryut2HcqiFhNeDVGwru8tLg==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.0.5.tgz", + "integrity": "sha512-h8k+1oWHfwTkyTkb9egzwNMfJAEx4veaPSnMeKbVSjp4euqGSbQlm5+6VHwTr7u4FJslVVsUG5nopCaAYdOmSQ==", "dev": true, "license": "MIT", "dependencies": { @@ -3098,13 +3102,13 @@ } }, "node_modules/@vitest/runner": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.0.2.tgz", - "integrity": "sha512-OCh437Vi8Wdbif1e0OvQcbfM3sW4s2lpmOjAE7qfLrpzJX2M7J1IQlNvEcb/fu6kaIB9n9n35wS0G2Q3en5kHg==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.0.5.tgz", + "integrity": "sha512-TfRfZa6Bkk9ky4tW0z20WKXFEwwvWhRY+84CnSEtq4+3ZvDlJyY32oNTJtM7AW9ihW90tX/1Q78cb6FjoAs+ig==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "2.0.2", + "@vitest/utils": "2.0.5", "pathe": "^1.1.2" }, "funding": { @@ -3112,13 +3116,13 @@ } }, "node_modules/@vitest/snapshot": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.0.2.tgz", - "integrity": "sha512-Yc2ewhhZhx+0f9cSUdfzPRcsM6PhIb+S43wxE7OG0kTxqgqzo8tHkXFuFlndXeDMp09G3sY/X5OAo/RfYydf1g==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.0.5.tgz", + "integrity": "sha512-SgCPUeDFLaM0mIUHfaArq8fD2WbaXG/zVXjRupthYfYGzc8ztbFbu6dUNOblBG7XLMR1kEhS/DNnfCZ2IhdDew==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "2.0.2", + "@vitest/pretty-format": "2.0.5", "magic-string": "^0.30.10", "pathe": "^1.1.2" }, @@ -3127,9 +3131,9 @@ } }, "node_modules/@vitest/spy": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.0.2.tgz", - "integrity": "sha512-MgwJ4AZtCgqyp2d7WcQVE8aNG5vQ9zu9qMPYQHjsld/QVsrvg78beNrXdO4HYkP0lDahCO3P4F27aagIag+SGQ==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.0.5.tgz", + "integrity": "sha512-c/jdthAhvJdpfVuaexSrnawxZz6pywlTPe84LUB2m/4t3rl2fTo9NFGBG4oWgaD+FTgDDV8hJ/nibT7IfH3JfA==", "dev": true, "license": "MIT", "dependencies": { @@ -3140,13 +3144,13 @@ } }, "node_modules/@vitest/utils": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.0.2.tgz", - "integrity": "sha512-pxCY1v7kmOCWYWjzc0zfjGTA3Wmn8PKnlPvSrsA643P1NHl1fOyXj2Q9SaNlrlFE+ivCsxM80Ov3AR82RmHCWQ==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.0.5.tgz", + "integrity": "sha512-d8HKbqIcya+GR67mkZbrzhS5kKhtp8dQLcmRZLGTscGVg7yImT82cIrhtn2L8+VujWcy6KZweApgNmPsTAO/UQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "2.0.2", + "@vitest/pretty-format": "2.0.5", "estree-walker": "^3.0.3", "loupe": "^3.1.1", "tinyrainbow": "^1.2.0" @@ -3281,9 +3285,9 @@ } }, "node_modules/apexcharts": { - "version": "3.50.0", - "resolved": "https://registry.npmjs.org/apexcharts/-/apexcharts-3.50.0.tgz", - "integrity": "sha512-LJT1PNAm+NoIU3aogL2P+ViC0y/Cjik54FdzzGV54UNnGQLBoLe5ok3fxsJDTgyez45BGYT8gqNpYKqhdfy5sg==", + "version": "3.52.0", + "resolved": "https://registry.npmjs.org/apexcharts/-/apexcharts-3.52.0.tgz", + "integrity": "sha512-7dg0ADKs8AA89iYMZMe2sFDG0XK5PfqllKV9N+i3hKHm3vEtdhwz8AlXGm+/b0nJ6jKiaXsqci5LfVxNhtB+dA==", "license": "MIT", "dependencies": { "@yr/monotone-cubic-spline": "^1.0.3", @@ -3353,9 +3357,9 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "node_modules/autoprefixer": { - "version": "10.4.19", - "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.19.tgz", - "integrity": "sha512-BaENR2+zBZ8xXhM4pUaKUxlVdxZ0EZhjvbopwnXmxRUfqDmwSpC2lAi/QXvx7NRdPCo1WKEcEF6mV64si1z4Ew==", + "version": "10.4.20", + "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.20.tgz", + "integrity": "sha512-XY25y5xSv/wEoqzDyXXME4AFfkZI0P23z6Fs3YgymDnKJkCGOnkL0iTxCa85UTqaSgfcqyf3UA6+c7wUvx/16g==", "dev": true, "funding": [ { @@ -3371,12 +3375,13 @@ "url": "https://github.com/sponsors/ai" } ], + "license": "MIT", "dependencies": { - "browserslist": "^4.23.0", - "caniuse-lite": "^1.0.30001599", + "browserslist": "^4.23.3", + "caniuse-lite": "^1.0.30001646", "fraction.js": "^4.3.7", "normalize-range": "^0.1.2", - "picocolors": "^1.0.0", + "picocolors": "^1.0.1", "postcss-value-parser": "^4.2.0" }, "bin": { @@ -3390,9 +3395,10 @@ } }, "node_modules/axios": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", - "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==", + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.3.tgz", + "integrity": "sha512-Ar7ND9pU99eJ9GpoGQKhKf58GpUOgnzuaB7ueNQ5BMi0p+LZ5oaEnfF999fAArcTIBwXTCHAmGcHOZJaWPq9Nw==", + "license": "MIT", "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", @@ -3454,9 +3460,9 @@ } }, "node_modules/browserslist": { - "version": "4.23.0", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.0.tgz", - "integrity": "sha512-QW8HiM1shhT2GuzkvklfjcKDiWFXHOeFCIA/huJPwHsslwcydgk7X+z2zXpEijP98UCY7HbubZt5J2Zgvf0CaQ==", + "version": "4.23.3", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.3.tgz", + "integrity": "sha512-btwCFJVjI4YWDNfau8RhZ+B1Q/VLoUITrm3RlP6y1tYGWIOa+InuYiRGXUBXo8nA1qKmHMyLB/iVQg5TT4eFoA==", "dev": true, "funding": [ { @@ -3472,11 +3478,12 @@ "url": "https://github.com/sponsors/ai" } ], + "license": "MIT", "dependencies": { - "caniuse-lite": "^1.0.30001587", - "electron-to-chromium": "^1.4.668", - "node-releases": "^2.0.14", - "update-browserslist-db": "^1.0.13" + "caniuse-lite": "^1.0.30001646", + "electron-to-chromium": "^1.5.4", + "node-releases": "^2.0.18", + "update-browserslist-db": "^1.1.0" }, "bin": { "browserslist": "cli.js" @@ -3529,9 +3536,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001599", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001599.tgz", - "integrity": "sha512-LRAQHZ4yT1+f9LemSMeqdMpMxZcc4RMWdj4tiFe3G8tNkWK+E58g+/tzotb5cU6TbcVJLr4fySiAW7XmxQvZQA==", + "version": "1.0.30001651", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001651.tgz", + "integrity": "sha512-9Cf+Xv1jJNe1xPZLGuUXLNkE1BoDkqRqYyFJ9TDYSqhduqA4hu4oR9HluGoWYQC/aj8WHjsGVV+bwkh0+tegRg==", "dev": true, "funding": [ { @@ -3546,7 +3553,8 @@ "type": "github", "url": "https://github.com/sponsors/ai" } - ] + ], + "license": "CC-BY-4.0" }, "node_modules/chai": { "version": "5.1.1", @@ -4073,10 +4081,11 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.4.692", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.692.tgz", - "integrity": "sha512-d5rZRka9n2Y3MkWRN74IoAsxR0HK3yaAt7T50e3iT9VZmCCQDT3geXUO5ZRMhDToa1pkCeQXuNo+0g+NfDOVPA==", - "dev": true + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.5.tgz", + "integrity": "sha512-QR7/A7ZkMS8tZuoftC/jfqNkZLQO779SSW3YuZHP4eXpj3EffGLFcB/Xu9AAZQzLccTiCV+EmUo3ha4mQ9wnlA==", + "dev": true, + "license": "ISC" }, "node_modules/emoji-regex": { "version": "8.0.0", @@ -4136,10 +4145,11 @@ } }, "node_modules/escalade": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", + "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", "dev": true, + "license": "MIT", "engines": { "node": ">=6" } @@ -4859,9 +4869,9 @@ "dev": true }, "node_modules/hls.js": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.5.13.tgz", - "integrity": "sha512-xRgKo84nsC7clEvSfIdgn/Tc0NOT+d7vdiL/wvkLO+0k0juc26NRBPPG1SfB8pd5bHXIjMW/F5VM8VYYkOYYdw==", + "version": "1.5.14", + "resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.5.14.tgz", + "integrity": "sha512-5wLiQ2kWJMui6oUslaq8PnPOv1vjuee5gTxjJD0DSsccY12OXtDT0h137UuqjczNeHzeEYR0ROZQibKNMr7Mzg==", "license": "Apache-2.0" }, "node_modules/html-encoding-sniffer": { @@ -4898,9 +4908,9 @@ } }, "node_modules/https-proxy-agent": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", - "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", "dev": true, "license": "MIT", "dependencies": { @@ -5301,9 +5311,9 @@ } }, "node_modules/jsdom": { - "version": "24.1.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-24.1.0.tgz", - "integrity": "sha512-6gpM7pRXCwIOKxX47cgOyvyQDN/Eh0f1MeKySBV2xGdKtqJBLj8P25eY3EVCWo2mglDDzozR2r2MW4T+JiNUZA==", + "version": "24.1.1", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-24.1.1.tgz", + "integrity": "sha512-5O1wWV99Jhq4DV7rCLIoZ/UIhyQeDR7wHVyZAHAshbrvZsLs+Xzz7gtwnlJTJDjleiTKh54F4dXrX70vJQTyJQ==", "dev": true, "license": "MIT", "dependencies": { @@ -5313,11 +5323,11 @@ "form-data": "^4.0.0", "html-encoding-sniffer": "^4.0.0", "http-proxy-agent": "^7.0.2", - "https-proxy-agent": "^7.0.4", + "https-proxy-agent": "^7.0.5", "is-potential-custom-element-name": "^1.0.1", - "nwsapi": "^2.2.10", + "nwsapi": "^2.2.12", "parse5": "^7.1.2", - "rrweb-cssom": "^0.7.0", + "rrweb-cssom": "^0.7.1", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^4.1.4", @@ -5326,7 +5336,7 @@ "whatwg-encoding": "^3.1.1", "whatwg-mimetype": "^4.0.0", "whatwg-url": "^14.0.0", - "ws": "^8.17.0", + "ws": "^8.18.0", "xml-name-validator": "^5.0.0" }, "engines": { @@ -5342,9 +5352,9 @@ } }, "node_modules/jsdom/node_modules/rrweb-cssom": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.7.0.tgz", - "integrity": "sha512-KlSv0pm9kgQSRxXEMgtivPJ4h826YHsuob8pSHcfSZsSXGtvpEAie8S0AnXuObEJ7nhikOb4ahwxDm0H2yW17g==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.7.1.tgz", + "integrity": "sha512-TrEMa7JGdVm0UThDJSx7ddw5nVm3UJS9o9CCIZ72B1vSyEZoziDqBYP3XIoi/12lKrJR8rE3jeFHMok2F/Mnsg==", "dev": true, "license": "MIT" }, @@ -5384,9 +5394,9 @@ } }, "node_modules/konva": { - "version": "9.3.13", - "resolved": "https://registry.npmjs.org/konva/-/konva-9.3.13.tgz", - "integrity": "sha512-hs0ysHnqjK9noZ/rkfDNJINfbNhkXMgjgkJ8uc6vU0amu05mSDtRlukz5kKHOaSnWHA6miXcHJydvPABh18Y8A==", + "version": "9.3.14", + "resolved": "https://registry.npmjs.org/konva/-/konva-9.3.14.tgz", + "integrity": "sha512-Gmm5lyikGYJyogKQA7Fy6dKkfNh350V6DwfZkid0RVrGYP2cfCsxuMxgF5etKeCv7NjXYpJxKqi1dYkIkX/dcA==", "funding": [ { "type": "patreon", @@ -5642,9 +5652,10 @@ "peer": true }, "node_modules/monaco-languageserver-types": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/monaco-languageserver-types/-/monaco-languageserver-types-0.3.2.tgz", - "integrity": "sha512-KiGVYK/DiX1pnacnOjGNlM85bhV3ZTyFlM+ce7B8+KpWCbF1XJVovu51YyuGfm+K7+K54mIpT4DFX16xmi+tYA==", + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/monaco-languageserver-types/-/monaco-languageserver-types-0.4.0.tgz", + "integrity": "sha512-QQ3BZiU5LYkJElGncSNb5AKoJ/LCs6YBMCJMAz9EA7v+JaOdn3kx2cXpPTcZfKA5AEsR0vc97sAw+5mdNhVBmw==", + "license": "MIT", "dependencies": { "monaco-types": "^0.1.0", "vscode-languageserver-protocol": "^3.0.0", @@ -5669,6 +5680,7 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/monaco-types/-/monaco-types-0.1.0.tgz", "integrity": "sha512-aWK7SN9hAqNYi0WosPoMjenMeXJjwCxDibOqWffyQ/qXdzB/86xshGQobRferfmNz7BSNQ8GB0MD0oby9/5fTQ==", + "license": "MIT", "funding": { "url": "https://github.com/sponsors/remcohaszing" } @@ -5682,13 +5694,16 @@ } }, "node_modules/monaco-yaml": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/monaco-yaml/-/monaco-yaml-5.1.1.tgz", - "integrity": "sha512-BuZ0/ZCGjrPNRzYMZ/MoxH8F/SdM+mATENXnpOhDYABi1Eh+QvxSszEct+ACSCarZiwLvy7m6yEF/pvW8XJkyQ==", + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/monaco-yaml/-/monaco-yaml-5.2.2.tgz", + "integrity": "sha512-NWO/UhtJATlIsqwWPzK7YfbcIvPo3riFGsUkaGxNJoGiNPOvHD8vZ83ecqMQGkHPOpgHtSbe94uokE1AJvpbyQ==", + "license": "MIT", + "workspaces": [ + "examples/*" + ], "dependencies": { - "@types/json-schema": "^7.0.0", "jsonc-parser": "^3.0.0", - "monaco-languageserver-types": "^0.3.0", + "monaco-languageserver-types": "^0.4.0", "monaco-marker-data-provider": "^1.0.0", "monaco-types": "^0.1.0", "monaco-worker-manager": "^2.0.0", @@ -5727,17 +5742,17 @@ "dev": true }, "node_modules/msw": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/msw/-/msw-2.3.1.tgz", - "integrity": "sha512-ocgvBCLn/5l3jpl1lssIb3cniuACJLoOfZu01e3n5dbJrpA5PeeWn28jCLgQDNt6d7QT8tF2fYRzm9JoEHtiig==", + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/msw/-/msw-2.3.5.tgz", + "integrity": "sha512-+GUI4gX5YC5Bv33epBrD+BGdmDvBg2XGruiWnI3GbIbRmMMBeZ5gs3mJ51OWSGHgJKztZ8AtZeYMMNMVrje2/Q==", "dev": true, "hasInstallScript": true, "license": "MIT", "dependencies": { "@bundled-es-modules/cookie": "^2.0.0", "@bundled-es-modules/statuses": "^1.0.1", + "@bundled-es-modules/tough-cookie": "^0.1.6", "@inquirer/confirm": "^3.0.0", - "@mswjs/cookies": "^1.1.0", "@mswjs/interceptors": "^0.29.0", "@open-draft/until": "^2.1.0", "@types/cookie": "^0.6.0", @@ -5834,10 +5849,11 @@ } }, "node_modules/node-releases": { - "version": "2.0.14", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz", - "integrity": "sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==", - "dev": true + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz", + "integrity": "sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==", + "dev": true, + "license": "MIT" }, "node_modules/normalize-path": { "version": "3.0.0", @@ -5889,9 +5905,9 @@ } }, "node_modules/nwsapi": { - "version": "2.2.10", - "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.10.tgz", - "integrity": "sha512-QK0sRs7MKv0tKe1+5uZIQk/C8XGza4DAnztJG8iD+TpJIORARrCxczA738awHrZoHeTjSSoHqao2teO0dC/gFQ==", + "version": "2.2.12", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.12.tgz", + "integrity": "sha512-qXDmcVlZV4XRtKFzddidpfVP4oMSGhga+xdMc25mv8kaLUHtgzCDhUxkrN8exkGdTlLNaXj7CV3GtON7zuGZ+w==", "dev": true, "license": "MIT" }, @@ -6165,9 +6181,9 @@ } }, "node_modules/postcss": { - "version": "8.4.39", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.39.tgz", - "integrity": "sha512-0vzE+lAiG7hZl1/9I8yzKLx3aR9Xbof3fBHKunvMfOCYAtMhrsnccJY2iTURb9EZd5+pLuiNV9/c/GZJOHsgIw==", + "version": "8.4.41", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.41.tgz", + "integrity": "sha512-TesUflQ0WKZqAvg52PWL6kHgLKP6xB6heTOdoYM0Wt2UHyxNa4K25EZZMgKns3BH1RLVbZCREPpLY0rhnNoHVQ==", "funding": [ { "type": "opencollective", @@ -6741,12 +6757,12 @@ } }, "node_modules/react-router": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/react-router/-/react-router-6.24.1.tgz", - "integrity": "sha512-PTXFXGK2pyXpHzVo3rR9H7ip4lSPZZc0bHG5CARmj65fTT6qG7sTngmb6lcYu1gf3y/8KxORoy9yn59pGpCnpg==", + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-6.26.0.tgz", + "integrity": "sha512-wVQq0/iFYd3iZ9H2l3N3k4PL8EEHcb0XlU2Na8nEwmiXgIUElEH6gaJDtUQxJ+JFzmIXaQjfdpcGWaM6IoQGxg==", "license": "MIT", "dependencies": { - "@remix-run/router": "1.17.1" + "@remix-run/router": "1.19.0" }, "engines": { "node": ">=14.0.0" @@ -6756,13 +6772,13 @@ } }, "node_modules/react-router-dom": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.24.1.tgz", - "integrity": "sha512-U19KtXqooqw967Vw0Qcn5cOvrX5Ejo9ORmOtJMzYWtCT4/WOfFLIZGGsVLxcd9UkBO0mSTZtXqhZBsWlHr7+Sg==", + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.26.0.tgz", + "integrity": "sha512-RRGUIiDtLrkX3uYcFiCIxKFWMcWQGMojpYZfcstc63A1+sSnVgILGIm9gNUA6na3Fm1QuPGSBQH2EMbAZOnMsQ==", "license": "MIT", "dependencies": { - "@remix-run/router": "1.17.1", - "react-router": "6.24.1" + "@remix-run/router": "1.19.0", + "react-router": "6.26.0" }, "engines": { "node": ">=14.0.0" @@ -7246,7 +7262,8 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", - "dev": true + "dev": true, + "license": "ISC" }, "node_modules/signal-exit": { "version": "4.1.0", @@ -7300,7 +7317,8 @@ "version": "0.0.2", "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/statuses": { "version": "2.0.1", @@ -7426,26 +7444,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/strip-literal": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-2.1.0.tgz", - "integrity": "sha512-Op+UycaUt/8FbN/Z2TWPBLge3jWrP3xj10f3fnYxf052bKuS3EKs1ZQcVGjnEMdsNVAM+plXRdmjrZ/KgG3Skw==", - "dev": true, - "license": "MIT", - "dependencies": { - "js-tokens": "^9.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/antfu" - } - }, - "node_modules/strip-literal/node_modules/js-tokens": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.0.tgz", - "integrity": "sha512-WriZw1luRMlmV3LGJaR6QOJjWwgLUTf89OwT2lUOyjX2dJGBwgmIkbcz+7WFZjrZM635JOIR517++e/67CP9dQ==", - "dev": true, - "license": "MIT" - }, "node_modules/sucrase": { "version": "3.34.0", "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.34.0.tgz", @@ -7648,9 +7646,9 @@ } }, "node_modules/tailwindcss": { - "version": "3.4.4", - "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.4.tgz", - "integrity": "sha512-ZoyXOdJjISB7/BcLTR6SEsLgKtDStYyYZVLsUtWChO4Ps20CBad7lfJKVDiejocV4ME1hLmyY0WJE3hSDcmQ2A==", + "version": "3.4.9", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.9.tgz", + "integrity": "sha512-1SEOvRr6sSdV5IDf9iC+NU4dhwdqzF4zKKq3sAbasUWHEM6lsMhX+eNN5gkPx1BvLFEnZQEUFbXnGj8Qlp83Pg==", "license": "MIT", "dependencies": { "@alloc/quick-lru": "^5.2.0", @@ -7931,9 +7929,9 @@ } }, "node_modules/typescript": { - "version": "5.5.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", - "integrity": "sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==", + "version": "5.5.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", + "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", "dev": true, "license": "Apache-2.0", "bin": { @@ -7992,9 +7990,9 @@ } }, "node_modules/update-browserslist-db": { - "version": "1.0.13", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz", - "integrity": "sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz", + "integrity": "sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ==", "dev": true, "funding": [ { @@ -8010,9 +8008,10 @@ "url": "https://github.com/sponsors/ai" } ], + "license": "MIT", "dependencies": { - "escalade": "^3.1.1", - "picocolors": "^1.0.0" + "escalade": "^3.1.2", + "picocolors": "^1.0.1" }, "bin": { "update-browserslist-db": "cli.js" @@ -8120,14 +8119,14 @@ } }, "node_modules/vite": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.3.3.tgz", - "integrity": "sha512-NPQdeCU0Dv2z5fu+ULotpuq5yfCS1BzKUIPhNbP3YBfAMGJXbt2nS+sbTFu+qchaqWTD+H3JK++nRwr6XIcp6A==", + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.0.tgz", + "integrity": "sha512-5xokfMX0PIiwCMCMb9ZJcMyh5wbBun0zUzKib+L65vAZ8GY9ePZMXxFrHbr/Kyll2+LSCY7xtERPpxkBDKngwg==", "dev": true, "license": "MIT", "dependencies": { "esbuild": "^0.21.3", - "postcss": "^8.4.39", + "postcss": "^8.4.40", "rollup": "^4.13.0" }, "bin": { @@ -8147,6 +8146,7 @@ "less": "*", "lightningcss": "^1.21.0", "sass": "*", + "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" @@ -8164,6 +8164,9 @@ "sass": { "optional": true }, + "sass-embedded": { + "optional": true + }, "stylus": { "optional": true }, @@ -8176,9 +8179,9 @@ } }, "node_modules/vite-node": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.0.2.tgz", - "integrity": "sha512-w4vkSz1Wo+NIQg8pjlEn0jQbcM/0D+xVaYjhw3cvarTanLLBh54oNiRbsT8PNK5GfuST0IlVXjsNRoNlqvY/fw==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.0.5.tgz", + "integrity": "sha512-LdsW4pxj0Ot69FAoXZ1yTnA9bjGohr2yNBU7QKRxpz8ITSkhuDl6h3zS/tvgz4qrNjeRnvrWeXQ8ZF7Um4W00Q==", "dev": true, "license": "MIT", "dependencies": { @@ -8207,19 +8210,19 @@ } }, "node_modules/vitest": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.0.2.tgz", - "integrity": "sha512-WlpZ9neRIjNBIOQwBYfBSr0+of5ZCbxT2TVGKW4Lv0c8+srCFIiRdsP7U009t8mMn821HQ4XKgkx5dVWpyoyLw==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.0.5.tgz", + "integrity": "sha512-8GUxONfauuIdeSl5f9GTgVEpg5BTOlplET4WEDaeY2QBiN8wSm68vxN/tb5z405OwppfoCavnwXafiaYBC/xOA==", "dev": true, "license": "MIT", "dependencies": { "@ampproject/remapping": "^2.3.0", - "@vitest/expect": "2.0.2", - "@vitest/pretty-format": "^2.0.2", - "@vitest/runner": "2.0.2", - "@vitest/snapshot": "2.0.2", - "@vitest/spy": "2.0.2", - "@vitest/utils": "2.0.2", + "@vitest/expect": "2.0.5", + "@vitest/pretty-format": "^2.0.5", + "@vitest/runner": "2.0.5", + "@vitest/snapshot": "2.0.5", + "@vitest/spy": "2.0.5", + "@vitest/utils": "2.0.5", "chai": "^5.1.1", "debug": "^4.3.5", "execa": "^8.0.1", @@ -8230,8 +8233,8 @@ "tinypool": "^1.0.0", "tinyrainbow": "^1.2.0", "vite": "^5.0.0", - "vite-node": "2.0.2", - "why-is-node-running": "^2.2.2" + "vite-node": "2.0.5", + "why-is-node-running": "^2.3.0" }, "bin": { "vitest": "vitest.mjs" @@ -8245,8 +8248,8 @@ "peerDependencies": { "@edge-runtime/vm": "*", "@types/node": "^18.0.0 || >=20.0.0", - "@vitest/browser": "2.0.2", - "@vitest/ui": "2.0.2", + "@vitest/browser": "2.0.5", + "@vitest/ui": "2.0.5", "happy-dom": "*", "jsdom": "*" }, @@ -8275,6 +8278,7 @@ "version": "8.2.0", "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.0.tgz", "integrity": "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA==", + "license": "MIT", "engines": { "node": ">=14.0.0" } @@ -8283,6 +8287,7 @@ "version": "3.17.5", "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.17.5.tgz", "integrity": "sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg==", + "license": "MIT", "dependencies": { "vscode-jsonrpc": "8.2.0", "vscode-languageserver-types": "3.17.5" @@ -8296,12 +8301,14 @@ "node_modules/vscode-languageserver-types": { "version": "3.17.5", "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.5.tgz", - "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==" + "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==", + "license": "MIT" }, "node_modules/vscode-uri": { "version": "3.0.8", "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.8.tgz", - "integrity": "sha512-AyFQ0EVmsOZOlAnxoFOGOq1SQDWAB7C6aqMGS23svWAllfOaxbuFvcT8D1i8z3Gyn8fraVeZNNmN6e9bxxXkKw==" + "integrity": "sha512-AyFQ0EVmsOZOlAnxoFOGOq1SQDWAB7C6aqMGS23svWAllfOaxbuFvcT8D1i8z3Gyn8fraVeZNNmN6e9bxxXkKw==", + "license": "MIT" }, "node_modules/w3c-xmlserializer": { "version": "5.0.0", @@ -8386,10 +8393,11 @@ } }, "node_modules/why-is-node-running": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.2.2.tgz", - "integrity": "sha512-6tSwToZxTOcotxHeA+qGCq1mVzKR3CwcJGmVcY+QE8SHy6TnpFnh8PAvPNHYr7EcuVeG0QSMxtYCuO1ta/G/oA==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", "dev": true, + "license": "MIT", "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" @@ -8440,9 +8448,9 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/ws": { - "version": "8.17.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz", - "integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "dev": true, "license": "MIT", "engines": { diff --git a/web/package.json b/web/package.json index 328e995b79..54cb39ebd2 100644 --- a/web/package.json +++ b/web/package.json @@ -14,7 +14,7 @@ "coverage": "vitest run --coverage" }, "dependencies": { - "@cycjimmy/jsmpeg-player": "^6.0.5", + "@cycjimmy/jsmpeg-player": "^6.1.1", "@hookform/resolvers": "^3.9.0", "@radix-ui/react-alert-dialog": "^1.1.1", "@radix-ui/react-aspect-ratio": "^1.1.0", @@ -36,19 +36,19 @@ "@radix-ui/react-toggle": "^1.1.0", "@radix-ui/react-toggle-group": "^1.1.0", "@radix-ui/react-tooltip": "^1.1.2", - "apexcharts": "^3.50.0", - "axios": "^1.7.2", + "apexcharts": "^3.52.0", + "axios": "^1.7.3", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", "copy-to-clipboard": "^3.3.3", "date-fns": "^3.6.0", - "hls.js": "^1.5.13", + "hls.js": "^1.5.14", "idb-keyval": "^6.2.1", "immer": "^10.1.1", - "konva": "^9.3.13", + "konva": "^9.3.14", "lodash": "^4.17.21", "lucide-react": "^0.407.0", - "monaco-yaml": "^5.1.1", + "monaco-yaml": "^5.2.2", "next-themes": "^0.3.0", "nosleep.js": "^0.12.0", "react": "^18.3.1", @@ -60,7 +60,7 @@ "react-hook-form": "^7.52.1", "react-icons": "^5.2.1", "react-konva": "^18.2.10", - "react-router-dom": "^6.24.1", + "react-router-dom": "^6.26.0", "react-swipeable": "^7.0.1", "react-tracked": "^2.0.0", "react-transition-group": "^4.4.5", @@ -82,7 +82,7 @@ "devDependencies": { "@tailwindcss/forms": "^0.5.7", "@testing-library/jest-dom": "^6.4.6", - "@types/lodash": "^4.17.6", + "@types/lodash": "^4.17.7", "@types/node": "^20.14.10", "@types/react": "^18.3.2", "@types/react-dom": "^18.3.0", @@ -93,8 +93,8 @@ "@typescript-eslint/eslint-plugin": "^7.5.0", "@typescript-eslint/parser": "^7.5.0", "@vitejs/plugin-react-swc": "^3.6.0", - "@vitest/coverage-v8": "^2.0.2", - "autoprefixer": "^10.4.19", + "@vitest/coverage-v8": "^2.0.5", + "autoprefixer": "^10.4.20", "eslint": "^8.57.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-jest": "^28.2.0", @@ -104,14 +104,14 @@ "eslint-plugin-vitest-globals": "^1.5.0", "fake-indexeddb": "^6.0.0", "jest-websocket-mock": "^2.5.0", - "jsdom": "^24.0.0", - "msw": "^2.3.0", + "jsdom": "^24.1.1", + "msw": "^2.3.5", "postcss": "^8.4.39", "prettier": "^3.3.2", "prettier-plugin-tailwindcss": "^0.6.5", - "tailwindcss": "^3.4.3", - "typescript": "^5.5.3", - "vite": "^5.3.3", - "vitest": "^2.0.2" + "tailwindcss": "^3.4.9", + "typescript": "^5.5.4", + "vite": "^5.4.0", + "vitest": "^2.0.5" } } From 99e03576bfd6b473db9fb368d136453c10b2c528 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 9 Aug 2024 16:22:24 -0600 Subject: [PATCH 019/169] Remove user args from http jpeg (#12909) --- frigate/ffmpeg_presets.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frigate/ffmpeg_presets.py b/frigate/ffmpeg_presets.py index d07ae369f1..402046bfed 100644 --- a/frigate/ffmpeg_presets.py +++ b/frigate/ffmpeg_presets.py @@ -214,8 +214,7 @@ def parse_preset_hardware_acceleration_encode( PRESETS_INPUT = { - "preset-http-jpeg-generic": _user_agent_args - + [ + "preset-http-jpeg-generic": [ "-r", "{}", "-stream_loop", From 9b96211faf1f70159135e8b69fc02b9976fa9554 Mon Sep 17 00:00:00 2001 From: Stavros Kois <47820033+stavros-k@users.noreply.github.com> Date: Sat, 10 Aug 2024 19:25:13 +0300 Subject: [PATCH 020/169] add shortcut and query for fullscreen in live view (#12924) * add shortcut and query for live view * Update web/src/views/live/LiveDashboardView.tsx * Update web/src/views/live/LiveDashboardView.tsx Co-authored-by: Nicolas Mowen * Apply suggestions from code review Co-authored-by: Nicolas Mowen * Update LiveDashboardView.tsx --------- Co-authored-by: Nicolas Mowen --- web/src/views/live/LiveDashboardView.tsx | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/web/src/views/live/LiveDashboardView.tsx b/web/src/views/live/LiveDashboardView.tsx index 00c0462243..51f46d2f2a 100644 --- a/web/src/views/live/LiveDashboardView.tsx +++ b/web/src/views/live/LiveDashboardView.tsx @@ -31,6 +31,9 @@ import { cn } from "@/lib/utils"; import { LivePlayerError, LivePlayerMode } from "@/types/live"; import { FaCompress, FaExpand } from "react-icons/fa"; import { useResizeObserver } from "@/hooks/resize-observer"; +import useKeyboardListener, { + KeyModifiers, +} from "@/hooks/use-keyboard-listener"; type LiveDashboardViewProps = { cameras: CameraConfig[]; @@ -247,6 +250,23 @@ export default function LiveDashboardView({ [setPreferredLiveModes], ); + const onKeyboardShortcut = useCallback( + (key: string, modifiers: KeyModifiers) => { + if (!modifiers.down) { + return; + } + + switch (key) { + case "f": + toggleFullscreen(); + break; + } + }, + [toggleFullscreen], + ); + + useKeyboardListener(["f"], onKeyboardShortcut); + return (
Date: Sun, 11 Aug 2024 07:25:09 -0500 Subject: [PATCH 021/169] Add confirmation dialog before deleting review items (#12950) --- web/src/components/card/ReviewCard.tsx | 189 +++++++++++++----- .../components/filter/ReviewActionGroup.tsx | 130 ++++++++---- web/src/components/player/VideoControls.tsx | 2 +- web/src/hooks/use-keyboard-listener.tsx | 29 ++- web/src/views/live/LiveDashboardView.tsx | 29 +-- 5 files changed, 263 insertions(+), 116 deletions(-) diff --git a/web/src/components/card/ReviewCard.tsx b/web/src/components/card/ReviewCard.tsx index 64f52bd1b0..33032d2b81 100644 --- a/web/src/components/card/ReviewCard.tsx +++ b/web/src/components/card/ReviewCard.tsx @@ -6,7 +6,7 @@ import { getIconForLabel } from "@/utils/iconUtil"; import { isDesktop, isIOS, isSafari } from "react-device-detect"; import useSWR from "swr"; import TimeAgo from "../dynamic/TimeAgo"; -import { useCallback, useMemo, useState } from "react"; +import { useCallback, useMemo, useRef, useState } from "react"; import useImageLoaded from "@/hooks/use-image-loaded"; import ImageLoadingIndicator from "../indicators/ImageLoadingIndicator"; import { FaCompactDisc } from "react-icons/fa"; @@ -18,9 +18,20 @@ import { ContextMenuItem, ContextMenuTrigger, } from "../ui/context-menu"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "../ui/alert-dialog"; import { Drawer, DrawerContent } from "../ui/drawer"; import axios from "axios"; import { toast } from "sonner"; +import useKeyboardListener from "@/hooks/use-keyboard-listener"; type ReviewCardProps = { event: ReviewSegment; @@ -46,6 +57,8 @@ export default function ReviewCard({ ); const [optionsOpen, setOptionsOpen] = useState(false); + const [deleteDialogOpen, setDeleteDialogOpen] = useState(false); + const bypassDialogRef = useRef(false); const onMarkAsReviewed = useCallback(async () => { await axios.post(`reviews/viewed`, { ids: [event.id] }); @@ -92,6 +105,18 @@ export default function ReviewCard({ setOptionsOpen(false); }, [event]); + useKeyboardListener(["Shift"], (_, modifiers) => { + bypassDialogRef.current = modifiers.shift; + }); + + const handleDelete = useCallback(() => { + if (bypassDialogRef.current) { + onDelete(); + } else { + setDeleteDialogOpen(true); + } + }, [bypassDialogRef, onDelete]); + const content = (
- {content} - - -
- -
Export
-
- - {!event.has_been_reviewed && ( + <> + setDeleteDialogOpen(!deleteDialogOpen)} + > + + + Confirm Delete + + + Are you sure you want to delete all recorded video associated with + this review item? +
+
+ Hold the Shift key to bypass this dialog in the future. + + + setOptionsOpen(false)}> + Cancel + + + Delete + + + + + + {content} +
- -
Mark as reviewed
+ +
Export
 - )} - -
- -
Delete
-
- - - + {!event.has_been_reviewed && ( + +
+ +
Mark as reviewed
+
+ + )} + +
+ +
+ {bypassDialogRef.current ? "Delete Now" : "Delete"} +
+
+ + + + ); } return ( - - {content} - -
- -
Export
-
- {!event.has_been_reviewed && ( + <> + setDeleteDialogOpen(!deleteDialogOpen)} + > + + + Confirm Delete + + + Are you sure you want to delete all recorded video associated with + this review item? +
+
+ Hold the Shift key to bypass this dialog in the future. + + + setOptionsOpen(false)}> + Cancel + + + Delete + + + + + + {content} +
- -
Mark as reviewed
+ +
Export
- )} -
- -
Delete
-
- - + {!event.has_been_reviewed && ( +
+ +
Mark as reviewed
+
+ )} +
+ +
+ {bypassDialogRef.current ? "Delete Now" : "Delete"} +
+
+ + + ); } diff --git a/web/src/components/filter/ReviewActionGroup.tsx b/web/src/components/filter/ReviewActionGroup.tsx index 49e9f561a0..c637b1e352 100644 --- a/web/src/components/filter/ReviewActionGroup.tsx +++ b/web/src/components/filter/ReviewActionGroup.tsx @@ -1,10 +1,21 @@ import { FaCircleCheck } from "react-icons/fa6"; -import { useCallback } from "react"; +import { useCallback, useState } from "react"; import axios from "axios"; import { Button } from "../ui/button"; import { isDesktop } from "react-device-detect"; import { FaCompactDisc } from "react-icons/fa"; import { HiTrash } from "react-icons/hi"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "../ui/alert-dialog"; +import useKeyboardListener from "@/hooks/use-keyboard-listener"; type ReviewActionGroupProps = { selectedReviews: string[]; @@ -34,49 +45,94 @@ export default function ReviewActionGroup({ pullLatestData(); }, [selectedReviews, setSelectedReviews, pullLatestData]); + const [deleteDialogOpen, setDeleteDialogOpen] = useState(false); + const [bypassDialog, setBypassDialog] = useState(false); + + useKeyboardListener(["Shift"], (_, modifiers) => { + setBypassDialog(modifiers.shift); + }); + + const handleDelete = useCallback(() => { + if (bypassDialog) { + onDelete(); + } else { + setDeleteDialogOpen(true); + } + }, [bypassDialog, onDelete]); + return ( -
-
-
{`${selectedReviews.length} selected`}
-
{"|"}
-
- Unselect + <> + setDeleteDialogOpen(!deleteDialogOpen)} + > + + + Confirm Delete + + + Are you sure you want to delete all recorded video associated with + the selected review items? +
+
+ Hold the Shift key to bypass this dialog in the future. + + + Cancel + + Delete + + + + + +
+
+
{`${selectedReviews.length} selected`}
+
{"|"}
+
+ Unselect +
-
-
- {selectedReviews.length == 1 && ( +
+ {selectedReviews.length == 1 && ( + + )} - )} - - + +
-
+ ); } diff --git a/web/src/components/player/VideoControls.tsx b/web/src/components/player/VideoControls.tsx index 70d9a4be8c..50b2cc0450 100644 --- a/web/src/components/player/VideoControls.tsx +++ b/web/src/components/player/VideoControls.tsx @@ -141,7 +141,7 @@ export default function VideoControls({ }, [volume, muted]); const onKeyboardShortcut = useCallback( - (key: string, modifiers: KeyModifiers) => { + (key: string | null, modifiers: KeyModifiers) => { if (!modifiers.down) { return; } diff --git a/web/src/hooks/use-keyboard-listener.tsx b/web/src/hooks/use-keyboard-listener.tsx index f127cf0d88..ad9462a053 100644 --- a/web/src/hooks/use-keyboard-listener.tsx +++ b/web/src/hooks/use-keyboard-listener.tsx @@ -4,11 +4,12 @@ export type KeyModifiers = { down: boolean; repeat: boolean; ctrl: boolean; + shift: boolean; }; export default function useKeyboardListener( keys: string[], - listener: (key: string, modifiers: KeyModifiers) => void, + listener: (key: string | null, modifiers: KeyModifiers) => void, ) { const keyDownListener = useCallback( (e: KeyboardEvent) => { @@ -16,13 +17,18 @@ export default function useKeyboardListener( return; } + const modifiers = { + down: true, + repeat: e.repeat, + ctrl: e.ctrlKey || e.metaKey, + shift: e.shiftKey, + }; + if (keys.includes(e.key)) { e.preventDefault(); - listener(e.key, { - down: true, - repeat: e.repeat, - ctrl: e.ctrlKey || e.metaKey, - }); + listener(e.key, modifiers); + } else if (e.key === "Shift" || e.key === "Control" || e.key === "Meta") { + listener(null, modifiers); } }, [keys, listener], @@ -34,9 +40,18 @@ export default function useKeyboardListener( return; } + const modifiers = { + down: false, + repeat: false, + ctrl: false, + shift: false, + }; + if (keys.includes(e.key)) { e.preventDefault(); - listener(e.key, { down: false, repeat: false, ctrl: false }); + listener(e.key, modifiers); + } else if (e.key === "Shift" || e.key === "Control" || e.key === "Meta") { + listener(null, modifiers); } }, [keys, listener], diff --git a/web/src/views/live/LiveDashboardView.tsx b/web/src/views/live/LiveDashboardView.tsx index 51f46d2f2a..a91afb356b 100644 --- a/web/src/views/live/LiveDashboardView.tsx +++ b/web/src/views/live/LiveDashboardView.tsx @@ -31,9 +31,7 @@ import { cn } from "@/lib/utils"; import { LivePlayerError, LivePlayerMode } from "@/types/live"; import { FaCompress, FaExpand } from "react-icons/fa"; import { useResizeObserver } from "@/hooks/resize-observer"; -import useKeyboardListener, { - KeyModifiers, -} from "@/hooks/use-keyboard-listener"; +import useKeyboardListener from "@/hooks/use-keyboard-listener"; type LiveDashboardViewProps = { cameras: CameraConfig[]; @@ -250,22 +248,17 @@ export default function LiveDashboardView({ [setPreferredLiveModes], ); - const onKeyboardShortcut = useCallback( - (key: string, modifiers: KeyModifiers) => { - if (!modifiers.down) { - return; - } - - switch (key) { - case "f": - toggleFullscreen(); - break; - } - }, - [toggleFullscreen], - ); + useKeyboardListener(["f"], (key, modifiers) => { + if (!modifiers.down) { + return; + } - useKeyboardListener(["f"], onKeyboardShortcut); + switch (key) { + case "f": + toggleFullscreen(); + break; + } + }); return (
Date: Sun, 11 Aug 2024 06:32:39 -0600 Subject: [PATCH 022/169] Catch case where user tries to end definite manual event (#12951) * Catch case where user tries to end definite manual event * Formatting --- frigate/review/maintainer.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/frigate/review/maintainer.py b/frigate/review/maintainer.py index 8fb1df362c..dfc7259b23 100644 --- a/frigate/review/maintainer.py +++ b/frigate/review/maintainer.py @@ -503,8 +503,15 @@ def run(self) -> None: # temporarily make it so this event can not end current_segment.last_update = sys.maxsize elif manual_info["state"] == ManualEventState.end: - self.indefinite_events[camera].pop(manual_info["event_id"]) - current_segment.last_update = manual_info["end_time"] + event_id = manual_info["event_id"] + + if event_id in self.indefinite_events[camera]: + self.indefinite_events[camera].pop(event_id) + current_segment.last_update = manual_info["end_time"] + else: + logger.error( + f"Event with ID {event_id} has a set duration and can not be ended manually." + ) else: if topic == DetectionTypeEnum.video: self.check_if_new_segment( From 67ba3dbd8b1b414fb38994d42407f08b3fd19087 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 11 Aug 2024 08:15:04 -0500 Subject: [PATCH 023/169] Add pan/pinch/zoom capability on plus snapshots (#12953) --- web/src/pages/SubmitPlus.tsx | 75 ++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/web/src/pages/SubmitPlus.tsx b/web/src/pages/SubmitPlus.tsx index 57c0fdf37a..1fcc6ef2a6 100644 --- a/web/src/pages/SubmitPlus.tsx +++ b/web/src/pages/SubmitPlus.tsx @@ -47,6 +47,7 @@ import { LuFolderX } from "react-icons/lu"; import { PiSlidersHorizontalFill } from "react-icons/pi"; import useSWR from "swr"; import useSWRInfinite from "swr/infinite"; +import { TransformWrapper, TransformComponent } from "react-zoom-pan-pinch"; const API_LIMIT = 100; @@ -254,36 +255,52 @@ export default function SubmitPlus() { open={upload != undefined} onOpenChange={(open) => (!open ? setUpload(undefined) : null)} > - - - Submit To Frigate+ - - Objects in locations you want to avoid are not false - positives. Submitting them as false positives will confuse - the model. - - - {`${upload?.label}`} - - - - - + {upload?.id && ( + {`${upload?.label}`} + )} + + + + + + + From 13d121f4436da591ffd1f799c50c7ae421c8a510 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Sun, 11 Aug 2024 07:32:17 -0600 Subject: [PATCH 024/169] Catch case where recording starts right at end of request (#12956) --- frigate/api/media.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frigate/api/media.py b/frigate/api/media.py index 98bb2f9520..911e13f7e1 100644 --- a/frigate/api/media.py +++ b/frigate/api/media.py @@ -546,6 +546,11 @@ def vod_ts(camera_name, start_ts, end_ts): if recording.end_time > end_ts: duration -= int((recording.end_time - end_ts) * 1000) + if duration == 0: + # this means the segment starts right at the end of the requested time range + # and it does not need to be included + continue + if 0 < duration < max_duration_ms: clip["keyFrameDurations"] = [duration] clips.append(clip) From 132a712341effa491936a6b0b10a1313b7fa7bd2 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 12 Aug 2024 07:21:21 -0600 Subject: [PATCH 025/169] Hide record switch when disabled (#12997) --- web/src/views/live/LiveCameraView.tsx | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx index cd7190e86e..db162198c1 100644 --- a/web/src/views/live/LiveCameraView.tsx +++ b/web/src/views/live/LiveCameraView.tsx @@ -412,6 +412,7 @@ export default function LiveCameraView({ )} sendDetect(detectState == "ON" ? "OFF" : "ON")} /> - sendRecord(recordState == "ON" ? "OFF" : "ON")} - /> + {recordingEnabled && ( + + sendRecord(recordState == "ON" ? "OFF" : "ON") + } + /> + )} Date: Mon, 12 Aug 2024 14:30:16 -0600 Subject: [PATCH 026/169] Recordings Fixes (#13005) * If recordings don't exist mark as no recordings * Fix reloading recordings failing * Fix mark items not clearing selected * Cleanup * Default to last full hour when error occurs * Remove check * Cleanup * Handle empty recordings list case * Ensure that the start time is within the time range * Catch other reset cases --- .../player/dynamic/DynamicVideoPlayer.tsx | 6 +++- web/src/pages/Events.tsx | 4 +-- web/src/views/events/EventView.tsx | 4 +++ web/src/views/events/RecordingView.tsx | 29 ++++++++++++++++--- 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/web/src/components/player/dynamic/DynamicVideoPlayer.tsx b/web/src/components/player/dynamic/DynamicVideoPlayer.tsx index a40d521eac..caa709430f 100644 --- a/web/src/components/player/dynamic/DynamicVideoPlayer.tsx +++ b/web/src/components/player/dynamic/DynamicVideoPlayer.tsx @@ -167,7 +167,11 @@ export default function DynamicVideoPlayer({ ); useEffect(() => { - if (!controller || !recordings) { + if (!controller || !recordings?.length) { + if (recordings?.length == 0) { + setNoRecording(true); + } + return; } diff --git a/web/src/pages/Events.tsx b/web/src/pages/Events.tsx index ce646f5bda..0326fe3dc6 100644 --- a/web/src/pages/Events.tsx +++ b/web/src/pages/Events.tsx @@ -101,7 +101,7 @@ export default function Events() { // review paging - const [beforeTs, setBeforeTs] = useState(Date.now() / 1000); + const [beforeTs, setBeforeTs] = useState(Math.ceil(Date.now() / 1000)); const last24Hours = useMemo(() => { return { before: beforeTs, after: getHoursAgo(24) }; }, [beforeTs]); @@ -455,5 +455,5 @@ export default function Events() { function getHoursAgo(hours: number): number { const now = new Date(); now.setHours(now.getHours() - hours); - return now.getTime() / 1000; + return Math.ceil(now.getTime() / 1000); } diff --git a/web/src/views/events/EventView.tsx b/web/src/views/events/EventView.tsx index 8b4569be0f..adfb082066 100644 --- a/web/src/views/events/EventView.tsx +++ b/web/src/views/events/EventView.tsx @@ -395,6 +395,7 @@ export default function EventView({ markAllItemsAsReviewed={markAllItemsAsReviewed} onSelectReview={onSelectReview} onSelectAllReviews={onSelectAllReviews} + setSelectedReviews={setSelectedReviews} pullLatestData={pullLatestData} /> )} @@ -437,6 +438,7 @@ type DetectionReviewProps = { markAllItemsAsReviewed: (currentItems: ReviewSegment[]) => void; onSelectReview: (review: ReviewSegment, ctrl: boolean) => void; onSelectAllReviews: () => void; + setSelectedReviews: (reviewIds: string[]) => void; pullLatestData: () => void; }; function DetectionReview({ @@ -455,6 +457,7 @@ function DetectionReview({ markAllItemsAsReviewed, onSelectReview, onSelectAllReviews, + setSelectedReviews, pullLatestData, }: DetectionReviewProps) { const reviewTimelineRef = useRef(null); @@ -692,6 +695,7 @@ function DetectionReview({ className="text-white" variant="select" onClick={() => { + setSelectedReviews([]); markAllItemsAsReviewed(currentItems ?? []); }} > diff --git a/web/src/views/events/RecordingView.tsx b/web/src/views/events/RecordingView.tsx index c1b30b98eb..1bc82d9e9d 100644 --- a/web/src/views/events/RecordingView.tsx +++ b/web/src/views/events/RecordingView.tsx @@ -84,7 +84,11 @@ export function RecordingView({ const previewRowRef = useRef(null); const previewRefs = useRef<{ [camera: string]: PreviewController }>({}); - const [playbackStart, setPlaybackStart] = useState(startTime); + const [playbackStart, setPlaybackStart] = useState( + startTime >= timeRange.after && startTime <= timeRange.before + ? startTime + : timeRange.before - 60, + ); const mainCameraReviewItems = useMemo( () => reviewItems?.filter((cam) => cam.camera == mainCamera) ?? [], @@ -107,8 +111,10 @@ export function RecordingView({ return chunk.after <= startTime && chunk.before >= startTime; }), ); - const currentTimeRange = useMemo( - () => chunkedTimeRange[selectedRangeIdx], + const currentTimeRange = useMemo( + () => + chunkedTimeRange[selectedRangeIdx] ?? + chunkedTimeRange[chunkedTimeRange.length - 1], [selectedRangeIdx, chunkedTimeRange], ); const reviewFilterList = useMemo(() => { @@ -198,6 +204,10 @@ export function RecordingView({ const manuallySetCurrentTime = useCallback( (time: number) => { + if (!currentTimeRange) { + return; + } + setCurrentTime(time); if (currentTimeRange.after <= time && currentTimeRange.before >= time) { @@ -420,7 +430,18 @@ export function RecordingView({ filterList={reviewFilterList} showReviewed setShowReviewed={() => {}} - onUpdateFilter={updateFilter} + onUpdateFilter={(newFilter) => { + // if we are resetting the date to last 24 hours + // then we need to reset the playbackStart time + if ( + filter?.before != undefined && + newFilter?.before == undefined + ) { + setPlaybackStart(Date.now() / 1000 - 360); + } + + updateFilter(newFilter); + }} setMotionOnly={() => {}} /> )} From 05bc3839cca34dab30b431e8c1b786a9b0387415 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 12 Aug 2024 15:12:49 -0600 Subject: [PATCH 027/169] Reset recordings when changing the date (#13009) --- web/src/pages/Events.tsx | 1 + web/src/views/events/RecordingView.tsx | 13 +------------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/web/src/pages/Events.tsx b/web/src/pages/Events.tsx index 0326fe3dc6..9ee81b1dc9 100644 --- a/web/src/pages/Events.tsx +++ b/web/src/pages/Events.tsx @@ -416,6 +416,7 @@ export default function Events() { if (selectedReviewData) { return ( {}} - onUpdateFilter={(newFilter) => { - // if we are resetting the date to last 24 hours - // then we need to reset the playbackStart time - if ( - filter?.before != undefined && - newFilter?.before == undefined - ) { - setPlaybackStart(Date.now() / 1000 - 360); - } - - updateFilter(newFilter); - }} + onUpdateFilter={updateFilter} setMotionOnly={() => {}} /> )} From b0d42ea116cace72718d76caffe5b279b0959d63 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 13 Aug 2024 08:23:46 -0600 Subject: [PATCH 028/169] Fix last hour preview (#13027) --- web/src/pages/Events.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/pages/Events.tsx b/web/src/pages/Events.tsx index 9ee81b1dc9..07675d87e0 100644 --- a/web/src/pages/Events.tsx +++ b/web/src/pages/Events.tsx @@ -111,7 +111,7 @@ export default function Events() { } return { - before: Math.floor(reviewSearchParams["before"]), + before: Math.ceil(reviewSearchParams["before"]), after: Math.floor(reviewSearchParams["after"]), }; }, [last24Hours, reviewSearchParams]); From 1b876bf8d345c8e5fb1bdd3ae0f51571b0294de8 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 13 Aug 2024 09:12:06 -0600 Subject: [PATCH 029/169] UI fixes (#13030) * Fix difficulty overwriting export name * Fix NaN for score selector --- web/src/components/card/ExportCard.tsx | 14 +++++++---- web/src/pages/SubmitPlus.tsx | 32 ++++++++++++++++---------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/web/src/components/card/ExportCard.tsx b/web/src/components/card/ExportCard.tsx index d39cbbedab..1ad98be0d2 100644 --- a/web/src/components/card/ExportCard.tsx +++ b/web/src/components/card/ExportCard.tsx @@ -44,7 +44,7 @@ export default function ExportCard({ const [editName, setEditName] = useState<{ original: string; - update: string; + update?: string; }>(); const submitRename = useCallback(() => { @@ -52,7 +52,7 @@ export default function ExportCard({ return; } - onRename(exportedRecording.id, editName.update); + onRename(exportedRecording.id, editName.update ?? ""); setEditName(undefined); }, [editName, exportedRecording, onRename, setEditName]); @@ -64,7 +64,7 @@ export default function ExportCard({ modifiers.down && !modifiers.repeat && editName && - editName.update.length > 0 + (editName.update?.length ?? 0) > 0 ) { submitRename(); } @@ -92,7 +92,11 @@ export default function ExportCard({ className="mt-3" type="search" placeholder={editName?.original} - value={editName?.update || editName?.original} + value={ + editName?.update == undefined + ? editName?.original + : editName?.update + } onChange={(e) => setEditName({ original: editName.original ?? "", @@ -159,7 +163,7 @@ export default function ExportCard({ onClick={() => setEditName({ original: exportedRecording.name, - update: "", + update: undefined, }) } > diff --git a/web/src/pages/SubmitPlus.tsx b/web/src/pages/SubmitPlus.tsx index 1fcc6ef2a6..9d2b9ae5bd 100644 --- a/web/src/pages/SubmitPlus.tsx +++ b/web/src/pages/SubmitPlus.tsx @@ -494,12 +494,16 @@ function PlusFilterGroup({ className="w-12" inputMode="numeric" value={Math.round((currentScoreRange?.at(0) ?? 0.5) * 100)} - onChange={(e) => - setCurrentScoreRange([ - parseInt(e.target.value) / 100.0, - currentScoreRange?.at(1) ?? 1.0, - ]) - } + onChange={(e) => { + const value = e.target.value; + + if (value) { + setCurrentScoreRange([ + parseInt(value) / 100.0, + currentScoreRange?.at(1) ?? 1.0, + ]); + } + }} /> - setCurrentScoreRange([ - currentScoreRange?.at(0) ?? 0.5, - parseInt(e.target.value) / 100.0, - ]) - } + onChange={(e) => { + const value = e.target.value; + + if (value) { + setCurrentScoreRange([ + currentScoreRange?.at(0) ?? 0.5, + parseInt(value) / 100.0, + ]); + } + }} />
From f6b61c26ae0a17e4e6e8e4d74f0491281f62ad88 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 13 Aug 2024 13:26:01 -0600 Subject: [PATCH 030/169] Rename bug report (#13039) --- .github/DISCUSSION_TEMPLATE/{bug-report.yml => report-a-bug.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/DISCUSSION_TEMPLATE/{bug-report.yml => report-a-bug.yml} (100%) diff --git a/.github/DISCUSSION_TEMPLATE/bug-report.yml b/.github/DISCUSSION_TEMPLATE/report-a-bug.yml similarity index 100% rename from .github/DISCUSSION_TEMPLATE/bug-report.yml rename to .github/DISCUSSION_TEMPLATE/report-a-bug.yml From 2e724291db200b08053624e06b38ef329f500152 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Wed, 14 Aug 2024 19:41:41 -0600 Subject: [PATCH 031/169] Catch case where github sends bad json data (#13077) --- frigate/stats/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frigate/stats/util.py b/frigate/stats/util.py index 09710b3581..ac28e3d89b 100644 --- a/frigate/stats/util.py +++ b/frigate/stats/util.py @@ -4,6 +4,7 @@ import os import shutil import time +from json import JSONDecodeError from typing import Any, Optional import psutil @@ -35,7 +36,7 @@ def get_latest_version(config: FrigateConfig) -> str: "https://api.github.com/repos/blakeblackshear/frigate/releases/latest", timeout=10, ) - except RequestException: + except (RequestException, JSONDecodeError): return "unknown" response = request.json() From 4dce8ff60a5be52794ed5a00bbb01886b3886ac1 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Thu, 15 Aug 2024 09:51:44 -0500 Subject: [PATCH 032/169] Add shortcut key "r" to mark selected items as reviewed (#13087) * Add shortcut key "r" to mark selected items as reviewed * unselect after keypress --- web/src/views/events/EventView.tsx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/web/src/views/events/EventView.tsx b/web/src/views/events/EventView.tsx index adfb082066..f3b2395f83 100644 --- a/web/src/views/events/EventView.tsx +++ b/web/src/views/events/EventView.tsx @@ -606,7 +606,7 @@ function DetectionReview({ // keyboard - useKeyboardListener(["a"], (key, modifiers) => { + useKeyboardListener(["a", "r"], (key, modifiers) => { if (modifiers.repeat || !modifiers.down) { return; } @@ -614,6 +614,16 @@ function DetectionReview({ if (key == "a" && modifiers.ctrl) { onSelectAllReviews(); } + + if (key == "r" && selectedReviews.length > 0) { + currentItems?.forEach((item) => { + if (selectedReviews.includes(item.id)) { + item.has_been_reviewed = true; + markItemAsReviewed(item); + } + }); + setSelectedReviews([]); + } }); return ( From 4133e454c41d3c45cbaba9cdca523908f29a6cdb Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Thu, 15 Aug 2024 16:13:11 -0500 Subject: [PATCH 033/169] Remove dashboard keyboard listener (#13102) --- web/src/views/live/LiveDashboardView.tsx | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/web/src/views/live/LiveDashboardView.tsx b/web/src/views/live/LiveDashboardView.tsx index a91afb356b..00c0462243 100644 --- a/web/src/views/live/LiveDashboardView.tsx +++ b/web/src/views/live/LiveDashboardView.tsx @@ -31,7 +31,6 @@ import { cn } from "@/lib/utils"; import { LivePlayerError, LivePlayerMode } from "@/types/live"; import { FaCompress, FaExpand } from "react-icons/fa"; import { useResizeObserver } from "@/hooks/resize-observer"; -import useKeyboardListener from "@/hooks/use-keyboard-listener"; type LiveDashboardViewProps = { cameras: CameraConfig[]; @@ -248,18 +247,6 @@ export default function LiveDashboardView({ [setPreferredLiveModes], ); - useKeyboardListener(["f"], (key, modifiers) => { - if (!modifiers.down) { - return; - } - - switch (key) { - case "f": - toggleFullscreen(); - break; - } - }); - return (
Date: Sat, 17 Aug 2024 13:16:48 -0500 Subject: [PATCH 034/169] Live player fixes (#13143) * Jump to live when exceeding buffer time threshold in MSE player * clean up * Try adjusting playback rate instead of jumping to live * clean up * fallback to webrtc if enabled before jsmpeg * baseline * clean up * remove comments * adaptive playback rate and intelligent switching improvements * increase logging and reset live mode after camera is no longer active on dashboard only * jump to live on safari/iOS * clean up * clean up * refactor camera live mode hook * remove key listener * resolve conflicts --- web/src/components/player/LivePlayer.tsx | 14 +-- web/src/components/player/MsePlayer.tsx | 138 +++++++++++++++++++-- web/src/hooks/use-camera-live-mode.ts | 90 ++++++++------ web/src/types/frigateConfig.ts | 7 +- web/src/views/live/DraggableGridLayout.tsx | 37 ++---- web/src/views/live/LiveCameraView.tsx | 29 +++-- web/src/views/live/LiveDashboardView.tsx | 35 +----- 7 files changed, 229 insertions(+), 121 deletions(-) diff --git a/web/src/components/player/LivePlayer.tsx b/web/src/components/player/LivePlayer.tsx index 67057a2784..9a0b6f3db6 100644 --- a/web/src/components/player/LivePlayer.tsx +++ b/web/src/components/player/LivePlayer.tsx @@ -13,7 +13,6 @@ import { LivePlayerMode, VideoResolutionType, } from "@/types/live"; -import useCameraLiveMode from "@/hooks/use-camera-live-mode"; import { getIconForLabel } from "@/utils/iconUtil"; import Chip from "../indicators/Chip"; import { capitalizeFirstLetter } from "@/utils/stringUtil"; @@ -25,7 +24,7 @@ type LivePlayerProps = { containerRef?: React.MutableRefObject; className?: string; cameraConfig: CameraConfig; - preferredLiveMode?: LivePlayerMode; + preferredLiveMode: LivePlayerMode; showStillWithoutActivity?: boolean; windowVisible?: boolean; playAudio?: boolean; @@ -36,6 +35,7 @@ type LivePlayerProps = { onClick?: () => void; setFullResolution?: React.Dispatch>; onError?: (error: LivePlayerError) => void; + onResetLiveMode?: () => void; }; export default function LivePlayer({ @@ -54,6 +54,7 @@ export default function LivePlayer({ onClick, setFullResolution, onError, + onResetLiveMode, }: LivePlayerProps) { const internalContainerRef = useRef(null); // camera activity @@ -70,8 +71,6 @@ export default function LivePlayer({ // camera live state - const liveMode = useCameraLiveMode(cameraConfig, preferredLiveMode); - const [liveReady, setLiveReady] = useState(false); const liveReadyRef = useRef(liveReady); @@ -91,6 +90,7 @@ export default function LivePlayer({ const timer = setTimeout(() => { if (liveReadyRef.current && !cameraActiveRef.current) { setLiveReady(false); + onResetLiveMode?.(); } }, 500); @@ -152,7 +152,7 @@ export default function LivePlayer({ let player; if (!autoLive) { player = null; - } else if (liveMode == "webrtc") { + } else if (preferredLiveMode == "webrtc") { player = ( ); - } else if (liveMode == "mse") { + } else if (preferredLiveMode == "mse") { if ("MediaSource" in window || "ManagedMediaSource" in window) { player = ( ); } - } else if (liveMode == "jsmpeg") { + } else if (preferredLiveMode == "jsmpeg") { if (cameraActive || !showStillWithoutActivity || liveReady) { player = ( ([]); + const bufferIndex = useRef(0); const [wsState, setWsState] = useState(WebSocket.CLOSED); const [connectTS, setConnectTS] = useState(0); @@ -133,6 +139,13 @@ function MSEPlayer({ } }, [bufferTimeout]); + const handlePause = useCallback(() => { + // don't let the user pause the live stream + if (isPlaying && playbackEnabled) { + videoRef.current?.play(); + } + }, [isPlaying, playbackEnabled]); + const onOpen = () => { setWsState(WebSocket.OPEN); @@ -193,6 +206,7 @@ function MSEPlayer({ const onMse = () => { if ("ManagedMediaSource" in window) { + // safari const MediaSource = window.ManagedMediaSource; msRef.current?.addEventListener( @@ -224,6 +238,7 @@ function MSEPlayer({ videoRef.current.srcObject = msRef.current; } } else { + // non safari msRef.current?.addEventListener( "sourceopen", () => { @@ -247,15 +262,35 @@ function MSEPlayer({ }, { once: true }, ); - videoRef.current!.src = URL.createObjectURL(msRef.current!); - videoRef.current!.srcObject = null; + if (videoRef.current && msRef.current) { + videoRef.current.src = URL.createObjectURL(msRef.current); + videoRef.current.srcObject = null; + } } play(); onmessageRef.current["mse"] = (msg) => { if (msg.type !== "mse") return; - const sb = msRef.current?.addSourceBuffer(msg.value); + let sb: SourceBuffer | undefined; + try { + sb = msRef.current?.addSourceBuffer(msg.value); + if (sb?.mode) { + sb.mode = "segments"; + } + } catch (e) { + // Safari sometimes throws this error + if (e instanceof DOMException && e.name === "InvalidStateError") { + if (wsRef.current) { + onDisconnect(); + } + onError?.("mse-decode"); + return; + } else { + throw e; // Re-throw if it's not the error we're handling + } + } + sb?.addEventListener("updateend", () => { if (sb.updating) return; @@ -302,6 +337,43 @@ function MSEPlayer({ return video.buffered.end(video.buffered.length - 1) - video.currentTime; }; + const jumpToLive = () => { + if (!videoRef.current) return; + + const buffered = videoRef.current.buffered; + if (buffered.length > 0) { + const liveEdge = buffered.end(buffered.length - 1); + // Jump to the live edge + videoRef.current.currentTime = liveEdge - 0.75; + lastJumpTimeRef.current = Date.now(); + } + }; + + const calculateAdaptiveBufferThreshold = () => { + const filledEntries = bufferTimes.current.length; + const sum = bufferTimes.current.reduce((a, b) => a + b, 0); + const averageBufferTime = filledEntries ? sum / filledEntries : 0; + return averageBufferTime * (isSafari || isIOS ? 3 : 1.5); + }; + + const calculateAdaptivePlaybackRate = ( + bufferTime: number, + bufferThreshold: number, + ) => { + const alpha = 0.2; // aggressiveness of playback rate increase + const beta = 0.5; // steepness of exponential growth + + // don't adjust playback rate if we're close enough to live + if ( + (bufferTime <= bufferThreshold && bufferThreshold < 3) || + bufferTime < 3 + ) { + return 1; + } + const rate = 1 + alpha * Math.exp(beta * bufferTime - bufferThreshold); + return Math.min(rate, 2); + }; + useEffect(() => { if (!playbackEnabled) { return; @@ -386,21 +458,71 @@ function MSEPlayer({ handleLoadedMetadata?.(); onPlaying?.(); setIsPlaying(true); + lastJumpTimeRef.current = Date.now(); }} muted={!audioEnabled} - onPause={() => videoRef.current?.play()} + onPause={handlePause} onProgress={() => { + const bufferTime = getBufferedTime(videoRef.current); + + if ( + videoRef.current && + (videoRef.current.playbackRate === 1 || bufferTime < 3) + ) { + if (bufferTimes.current.length < MAX_BUFFER_ENTRIES) { + bufferTimes.current.push(bufferTime); + } else { + bufferTimes.current[bufferIndex.current] = bufferTime; + bufferIndex.current = + (bufferIndex.current + 1) % MAX_BUFFER_ENTRIES; + } + } + + const bufferThreshold = calculateAdaptiveBufferThreshold(); + // if we have > 3 seconds of buffered data and we're still not playing, // something might be wrong - maybe codec issue, no audio, etc // so mark the player as playing so that error handlers will fire + if (!isPlaying && playbackEnabled && bufferTime > 3) { + setIsPlaying(true); + lastJumpTimeRef.current = Date.now(); + onPlaying?.(); + } + + // if we have more than 10 seconds of buffer, something's wrong so error out if ( - !isPlaying && + isPlaying && playbackEnabled && - getBufferedTime(videoRef.current) > 3 + (bufferThreshold > 10 || bufferTime > 10) ) { - setIsPlaying(true); - onPlaying?.(); + onDisconnect(); + onError?.("stalled"); } + + const playbackRate = calculateAdaptivePlaybackRate( + bufferTime, + bufferThreshold, + ); + + // if we're above our rolling average threshold or have > 3 seconds of + // buffered data and we're playing, we may have drifted from actual live + // time, so increase playback rate to compensate - non safari/ios only + if ( + videoRef.current && + isPlaying && + playbackEnabled && + Date.now() - lastJumpTimeRef.current > BUFFERING_COOLDOWN_TIMEOUT + ) { + // Jump to live on Safari/iOS due to a change of playback rate causing re-buffering + if (isSafari || isIOS) { + if (bufferTime > 3) { + jumpToLive(); + } + } else { + videoRef.current.playbackRate = playbackRate; + } + } + if (onError != undefined) { if (videoRef.current?.paused) { return; diff --git a/web/src/hooks/use-camera-live-mode.ts b/web/src/hooks/use-camera-live-mode.ts index 2a19970459..edf1659510 100644 --- a/web/src/hooks/use-camera-live-mode.ts +++ b/web/src/hooks/use-camera-live-mode.ts @@ -1,49 +1,65 @@ import { CameraConfig, FrigateConfig } from "@/types/frigateConfig"; -import { useMemo } from "react"; +import { useCallback, useEffect, useState } from "react"; import useSWR from "swr"; -import { usePersistence } from "./use-persistence"; import { LivePlayerMode } from "@/types/live"; export default function useCameraLiveMode( - cameraConfig: CameraConfig, - preferredMode?: LivePlayerMode, -): LivePlayerMode | undefined { + cameras: CameraConfig[], + windowVisible: boolean, +) { const { data: config } = useSWR("config"); + const [preferredLiveModes, setPreferredLiveModes] = useState<{ + [key: string]: LivePlayerMode; + }>({}); - const restreamEnabled = useMemo(() => { - if (!config) { - return false; - } + useEffect(() => { + if (!cameras) return; - return ( - cameraConfig && - Object.keys(config.go2rtc.streams || {}).includes( - cameraConfig.live.stream_name, - ) + const mseSupported = + "MediaSource" in window || "ManagedMediaSource" in window; + + const newPreferredLiveModes = cameras.reduce( + (acc, camera) => { + const isRestreamed = + config && + Object.keys(config.go2rtc.streams || {}).includes( + camera.live.stream_name, + ); + + if (!mseSupported) { + acc[camera.name] = isRestreamed ? "webrtc" : "jsmpeg"; + } else { + acc[camera.name] = isRestreamed ? "mse" : "jsmpeg"; + } + return acc; + }, + {} as { [key: string]: LivePlayerMode }, ); - }, [config, cameraConfig]); - const defaultLiveMode = useMemo(() => { - if (config) { - if (restreamEnabled) { - return preferredMode || "mse"; - } - - return "jsmpeg"; - } - - return undefined; - }, [config, preferredMode, restreamEnabled]); - const [viewSource] = usePersistence( - `${cameraConfig.name}-source`, - defaultLiveMode, + + setPreferredLiveModes(newPreferredLiveModes); + }, [cameras, config, windowVisible]); + + const resetPreferredLiveMode = useCallback( + (cameraName: string) => { + const mseSupported = + "MediaSource" in window || "ManagedMediaSource" in window; + const isRestreamed = + config && Object.keys(config.go2rtc.streams || {}).includes(cameraName); + + setPreferredLiveModes((prevModes) => { + const newModes = { ...prevModes }; + + if (!mseSupported) { + newModes[cameraName] = isRestreamed ? "webrtc" : "jsmpeg"; + } else { + newModes[cameraName] = isRestreamed ? "mse" : "jsmpeg"; + } + + return newModes; + }); + }, + [config], ); - if ( - restreamEnabled && - (preferredMode == "mse" || preferredMode == "webrtc") - ) { - return preferredMode; - } else { - return viewSource; - } + return { preferredLiveModes, setPreferredLiveModes, resetPreferredLiveMode }; } diff --git a/web/src/types/frigateConfig.ts b/web/src/types/frigateConfig.ts index 26228dbaa3..a38cbd847f 100644 --- a/web/src/types/frigateConfig.ts +++ b/web/src/types/frigateConfig.ts @@ -298,7 +298,12 @@ export interface FrigateConfig { retry_interval: number; }; - go2rtc: Record; + go2rtc: { + streams: string[]; + webrtc: { + candidates: string[]; + }; + }; camera_groups: { [groupName: string]: CameraGroupConfig }; diff --git a/web/src/views/live/DraggableGridLayout.tsx b/web/src/views/live/DraggableGridLayout.tsx index 165829719d..fc2d9bb528 100644 --- a/web/src/views/live/DraggableGridLayout.tsx +++ b/web/src/views/live/DraggableGridLayout.tsx @@ -41,6 +41,7 @@ import { TooltipContent, } from "@/components/ui/tooltip"; import { Toaster } from "@/components/ui/sonner"; +import useCameraLiveMode from "@/hooks/use-camera-live-mode"; type DraggableGridLayoutProps = { cameras: CameraConfig[]; @@ -75,36 +76,8 @@ export default function DraggableGridLayout({ // preferred live modes per camera - const [preferredLiveModes, setPreferredLiveModes] = useState<{ - [key: string]: LivePlayerMode; - }>({}); - - useEffect(() => { - if (!cameras) return; - - const mseSupported = - "MediaSource" in window || "ManagedMediaSource" in window; - - const newPreferredLiveModes = cameras.reduce( - (acc, camera) => { - const isRestreamed = - config && - Object.keys(config.go2rtc.streams || {}).includes( - camera.live.stream_name, - ); - - if (!mseSupported) { - acc[camera.name] = isRestreamed ? "webrtc" : "jsmpeg"; - } else { - acc[camera.name] = isRestreamed ? "mse" : "jsmpeg"; - } - return acc; - }, - {} as { [key: string]: LivePlayerMode }, - ); - - setPreferredLiveModes(newPreferredLiveModes); - }, [cameras, config, windowVisible]); + const { preferredLiveModes, setPreferredLiveModes, resetPreferredLiveMode } = + useCameraLiveMode(cameras, windowVisible); const ResponsiveGridLayout = useMemo(() => WidthProvider(Responsive), []); @@ -477,6 +450,7 @@ export default function DraggableGridLayout({ return newModes; }); }} + onResetLiveMode={() => resetPreferredLiveMode(camera.name)} > {isEditMode && showCircles && } @@ -635,6 +609,7 @@ type LivePlayerGridItemProps = { preferredLiveMode: LivePlayerMode; onClick: () => void; onError: (e: LivePlayerError) => void; + onResetLiveMode: () => void; }; const LivePlayerGridItem = React.forwardRef< @@ -655,6 +630,7 @@ const LivePlayerGridItem = React.forwardRef< preferredLiveMode, onClick, onError, + onResetLiveMode, ...props }, ref, @@ -676,6 +652,7 @@ const LivePlayerGridItem = React.forwardRef< preferredLiveMode={preferredLiveMode} onClick={onClick} onError={onError} + onResetLiveMode={onResetLiveMode} containerRef={ref as React.RefObject} /> {children} diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx index db162198c1..9dab7d9166 100644 --- a/web/src/views/live/LiveCameraView.tsx +++ b/web/src/views/live/LiveCameraView.tsx @@ -227,6 +227,10 @@ export default function LiveCameraView({ return "webrtc"; } + if (!isRestreamed) { + return "jsmpeg"; + } + return "mse"; }, [lowBandwidth, mic, webRTC, isRestreamed]); @@ -286,14 +290,23 @@ export default function LiveCameraView({ } }, [fullscreen, isPortrait, cameraAspectRatio, containerAspectRatio]); - const handleError = useCallback((e: LivePlayerError) => { - if (e == "mse-decode") { - setWebRTC(true); - } else { - setWebRTC(false); - setLowBandwidth(true); - } - }, []); + const handleError = useCallback( + (e: LivePlayerError) => { + if (e) { + if ( + !webRTC && + config && + config.go2rtc?.webrtc?.candidates?.length > 0 + ) { + setWebRTC(true); + } else { + setWebRTC(false); + setLowBandwidth(true); + } + } + }, + [config, webRTC], + ); return ( diff --git a/web/src/views/live/LiveDashboardView.tsx b/web/src/views/live/LiveDashboardView.tsx index 00c0462243..cac604e268 100644 --- a/web/src/views/live/LiveDashboardView.tsx +++ b/web/src/views/live/LiveDashboardView.tsx @@ -28,8 +28,9 @@ import DraggableGridLayout from "./DraggableGridLayout"; import { IoClose } from "react-icons/io5"; import { LuLayoutDashboard } from "react-icons/lu"; import { cn } from "@/lib/utils"; -import { LivePlayerError, LivePlayerMode } from "@/types/live"; +import { LivePlayerError } from "@/types/live"; import { FaCompress, FaExpand } from "react-icons/fa"; +import useCameraLiveMode from "@/hooks/use-camera-live-mode"; import { useResizeObserver } from "@/hooks/resize-observer"; type LiveDashboardViewProps = { @@ -129,9 +130,6 @@ export default function LiveDashboardView({ // camera live views const [autoLiveView] = usePersistence("autoLiveView", true); - const [preferredLiveModes, setPreferredLiveModes] = useState<{ - [key: string]: LivePlayerMode; - }>({}); const [{ height: containerHeight }] = useResizeObserver(containerRef); @@ -186,32 +184,8 @@ export default function LiveDashboardView({ }; }, []); - useEffect(() => { - if (!cameras) return; - - const mseSupported = - "MediaSource" in window || "ManagedMediaSource" in window; - - const newPreferredLiveModes = cameras.reduce( - (acc, camera) => { - const isRestreamed = - config && - Object.keys(config.go2rtc.streams || {}).includes( - camera.live.stream_name, - ); - - if (!mseSupported) { - acc[camera.name] = isRestreamed ? "webrtc" : "jsmpeg"; - } else { - acc[camera.name] = isRestreamed ? "mse" : "jsmpeg"; - } - return acc; - }, - {} as { [key: string]: LivePlayerMode }, - ); - - setPreferredLiveModes(newPreferredLiveModes); - }, [cameras, config, windowVisible]); + const { preferredLiveModes, setPreferredLiveModes, resetPreferredLiveMode } = + useCameraLiveMode(cameras, windowVisible); const cameraRef = useCallback( (node: HTMLElement | null) => { @@ -381,6 +355,7 @@ export default function LiveDashboardView({ autoLive={autoLiveView} onClick={() => onSelectCamera(camera.name)} onError={(e) => handleError(camera.name, e)} + onResetLiveMode={() => resetPreferredLiveMode(camera.name)} /> ); })} From 3a124dbb845526215ad5b5b15e6ac7c3d3754ab9 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Sun, 18 Aug 2024 07:41:10 -0600 Subject: [PATCH 035/169] Fix plus view resetting (#13160) --- web/src/pages/SubmitPlus.tsx | 235 +++++++++++++++++++---------------- 1 file changed, 126 insertions(+), 109 deletions(-) diff --git a/web/src/pages/SubmitPlus.tsx b/web/src/pages/SubmitPlus.tsx index 9d2b9ae5bd..9424a3350f 100644 --- a/web/src/pages/SubmitPlus.tsx +++ b/web/src/pages/SubmitPlus.tsx @@ -242,119 +242,136 @@ export default function SubmitPlus() {
- {isValidating ? ( - - ) : events?.length === 0 ? ( -
- - No snapshots found -
+ {!events?.length ? ( + <> + {isValidating ? ( + + ) : ( +
+ + No snapshots found +
+ )} + ) : ( -
- (!open ? setUpload(undefined) : null)} - > - - - - Submit To Frigate+ - - Objects in locations you want to avoid are not false - positives. Submitting them as false positives will confuse - the model. - - - +
+ (!open ? setUpload(undefined) : null)} + > + + - {upload?.id && ( - {`${upload?.label}`} - )} - - - - - - - - - - - {events?.map((event) => { - if (event.data.type != "object" || event.plus_id) { - return; - } - - return ( -
setUpload(event)} - > -
- -
- -
- - {[event.label].map((object) => { - return getIconForLabel( - object, - "size-3 text-white", - ); - })} -
- {Math.round(event.data.score * 100)}% -
- -
- -
- - {[event.label] - .map((text) => capitalizeFirstLetter(text)) - .sort() - .join(", ") - .replaceAll("-verified", "")} - - + {upload?.id && ( + {`${upload?.label}`} + )} + + + + + + + + +
+ + {events?.map((event) => { + if (event.data.type != "object" || event.plus_id) { + return; + } + + return ( +
setUpload(event)} + > +
+ +
+ +
+ + {[event.label].map((object) => { + return getIconForLabel( + object, + "size-3 text-white", + ); + })} +
+ {Math.round(event.data.score * 100)}% +
+ +
+ +
+ + {[event.label] + .map((text) => capitalizeFirstLetter(text)) + .sort() + .join(", ") + .replaceAll("-verified", "")} + + +
+
- -
- ); - })} - {!isValidating && !isDone &&
} -
+ ); + })} +
+ {!isDone && isValidating ? ( +
+ +
+ ) : ( +
+ )} + )}
From 8e31244fb37b0ce5d6860d43c35688d1342e934e Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 18 Aug 2024 13:13:21 -0500 Subject: [PATCH 036/169] Adjust MSE player playback rate logic (#13164) * Fix MSE playback rate logic * don't adjust playback rate if we just started streaming * memoize onprogress --- web/src/components/player/MsePlayer.tsx | 184 +++++++++++++----------- 1 file changed, 97 insertions(+), 87 deletions(-) diff --git a/web/src/components/player/MsePlayer.tsx b/web/src/components/player/MsePlayer.tsx index 187567be63..52cf8f99c7 100644 --- a/web/src/components/player/MsePlayer.tsx +++ b/web/src/components/player/MsePlayer.tsx @@ -364,9 +364,11 @@ function MSEPlayer({ const beta = 0.5; // steepness of exponential growth // don't adjust playback rate if we're close enough to live + // or if we just started streaming if ( - (bufferTime <= bufferThreshold && bufferThreshold < 3) || - bufferTime < 3 + ((bufferTime <= bufferThreshold && bufferThreshold < 3) || + bufferTime < 3) && + bufferTimes.current.length <= MAX_BUFFER_ENTRIES ) { return 1; } @@ -374,6 +376,98 @@ function MSEPlayer({ return Math.min(rate, 2); }; + const onProgress = useCallback(() => { + const bufferTime = getBufferedTime(videoRef.current); + + if ( + videoRef.current && + (videoRef.current.playbackRate === 1 || bufferTime < 3) + ) { + if (bufferTimes.current.length < MAX_BUFFER_ENTRIES) { + bufferTimes.current.push(bufferTime); + } else { + bufferTimes.current[bufferIndex.current] = bufferTime; + bufferIndex.current = (bufferIndex.current + 1) % MAX_BUFFER_ENTRIES; + } + } + + const bufferThreshold = calculateAdaptiveBufferThreshold(); + + // if we have > 3 seconds of buffered data and we're still not playing, + // something might be wrong - maybe codec issue, no audio, etc + // so mark the player as playing so that error handlers will fire + if (!isPlaying && playbackEnabled && bufferTime > 3) { + setIsPlaying(true); + lastJumpTimeRef.current = Date.now(); + onPlaying?.(); + } + + // if we have more than 10 seconds of buffer, something's wrong so error out + if ( + isPlaying && + playbackEnabled && + (bufferThreshold > 10 || bufferTime > 10) + ) { + onDisconnect(); + onError?.("stalled"); + } + + const playbackRate = calculateAdaptivePlaybackRate( + bufferTime, + bufferThreshold, + ); + + // if we're above our rolling average threshold or have > 3 seconds of + // buffered data and we're playing, we may have drifted from actual live + // time + if (videoRef.current && isPlaying && playbackEnabled) { + if ( + (isSafari || isIOS) && + bufferTime > 3 && + Date.now() - lastJumpTimeRef.current > BUFFERING_COOLDOWN_TIMEOUT + ) { + // Jump to live on Safari/iOS due to a change of playback rate causing re-buffering + jumpToLive(); + } else { + // increase/decrease playback rate to compensate - non Safari/iOS only + if (videoRef.current.playbackRate !== playbackRate) { + videoRef.current.playbackRate = playbackRate; + } + } + } + + if (onError != undefined) { + if (videoRef.current?.paused) { + return; + } + + if (bufferTimeout) { + clearTimeout(bufferTimeout); + setBufferTimeout(undefined); + } + + setBufferTimeout( + setTimeout(() => { + if ( + document.visibilityState === "visible" && + wsRef.current != null && + videoRef.current + ) { + onDisconnect(); + onError("stalled"); + } + }, 3000), + ); + } + }, [ + bufferTimeout, + isPlaying, + onDisconnect, + onError, + onPlaying, + playbackEnabled, + ]); + useEffect(() => { if (!playbackEnabled) { return; @@ -462,91 +556,7 @@ function MSEPlayer({ }} muted={!audioEnabled} onPause={handlePause} - onProgress={() => { - const bufferTime = getBufferedTime(videoRef.current); - - if ( - videoRef.current && - (videoRef.current.playbackRate === 1 || bufferTime < 3) - ) { - if (bufferTimes.current.length < MAX_BUFFER_ENTRIES) { - bufferTimes.current.push(bufferTime); - } else { - bufferTimes.current[bufferIndex.current] = bufferTime; - bufferIndex.current = - (bufferIndex.current + 1) % MAX_BUFFER_ENTRIES; - } - } - - const bufferThreshold = calculateAdaptiveBufferThreshold(); - - // if we have > 3 seconds of buffered data and we're still not playing, - // something might be wrong - maybe codec issue, no audio, etc - // so mark the player as playing so that error handlers will fire - if (!isPlaying && playbackEnabled && bufferTime > 3) { - setIsPlaying(true); - lastJumpTimeRef.current = Date.now(); - onPlaying?.(); - } - - // if we have more than 10 seconds of buffer, something's wrong so error out - if ( - isPlaying && - playbackEnabled && - (bufferThreshold > 10 || bufferTime > 10) - ) { - onDisconnect(); - onError?.("stalled"); - } - - const playbackRate = calculateAdaptivePlaybackRate( - bufferTime, - bufferThreshold, - ); - - // if we're above our rolling average threshold or have > 3 seconds of - // buffered data and we're playing, we may have drifted from actual live - // time, so increase playback rate to compensate - non safari/ios only - if ( - videoRef.current && - isPlaying && - playbackEnabled && - Date.now() - lastJumpTimeRef.current > BUFFERING_COOLDOWN_TIMEOUT - ) { - // Jump to live on Safari/iOS due to a change of playback rate causing re-buffering - if (isSafari || isIOS) { - if (bufferTime > 3) { - jumpToLive(); - } - } else { - videoRef.current.playbackRate = playbackRate; - } - } - - if (onError != undefined) { - if (videoRef.current?.paused) { - return; - } - - if (bufferTimeout) { - clearTimeout(bufferTimeout); - setBufferTimeout(undefined); - } - - setBufferTimeout( - setTimeout(() => { - if ( - document.visibilityState === "visible" && - wsRef.current != null && - videoRef.current - ) { - onDisconnect(); - onError("stalled"); - } - }, 3000), - ); - } - }} + onProgress={onProgress} onError={(e) => { if ( // @ts-expect-error code does exist From 38a8d34ba59d13710746d05f556ec693b8c044f4 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 19 Aug 2024 10:45:55 -0600 Subject: [PATCH 037/169] Preview fixes (#13193) * Handle case where preview was saved late * fix timing --- web/src/components/player/PreviewPlayer.tsx | 15 ++++++------- web/src/hooks/use-camera-previews.ts | 24 +++++++++++++++++++-- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/web/src/components/player/PreviewPlayer.tsx b/web/src/components/player/PreviewPlayer.tsx index 3b314f9a6f..8c0394abd2 100644 --- a/web/src/components/player/PreviewPlayer.tsx +++ b/web/src/components/player/PreviewPlayer.tsx @@ -16,6 +16,7 @@ import { isAndroid, isChrome, isMobile } from "react-device-detect"; import { TimeRange } from "@/types/timeline"; import { Skeleton } from "../ui/skeleton"; import { cn } from "@/lib/utils"; +import { usePreviewForTimeRange } from "@/hooks/use-camera-previews"; type PreviewPlayerProps = { className?: string; @@ -39,15 +40,11 @@ export default function PreviewPlayer({ onClick, }: PreviewPlayerProps) { const [currentHourFrame, setCurrentHourFrame] = useState(); - - const currentPreview = useMemo(() => { - return cameraPreviews.find( - (preview) => - preview.camera == camera && - Math.round(preview.start) >= timeRange.after && - Math.floor(preview.end) <= timeRange.before, - ); - }, [cameraPreviews, camera, timeRange]); + const currentPreview = usePreviewForTimeRange( + cameraPreviews, + camera, + timeRange, + ); if (currentPreview) { return ( diff --git a/web/src/hooks/use-camera-previews.ts b/web/src/hooks/use-camera-previews.ts index 3bdbd7efbd..040b1081be 100644 --- a/web/src/hooks/use-camera-previews.ts +++ b/web/src/hooks/use-camera-previews.ts @@ -1,6 +1,6 @@ import { Preview } from "@/types/preview"; import { TimeRange } from "@/types/timeline"; -import { useEffect, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; import useSWR from "swr"; type OptionalCameraPreviewProps = { @@ -8,7 +8,6 @@ type OptionalCameraPreviewProps = { autoRefresh?: boolean; fetchPreviews?: boolean; }; - export function useCameraPreviews( initialTimeRange: TimeRange, { @@ -32,3 +31,24 @@ export function useCameraPreviews( return allPreviews; } + +// we need to add a buffer of 5 seconds to the end preview times +// this ensures that if preview generation is running slowly +// and the previews are generated 1-5 seconds late +// it is not falsely thrown out. +const PREVIEW_END_BUFFER = 5; // seconds + +export function usePreviewForTimeRange( + allPreviews: Preview[], + camera: string, + timeRange: TimeRange, +) { + return useMemo(() => { + return allPreviews.find( + (preview) => + preview.camera == camera && + Math.ceil(preview.start) >= timeRange.after && + Math.floor(preview.end) <= timeRange.before + PREVIEW_END_BUFFER, + ); + }, [allPreviews, camera, timeRange]); +} From 1da934e63c1f4af941461c9f99397f45aa15e3b4 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 19 Aug 2024 15:01:21 -0600 Subject: [PATCH 038/169] Dynamically detect if full screen is supported (#13197) --- web/src/components/player/HlsVideoPlayer.tsx | 6 ++-- .../player/dynamic/DynamicVideoPlayer.tsx | 3 ++ web/src/hooks/use-fullscreen.ts | 30 +++++++++++++++++-- web/src/pages/Live.tsx | 5 +++- web/src/views/events/RecordingView.tsx | 4 ++- web/src/views/live/LiveBirdseyeView.tsx | 20 ++++++++----- web/src/views/live/LiveCameraView.tsx | 4 ++- 7 files changed, 57 insertions(+), 15 deletions(-) diff --git a/web/src/components/player/HlsVideoPlayer.tsx b/web/src/components/player/HlsVideoPlayer.tsx index 5f499ade76..7f49a6cadc 100644 --- a/web/src/components/player/HlsVideoPlayer.tsx +++ b/web/src/components/player/HlsVideoPlayer.tsx @@ -6,7 +6,7 @@ import { useState, } from "react"; import Hls from "hls.js"; -import { isAndroid, isDesktop, isIOS, isMobile } from "react-device-detect"; +import { isAndroid, isDesktop, isMobile } from "react-device-detect"; import { TransformComponent, TransformWrapper } from "react-zoom-pan-pinch"; import VideoControls from "./VideoControls"; import { VideoResolutionType } from "@/types/live"; @@ -33,6 +33,7 @@ type HlsVideoPlayerProps = { visible: boolean; currentSource: string; hotKeys: boolean; + supportsFullscreen: boolean; fullscreen: boolean; onClipEnded?: () => void; onPlayerLoaded?: () => void; @@ -49,6 +50,7 @@ export default function HlsVideoPlayer({ visible, currentSource, hotKeys, + supportsFullscreen, fullscreen, onClipEnded, onPlayerLoaded, @@ -180,7 +182,7 @@ export default function HlsVideoPlayer({ seek: true, playbackRate: true, plusUpload: config?.plus?.enabled == true, - fullscreen: !isIOS, + fullscreen: supportsFullscreen, }} setControlsOpen={setControlsOpen} setMuted={(muted) => setMuted(muted, true)} diff --git a/web/src/components/player/dynamic/DynamicVideoPlayer.tsx b/web/src/components/player/dynamic/DynamicVideoPlayer.tsx index caa709430f..6c4e28e27e 100644 --- a/web/src/components/player/dynamic/DynamicVideoPlayer.tsx +++ b/web/src/components/player/dynamic/DynamicVideoPlayer.tsx @@ -24,6 +24,7 @@ type DynamicVideoPlayerProps = { startTimestamp?: number; isScrubbing: boolean; hotKeys: boolean; + supportsFullscreen: boolean; fullscreen: boolean; onControllerReady: (controller: DynamicVideoController) => void; onTimestampUpdate?: (timestamp: number) => void; @@ -40,6 +41,7 @@ export default function DynamicVideoPlayer({ startTimestamp, isScrubbing, hotKeys, + supportsFullscreen, fullscreen, onControllerReady, onTimestampUpdate, @@ -201,6 +203,7 @@ export default function DynamicVideoPlayer({ visible={!(isScrubbing || isLoading)} currentSource={source} hotKeys={hotKeys} + supportsFullscreen={supportsFullscreen} fullscreen={fullscreen} onTimeUpdate={onTimeUpdate} onPlayerLoaded={onPlayerLoaded} diff --git a/web/src/hooks/use-fullscreen.ts b/web/src/hooks/use-fullscreen.ts index a0f0a1e56f..c7082ab5c7 100644 --- a/web/src/hooks/use-fullscreen.ts +++ b/web/src/hooks/use-fullscreen.ts @@ -1,4 +1,4 @@ -import { RefObject, useCallback, useEffect, useState } from "react"; +import { RefObject, useCallback, useEffect, useMemo, useState } from "react"; import nosleep from "nosleep.js"; const NoSleep = new nosleep(); @@ -147,5 +147,31 @@ export function useFullscreen( } }, [elementRef, handleFullscreenChange, handleFullscreenError]); - return { fullscreen, toggleFullscreen, error, clearError }; + // compatibility + + const supportsFullScreen = useMemo(() => { + // @ts-expect-error we need to check that fullscreen exists + if (document.exitFullscreen) return true; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if ((document as any).msExitFullscreen) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return true; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if ((document as any).webkitExitFullscreen) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return true; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if ((document as any).mozCancelFullScreen) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return true; + return false; + }, []); + + return { + fullscreen, + toggleFullscreen, + supportsFullScreen, + error, + clearError, + }; } diff --git a/web/src/pages/Live.tsx b/web/src/pages/Live.tsx index cd8cc6b0ae..c088a5b044 100644 --- a/web/src/pages/Live.tsx +++ b/web/src/pages/Live.tsx @@ -36,7 +36,8 @@ function Live() { const mainRef = useRef(null); - const { fullscreen, toggleFullscreen } = useFullscreen(mainRef); + const { fullscreen, toggleFullscreen, supportsFullScreen } = + useFullscreen(mainRef); // document title @@ -100,6 +101,7 @@ function Live() {
{selectedCameraName === "birdseye" ? ( @@ -107,6 +109,7 @@ function Live() { diff --git a/web/src/views/events/RecordingView.tsx b/web/src/views/events/RecordingView.tsx index 1de3a690f1..f01da95788 100644 --- a/web/src/views/events/RecordingView.tsx +++ b/web/src/views/events/RecordingView.tsx @@ -257,7 +257,8 @@ export function RecordingView({ // fullscreen - const { fullscreen, toggleFullscreen } = useFullscreen(mainLayoutRef); + const { fullscreen, toggleFullscreen, supportsFullScreen } = + useFullscreen(mainLayoutRef); // layout @@ -549,6 +550,7 @@ export function RecordingView({ mainControllerRef.current = controller; }} isScrubbing={scrubbing || exportMode == "timeline"} + supportsFullscreen={supportsFullScreen} setFullResolution={setFullResolution} toggleFullscreen={toggleFullscreen} containerRef={mainLayoutRef} diff --git a/web/src/views/live/LiveBirdseyeView.tsx b/web/src/views/live/LiveBirdseyeView.tsx index 70e20e77a8..be69b43c8c 100644 --- a/web/src/views/live/LiveBirdseyeView.tsx +++ b/web/src/views/live/LiveBirdseyeView.tsx @@ -22,11 +22,13 @@ import { TransformWrapper, TransformComponent } from "react-zoom-pan-pinch"; import useSWR from "swr"; type LiveBirdseyeViewProps = { + supportsFullscreen: boolean; fullscreen: boolean; toggleFullscreen: () => void; }; export default function LiveBirdseyeView({ + supportsFullscreen, fullscreen, toggleFullscreen, }: LiveBirdseyeViewProps) { @@ -155,14 +157,16 @@ export default function LiveBirdseyeView({
- + {supportsFullscreen && ( + + )} {!isIOS && !isFirefox && config.birdseye.restream && ( void; }; export default function LiveCameraView({ config, camera, + supportsFullscreen, fullscreen, toggleFullscreen, }: LiveCameraViewProps) { @@ -376,7 +378,7 @@ export default function LiveCameraView({ )} )} - {!isIOS && ( + {supportsFullscreen && ( Date: Mon, 19 Aug 2024 15:01:48 -0600 Subject: [PATCH 039/169] Ensure only enabled birdseye cameras are considered active (#13194) * Ensure only enabled birdseye cameras are considered active * Cleanup --- frigate/output/birdseye.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frigate/output/birdseye.py b/frigate/output/birdseye.py index 2b17a4cf1e..6e0e2bc22e 100644 --- a/frigate/output/birdseye.py +++ b/frigate/output/birdseye.py @@ -395,7 +395,8 @@ def update_frame(self): [ cam for cam, cam_data in self.cameras.items() - if cam_data["last_active_frame"] > 0 + if self.config.cameras[cam].birdseye.enabled + and cam_data["last_active_frame"] > 0 and cam_data["current_frame"] - cam_data["last_active_frame"] < self.inactivity_threshold ] From 1c7ee5f4e404084de68e20b1fb253b076c1c19a9 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Wed, 21 Aug 2024 08:19:07 -0600 Subject: [PATCH 040/169] UI fixes (#13246) * Fix bad data in stats * Add support for changes dialog when leaving without saving config editor * Fix scrolling into view --- web/src/pages/ConfigEditor.tsx | 37 +++++++++++++++++++++++++ web/src/pages/Logs.tsx | 1 + web/src/views/system/GeneralMetrics.tsx | 16 +++++++---- 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/web/src/pages/ConfigEditor.tsx b/web/src/pages/ConfigEditor.tsx index 55df04e343..dc351e7f78 100644 --- a/web/src/pages/ConfigEditor.tsx +++ b/web/src/pages/ConfigEditor.tsx @@ -124,12 +124,49 @@ function ConfigEditor() { }; }); + // monitoring state + + const [hasChanges, setHasChanges] = useState(false); + + useEffect(() => { + if (!config || !modelRef.current) { + return; + } + + modelRef.current.onDidChangeContent(() => { + if (modelRef.current?.getValue() != config) { + setHasChanges(true); + } else { + setHasChanges(false); + } + }); + }, [config]); + useEffect(() => { if (config && modelRef.current) { modelRef.current.setValue(config); + setHasChanges(false); } }, [config]); + useEffect(() => { + let listener: ((e: BeforeUnloadEvent) => void) | undefined; + if (hasChanges) { + listener = (e) => { + e.preventDefault(); + e.returnValue = true; + return "Exit without saving?"; + }; + window.addEventListener("beforeunload", listener); + } + + return () => { + if (listener) { + window.removeEventListener("beforeunload", listener); + } + }; + }, [hasChanges]); + if (!config) { return ; } diff --git a/web/src/pages/Logs.tsx b/web/src/pages/Logs.tsx index 0691cb6359..5821900980 100644 --- a/web/src/pages/Logs.tsx +++ b/web/src/pages/Logs.tsx @@ -286,6 +286,7 @@ function Logs() { key={item} className={`flex items-center justify-between gap-2 ${logService == item ? "" : "text-muted-foreground"}`} value={item} + data-nav-item={item} aria-label={`Select ${item}`} >
{item}
diff --git a/web/src/views/system/GeneralMetrics.tsx b/web/src/views/system/GeneralMetrics.tsx index fa23d47b0f..f617e96545 100644 --- a/web/src/views/system/GeneralMetrics.tsx +++ b/web/src/views/system/GeneralMetrics.tsx @@ -163,7 +163,7 @@ export default function GeneralMetrics({ series[key] = { name: key, data: [] }; } - const data = stats.cpu_usages[detStats.pid.toString()].cpu; + const data = stats.cpu_usages[detStats.pid.toString()]?.cpu; if (data != undefined) { series[key].data.push({ @@ -304,7 +304,7 @@ export default function GeneralMetrics({ series[key] = { name: key, data: [] }; } - const data = stats.cpu_usages[procStats.pid.toString()].cpu; + const data = stats.cpu_usages[procStats.pid.toString()]?.cpu; if (data != undefined) { series[key].data.push({ @@ -338,10 +338,14 @@ export default function GeneralMetrics({ series[key] = { name: key, data: [] }; } - series[key].data.push({ - x: statsIdx + 1, - y: stats.cpu_usages[procStats.pid.toString()].mem, - }); + const data = stats.cpu_usages[procStats.pid.toString()]?.mem; + + if (data) { + series[key].data.push({ + x: statsIdx + 1, + y: data, + }); + } } }); }); From e01b6ee76b83dcea988f80d8c31f2c62615d5f5a Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Thu, 22 Aug 2024 07:06:26 -0600 Subject: [PATCH 041/169] Fix case where user's cgroup says it has 0 cpu cores (#13271) --- docker/main/rootfs/etc/s6-overlay/s6-rc.d/nginx/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/nginx/run b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/nginx/run index a6436abd41..677126a6dd 100755 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/nginx/run +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/nginx/run @@ -38,7 +38,7 @@ function get_cpus() { fi local cpus - if [ -n "${quota}" ] && [ -n "${period}" ]; then + if [ "${period}" != "0" ] && [ -n "${quota}" ] && [ -n "${period}" ]; then cpus=$((quota / period)) if [ "$cpus" -eq 0 ]; then cpus=1 From ff34af2c1f90d83ba47784c602f4b38598ff39e9 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 23 Aug 2024 07:44:31 -0500 Subject: [PATCH 042/169] Update discussion templates (#13291) * Revamp support discussion templates * move text to description * remove duplicate logs box * ffprobe on camera support * longer description on config support --- .../DISCUSSION_TEMPLATE/camera-support.yml | 41 +++++++++++-- .../DISCUSSION_TEMPLATE/config-support.yml | 18 +++++- .../DISCUSSION_TEMPLATE/detector-support.yml | 37 +++++++++-- .../DISCUSSION_TEMPLATE/general-support.yml | 39 ++++++++++-- .../hardware-acceleration-support.yml | 39 ++++++++++-- .github/DISCUSSION_TEMPLATE/question.yml | 14 ++++- .github/DISCUSSION_TEMPLATE/report-a-bug.yml | 61 +++++++++++++++++-- 7 files changed, 222 insertions(+), 27 deletions(-) diff --git a/.github/DISCUSSION_TEMPLATE/camera-support.yml b/.github/DISCUSSION_TEMPLATE/camera-support.yml index 6f99790df3..ec64973092 100644 --- a/.github/DISCUSSION_TEMPLATE/camera-support.yml +++ b/.github/DISCUSSION_TEMPLATE/camera-support.yml @@ -1,6 +1,16 @@ title: "[Camera Support]: " labels: ["support", "triage"] body: + - type: markdown + attributes: + value: | + Use this form for support or questions for an issue with your cameras. + + Before submitting your support request, please [search the discussions][discussions], read the [official Frigate documentation][docs], and read the [Frigate FAQ][faq] pinned at the Discussion page to see if your question has already been answered by the community. + + [discussions]: https://www.github.com/blakeblackshear/frigate/discussions + [docs]: https://docs.frigate.video + [faq]: https://github.com/blakeblackshear/frigate/discussions/12724 - type: textarea id: description attributes: @@ -11,9 +21,15 @@ body: id: version attributes: label: Version - description: Visible on the System page in the Web UI + description: Visible on the System page in the Web UI. Please include the full version including the build identifier (eg. 0.14.0-ea36ds1) validations: required: true + - type: input + attributes: + label: What browser(s) are you using? + placeholder: Google Chrome 88.0.4324.150 + description: > + Provide the full name and don't forget to add the version! - type: textarea id: config attributes: @@ -23,10 +39,18 @@ body: validations: required: true - type: textarea - id: logs + id: frigatelogs + attributes: + label: Relevant Frigate log output + description: Please copy and paste any relevant Frigate log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. + render: shell + validations: + required: true + - type: textarea + id: go2rtclogs attributes: - label: Relevant log output - description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + label: Relevant go2rtc log output + description: Please copy and paste any relevant go2rtc log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. render: shell validations: required: true @@ -34,7 +58,7 @@ body: id: ffprobe attributes: label: FFprobe output from your camera - description: Run `ffprobe ` and provide output below + description: Run `ffprobe ` from within the Frigate container if possible, and provide output below render: shell validations: required: true @@ -78,7 +102,7 @@ body: - TensorRT - RKNN - Other - - CPU (no coral) + - CPU (no Coral) validations: required: true - type: dropdown @@ -98,6 +122,11 @@ body: description: Dahua, hikvision, amcrest, reolink, etc and model number validations: required: true + - type: textarea + id: screenshots + attributes: + label: Screenshots of the Frigate UI's System metrics pages and any other screenshots pertinent to your issue. + description: Drag and drop for images is possible in this field. - type: textarea id: other attributes: diff --git a/.github/DISCUSSION_TEMPLATE/config-support.yml b/.github/DISCUSSION_TEMPLATE/config-support.yml index 3a9265f448..72b11e634f 100644 --- a/.github/DISCUSSION_TEMPLATE/config-support.yml +++ b/.github/DISCUSSION_TEMPLATE/config-support.yml @@ -1,6 +1,16 @@ title: "[Config Support]: " labels: ["support", "triage"] body: + - type: markdown + attributes: + value: | + Use this form for support or questions related to Frigate's configuration and config file. + + Before submitting your support request, please [search the discussions][discussions], read the [official Frigate documentation][docs], and read the [Frigate FAQ][faq] pinned at the Discussion page to see if your question has already been answered by the community. + + [discussions]: https://www.github.com/blakeblackshear/frigate/discussions + [docs]: https://docs.frigate.video + [faq]: https://github.com/blakeblackshear/frigate/discussions/12724 - type: textarea id: description attributes: @@ -11,7 +21,7 @@ body: id: version attributes: label: Version - description: Visible on the System page in the Web UI + description: Visible on the System page in the Web UI. Please include the full version including the build identifier (eg. 0.14.0-ea36ds1) validations: required: true - type: textarea @@ -26,7 +36,7 @@ body: id: logs attributes: label: Relevant log output - description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + description: Please copy and paste any relevant log output, including any go2rtc and Frigate logs. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. render: shell validations: required: true @@ -73,6 +83,10 @@ body: - CPU (no coral) validations: required: true + - type: textarea + id: screenshots + attributes: + label: Screenshots of the Frigate UI's System metrics pages and any other screenshots pertinent to your issue. Drag and drop or simple cut/paste is possible in this field. - type: textarea id: other attributes: diff --git a/.github/DISCUSSION_TEMPLATE/detector-support.yml b/.github/DISCUSSION_TEMPLATE/detector-support.yml index c09aec2ec6..8d1a57ff1e 100644 --- a/.github/DISCUSSION_TEMPLATE/detector-support.yml +++ b/.github/DISCUSSION_TEMPLATE/detector-support.yml @@ -1,6 +1,16 @@ title: "[Detector Support]: " labels: ["support", "triage"] body: + - type: markdown + attributes: + value: | + Use this form for support or questions related to Frigate's object detectors. + + Before submitting your support request, please [search the discussions][discussions], read the [official Frigate documentation][docs], and read the [Frigate FAQ][faq] pinned at the Discussion page to see if your question has already been answered by the community. + + [discussions]: https://www.github.com/blakeblackshear/frigate/discussions + [docs]: https://docs.frigate.video + [faq]: https://github.com/blakeblackshear/frigate/discussions/12724 - type: textarea id: description attributes: @@ -11,9 +21,15 @@ body: id: version attributes: label: Version - description: Visible on the System page in the Web UI + description: Visible on the System page in the Web UI. Please include the full version including the build identifier (eg. 0.14.0-ea36ds1) validations: required: true + - type: input + attributes: + label: What browser(s) are you using? + placeholder: Google Chrome 88.0.4324.150 + description: > + Provide the full name and don't forget to add the version! - type: textarea id: config attributes: @@ -31,10 +47,18 @@ body: validations: required: true - type: textarea - id: logs + id: frigatelogs + attributes: + label: Relevant Frigate log output + description: Please copy and paste any relevant Frigate log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. + render: shell + validations: + required: true + - type: textarea + id: go2rtclogs attributes: - label: Relevant log output - description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + label: Relevant go2rtc log output + description: Please copy and paste any relevant go2rtc log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. render: shell validations: required: true @@ -75,6 +99,11 @@ body: - CPU (no coral) validations: required: true + - type: textarea + id: screenshots + attributes: + label: Screenshots of the Frigate UI's System metrics pages and any other screenshots pertinent to your issue. + description: Drag and drop for images is possible in this field. - type: textarea id: other attributes: diff --git a/.github/DISCUSSION_TEMPLATE/general-support.yml b/.github/DISCUSSION_TEMPLATE/general-support.yml index cb9bd1992a..b3592797cf 100644 --- a/.github/DISCUSSION_TEMPLATE/general-support.yml +++ b/.github/DISCUSSION_TEMPLATE/general-support.yml @@ -1,6 +1,16 @@ title: "[Support]: " labels: ["support", "triage"] body: + - type: markdown + attributes: + value: | + Use this form for support for issues that don't fall into any specific category. + + Before submitting your support request, please [search the discussions][discussions], read the [official Frigate documentation][docs], and read the [Frigate FAQ][faq] pinned at the Discussion page to see if your question has already been answered by the community. + + [discussions]: https://www.github.com/blakeblackshear/frigate/discussions + [docs]: https://docs.frigate.video + [faq]: https://github.com/blakeblackshear/frigate/discussions/12724 - type: textarea id: description attributes: @@ -11,9 +21,15 @@ body: id: version attributes: label: Version - description: Visible on the System page in the Web UI + description: Visible on the System page in the Web UI. Please include the full version including the build identifier (eg. 0.14.0-ea36ds1) validations: required: true + - type: input + attributes: + label: What browser(s) are you using? + placeholder: Google Chrome 88.0.4324.150 + description: > + Provide the full name and don't forget to add the version! - type: textarea id: config attributes: @@ -23,10 +39,18 @@ body: validations: required: true - type: textarea - id: logs + id: frigatelogs + attributes: + label: Relevant Frigate log output + description: Please copy and paste any relevant Frigate log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. + render: shell + validations: + required: true + - type: textarea + id: go2rtclogs attributes: - label: Relevant log output - description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + label: Relevant go2rtc log output + description: Please copy and paste any relevant go2rtc log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. render: shell validations: required: true @@ -34,7 +58,7 @@ body: id: ffprobe attributes: label: FFprobe output from your camera - description: Run `ffprobe ` and provide output below + description: Run `ffprobe ` from within the Frigate container if possible, and provide output below render: shell validations: required: true @@ -98,6 +122,11 @@ body: description: Dahua, hikvision, amcrest, reolink, etc and model number validations: required: true + - type: textarea + id: screenshots + attributes: + label: Screenshots of the Frigate UI's System metrics pages and any other screenshots pertinent to your issue. + description: Drag and drop for images is possible in this field. - type: textarea id: other attributes: diff --git a/.github/DISCUSSION_TEMPLATE/hardware-acceleration-support.yml b/.github/DISCUSSION_TEMPLATE/hardware-acceleration-support.yml index 43960c537d..e50b21b9b3 100644 --- a/.github/DISCUSSION_TEMPLATE/hardware-acceleration-support.yml +++ b/.github/DISCUSSION_TEMPLATE/hardware-acceleration-support.yml @@ -1,6 +1,16 @@ title: "[HW Accel Support]: " labels: ["support", "triage"] body: + - type: markdown + attributes: + value: | + Use this form to submit a support request for hardware acceleration issues. + + Before submitting your support request, please [search the discussions][discussions], read the [official Frigate documentation][docs], and read the [Frigate FAQ][faq] pinned at the Discussion page to see if your question has already been answered by the community. + + [discussions]: https://www.github.com/blakeblackshear/frigate/discussions + [docs]: https://docs.frigate.video + [faq]: https://github.com/blakeblackshear/frigate/discussions/12724 - type: textarea id: description attributes: @@ -11,9 +21,15 @@ body: id: version attributes: label: Version - description: Visible on the System page in the Web UI + description: Visible on the System page in the Web UI. Please include the full version including the build identifier (eg. 0.14.0-ea36ds1) validations: required: true + - type: input + attributes: + label: In which browser(s) are you experiencing the issue with? + placeholder: Google Chrome 88.0.4324.150 + description: > + Provide the full name and don't forget to add the version! - type: textarea id: config attributes: @@ -31,10 +47,18 @@ body: validations: required: true - type: textarea - id: logs + id: frigatelogs + attributes: + label: Relevant Frigate log output + description: Please copy and paste any relevant Frigate log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. + render: shell + validations: + required: true + - type: textarea + id: go2rtclogs attributes: - label: Relevant log output - description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + label: Relevant go2rtc log output + description: Please copy and paste any relevant go2rtc log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. render: shell validations: required: true @@ -42,7 +66,7 @@ body: id: ffprobe attributes: label: FFprobe output from your camera - description: Run `ffprobe ` and provide output below + description: Run `ffprobe ` from within the Frigate container if possible, and provide output below render: shell validations: required: true @@ -87,6 +111,11 @@ body: description: Dahua, hikvision, amcrest, reolink, etc and model number validations: required: true + - type: textarea + id: screenshots + attributes: + label: Screenshots of the Frigate UI's System metrics pages and any other screenshots pertinent to your issue. + description: Drag and drop for images is possible in this field. - type: textarea id: other attributes: diff --git a/.github/DISCUSSION_TEMPLATE/question.yml b/.github/DISCUSSION_TEMPLATE/question.yml index 41339e6092..8043ab5a90 100644 --- a/.github/DISCUSSION_TEMPLATE/question.yml +++ b/.github/DISCUSSION_TEMPLATE/question.yml @@ -1,9 +1,21 @@ title: "[Question]: " labels: ["question"] body: + - type: markdown + attributes: + value: | + Use this form for questions you have about Frigate. + + Before submitting your question, please [search the discussions][discussions], read the [official Frigate documentation][docs], and read the [Frigate FAQ][faq] pinned at the Discussion page to see if your question has already been answered by the community. + + **If you are looking for support, start a new discussion and use a support category.** + + [discussions]: https://www.github.com/blakeblackshear/frigate/discussions + [docs]: https://docs.frigate.video + [faq]: https://github.com/blakeblackshear/frigate/discussions/12724 - type: textarea id: description attributes: - label: "What is your question:" + label: "What is your question?" validations: required: true diff --git a/.github/DISCUSSION_TEMPLATE/report-a-bug.yml b/.github/DISCUSSION_TEMPLATE/report-a-bug.yml index c1c34fb51e..472dffb73e 100644 --- a/.github/DISCUSSION_TEMPLATE/report-a-bug.yml +++ b/.github/DISCUSSION_TEMPLATE/report-a-bug.yml @@ -1,25 +1,65 @@ title: "[Bug]: " labels: ["bug", "triage"] body: + - type: markdown + attributes: + value: | + Use this form to submit a reproducible bug in Frigate or Frigate's UI. + + Before submitting your bug report, please [search the discussions][discussions], look at recent open and closed [pull requests][prs], read the [official Frigate documentation][docs], and read the [Frigate FAQ][faq] pinned at the Discussion page to see if your bug has already been fixed by the developers or reported by the community. + + **If you are unsure if your issue is actually a bug or not, please submit a support request first.** + + [discussions]: https://www.github.com/blakeblackshear/frigate/discussions + [prs]: https://www.github.com/blakeblackshear/frigate/pulls + [docs]: https://docs.frigate.video + [faq]: https://github.com/blakeblackshear/frigate/discussions/12724 + - type: checkboxes + attributes: + label: Checklist + description: Please verify that you've followed these steps + options: + - label: I have updated to the latest available Frigate version. + required: true + - label: I have cleared the cache of my browser. + required: true + - label: I have tried a different browser to see if it is related to my browser. + required: true + - label: I have tried reproducing the issue in [incognito mode](https://www.computerworld.com/article/1719851/how-to-go-incognito-in-chrome-firefox-safari-and-edge.html) to rule out problems with any third party extensions or plugins I have installed. - type: textarea id: description attributes: label: Describe the problem you are having + description: Provide a clear and concise description of what the bug is. validations: required: true - type: textarea id: steps attributes: label: Steps to reproduce + description: | + Please tell us exactly how to reproduce your issue. + Provide clear and concise step by step instructions and add code snippets if needed. + value: | + 1. + 2. + 3. + ... validations: required: true - type: input id: version attributes: label: Version - description: Visible on the System page in the Web UI + description: Visible on the System page in the Web UI. Please include the full version including the build identifier (eg. 0.14.0-ea36ds1) validations: required: true + - type: input + attributes: + label: In which browser(s) are you experiencing the issue with? + placeholder: Google Chrome 88.0.4324.150 + description: > + Provide the full name and don't forget to add the version! - type: textarea id: config attributes: @@ -29,10 +69,18 @@ body: validations: required: true - type: textarea - id: logs + id: frigatelogs + attributes: + label: Relevant Frigate log output + description: Please copy and paste any relevant Frigate log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. + render: shell + validations: + required: true + - type: textarea + id: go2rtclogs attributes: - label: Relevant log output - description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + label: Relevant go2rtc log output + description: Please copy and paste any relevant go2rtc log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. render: shell validations: required: true @@ -77,6 +125,11 @@ body: description: Dahua, hikvision, amcrest, reolink, etc and model number validations: required: true + - type: textarea + id: screenshots + attributes: + label: Screenshots of the Frigate UI's System metrics pages and any other screenshots pertinent to your issue. + description: Drag and drop for images is possible in this field. - type: textarea id: other attributes: From 65ca3c8fa3b53913f14d0e45d221e456d19b5335 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 23 Aug 2024 07:58:39 -0500 Subject: [PATCH 043/169] Fix discussion templates (#13292) * Fix yaml spacing for discussion templates * Remove browser question from detectors --- .github/DISCUSSION_TEMPLATE/config-support.yml | 16 ++++++++++++---- .github/DISCUSSION_TEMPLATE/detector-support.yml | 8 +------- .github/DISCUSSION_TEMPLATE/question.yml | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/DISCUSSION_TEMPLATE/config-support.yml b/.github/DISCUSSION_TEMPLATE/config-support.yml index 72b11e634f..954b2d3e3a 100644 --- a/.github/DISCUSSION_TEMPLATE/config-support.yml +++ b/.github/DISCUSSION_TEMPLATE/config-support.yml @@ -1,7 +1,7 @@ title: "[Config Support]: " labels: ["support", "triage"] body: - - type: markdown + - type: markdown attributes: value: | Use this form for support or questions related to Frigate's configuration and config file. @@ -33,10 +33,18 @@ body: validations: required: true - type: textarea - id: logs + id: frigatelogs attributes: - label: Relevant log output - description: Please copy and paste any relevant log output, including any go2rtc and Frigate logs. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. + label: Relevant Frigate log output + description: Please copy and paste any relevant Frigate log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. + render: shell + validations: + required: true + - type: textarea + id: go2rtclogs + attributes: + label: Relevant go2rtc log output + description: Please copy and paste any relevant go2rtc log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. render: shell validations: required: true diff --git a/.github/DISCUSSION_TEMPLATE/detector-support.yml b/.github/DISCUSSION_TEMPLATE/detector-support.yml index 8d1a57ff1e..18f24f4d2c 100644 --- a/.github/DISCUSSION_TEMPLATE/detector-support.yml +++ b/.github/DISCUSSION_TEMPLATE/detector-support.yml @@ -1,7 +1,7 @@ title: "[Detector Support]: " labels: ["support", "triage"] body: - - type: markdown + - type: markdown attributes: value: | Use this form for support or questions related to Frigate's object detectors. @@ -24,12 +24,6 @@ body: description: Visible on the System page in the Web UI. Please include the full version including the build identifier (eg. 0.14.0-ea36ds1) validations: required: true - - type: input - attributes: - label: What browser(s) are you using? - placeholder: Google Chrome 88.0.4324.150 - description: > - Provide the full name and don't forget to add the version! - type: textarea id: config attributes: diff --git a/.github/DISCUSSION_TEMPLATE/question.yml b/.github/DISCUSSION_TEMPLATE/question.yml index 8043ab5a90..6a4789c9c5 100644 --- a/.github/DISCUSSION_TEMPLATE/question.yml +++ b/.github/DISCUSSION_TEMPLATE/question.yml @@ -1,7 +1,7 @@ title: "[Question]: " labels: ["question"] body: - - type: markdown + - type: markdown attributes: value: | Use this form for questions you have about Frigate. From 2dc5a7f7675b279c49eb84d73d7b58ebfef2d990 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 23 Aug 2024 08:51:59 -0600 Subject: [PATCH 044/169] Fix delayed preview not showing (#13295) --- web/src/components/player/PreviewPlayer.tsx | 12 +++++----- web/src/hooks/use-camera-previews.ts | 25 ++++++++++++++------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/web/src/components/player/PreviewPlayer.tsx b/web/src/components/player/PreviewPlayer.tsx index 8c0394abd2..6ff5e1590d 100644 --- a/web/src/components/player/PreviewPlayer.tsx +++ b/web/src/components/player/PreviewPlayer.tsx @@ -16,7 +16,10 @@ import { isAndroid, isChrome, isMobile } from "react-device-detect"; import { TimeRange } from "@/types/timeline"; import { Skeleton } from "../ui/skeleton"; import { cn } from "@/lib/utils"; -import { usePreviewForTimeRange } from "@/hooks/use-camera-previews"; +import { + getPreviewForTimeRange, + usePreviewForTimeRange, +} from "@/hooks/use-camera-previews"; type PreviewPlayerProps = { className?: string; @@ -243,12 +246,7 @@ function PreviewVideoPlayer({ return; } - const preview = cameraPreviews.find( - (preview) => - preview.camera == camera && - Math.round(preview.start) >= timeRange.after && - Math.floor(preview.end) <= timeRange.before, - ); + const preview = getPreviewForTimeRange(cameraPreviews, camera, timeRange); if (preview != currentPreview) { controller.newPreviewLoaded = false; diff --git a/web/src/hooks/use-camera-previews.ts b/web/src/hooks/use-camera-previews.ts index 040b1081be..921fecd709 100644 --- a/web/src/hooks/use-camera-previews.ts +++ b/web/src/hooks/use-camera-previews.ts @@ -38,17 +38,26 @@ export function useCameraPreviews( // it is not falsely thrown out. const PREVIEW_END_BUFFER = 5; // seconds +export function getPreviewForTimeRange( + allPreviews: Preview[], + camera: string, + timeRange: TimeRange, +) { + return allPreviews.find( + (preview) => + preview.camera == camera && + Math.ceil(preview.start) >= timeRange.after && + Math.floor(preview.end) <= timeRange.before + PREVIEW_END_BUFFER, + ); +} + export function usePreviewForTimeRange( allPreviews: Preview[], camera: string, timeRange: TimeRange, ) { - return useMemo(() => { - return allPreviews.find( - (preview) => - preview.camera == camera && - Math.ceil(preview.start) >= timeRange.after && - Math.floor(preview.end) <= timeRange.before + PREVIEW_END_BUFFER, - ); - }, [allPreviews, camera, timeRange]); + return useMemo( + () => getPreviewForTimeRange(allPreviews, camera, timeRange), + [allPreviews, camera, timeRange], + ); } From fdb5d5396004ab6c95ffee2c746c4eec34ce2c29 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 23 Aug 2024 18:05:14 -0500 Subject: [PATCH 045/169] Update discussion templates (#13303) * Update discussion templates * camera support go2rtc --- .github/DISCUSSION_TEMPLATE/camera-support.yml | 8 +++++--- .github/DISCUSSION_TEMPLATE/config-support.yml | 3 ++- .github/DISCUSSION_TEMPLATE/detector-support.yml | 8 +++++--- .github/DISCUSSION_TEMPLATE/general-support.yml | 6 +++--- .../hardware-acceleration-support.yml | 8 +++++--- .github/DISCUSSION_TEMPLATE/report-a-bug.yml | 16 +++++++++++++--- 6 files changed, 33 insertions(+), 16 deletions(-) diff --git a/.github/DISCUSSION_TEMPLATE/camera-support.yml b/.github/DISCUSSION_TEMPLATE/camera-support.yml index ec64973092..bbbffea500 100644 --- a/.github/DISCUSSION_TEMPLATE/camera-support.yml +++ b/.github/DISCUSSION_TEMPLATE/camera-support.yml @@ -50,7 +50,7 @@ body: id: go2rtclogs attributes: label: Relevant go2rtc log output - description: Please copy and paste any relevant go2rtc log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. + description: Please copy and paste any relevant go2rtc log output. Include logs before and after your exact error when possible. Logs can be viewed via the Frigate UI, Docker, or the go2rtc dashboard. This will be automatically formatted into code, so no need for backticks. render: shell validations: required: true @@ -125,8 +125,10 @@ body: - type: textarea id: screenshots attributes: - label: Screenshots of the Frigate UI's System metrics pages and any other screenshots pertinent to your issue. - description: Drag and drop for images is possible in this field. + label: Screenshots of the Frigate UI's System metrics pages + description: Drag and drop for images is possible in this field. Please post screenshots of at least General and Cameras tabs. + validations: + required: true - type: textarea id: other attributes: diff --git a/.github/DISCUSSION_TEMPLATE/config-support.yml b/.github/DISCUSSION_TEMPLATE/config-support.yml index 954b2d3e3a..5b70b1d914 100644 --- a/.github/DISCUSSION_TEMPLATE/config-support.yml +++ b/.github/DISCUSSION_TEMPLATE/config-support.yml @@ -94,7 +94,8 @@ body: - type: textarea id: screenshots attributes: - label: Screenshots of the Frigate UI's System metrics pages and any other screenshots pertinent to your issue. Drag and drop or simple cut/paste is possible in this field. + label: Screenshots of the Frigate UI's System metrics pages + description: Drag and drop or simple cut/paste is possible in this field - type: textarea id: other attributes: diff --git a/.github/DISCUSSION_TEMPLATE/detector-support.yml b/.github/DISCUSSION_TEMPLATE/detector-support.yml index 18f24f4d2c..e4ae976a3e 100644 --- a/.github/DISCUSSION_TEMPLATE/detector-support.yml +++ b/.github/DISCUSSION_TEMPLATE/detector-support.yml @@ -52,7 +52,7 @@ body: id: go2rtclogs attributes: label: Relevant go2rtc log output - description: Please copy and paste any relevant go2rtc log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. + description: Please copy and paste any relevant go2rtc log output. Include logs before and after your exact error when possible. Logs can be viewed via the Frigate UI, Docker, or the go2rtc dashboard. This will be automatically formatted into code, so no need for backticks. render: shell validations: required: true @@ -96,8 +96,10 @@ body: - type: textarea id: screenshots attributes: - label: Screenshots of the Frigate UI's System metrics pages and any other screenshots pertinent to your issue. - description: Drag and drop for images is possible in this field. + label: Screenshots of the Frigate UI's System metrics pages + description: Drag and drop for images is possible in this field. Please post screenshots of at least General and Cameras tabs. + validations: + required: true - type: textarea id: other attributes: diff --git a/.github/DISCUSSION_TEMPLATE/general-support.yml b/.github/DISCUSSION_TEMPLATE/general-support.yml index b3592797cf..0ae7d7083b 100644 --- a/.github/DISCUSSION_TEMPLATE/general-support.yml +++ b/.github/DISCUSSION_TEMPLATE/general-support.yml @@ -50,7 +50,7 @@ body: id: go2rtclogs attributes: label: Relevant go2rtc log output - description: Please copy and paste any relevant go2rtc log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. + description: Please copy and paste any relevant go2rtc log output. Include logs before and after your exact error when possible. Logs can be viewed via the Frigate UI, Docker, or the go2rtc dashboard. This will be automatically formatted into code, so no need for backticks. render: shell validations: required: true @@ -125,8 +125,8 @@ body: - type: textarea id: screenshots attributes: - label: Screenshots of the Frigate UI's System metrics pages and any other screenshots pertinent to your issue. - description: Drag and drop for images is possible in this field. + label: Screenshots of the Frigate UI's System metrics pages + description: Drag and drop for images is possible in this field - type: textarea id: other attributes: diff --git a/.github/DISCUSSION_TEMPLATE/hardware-acceleration-support.yml b/.github/DISCUSSION_TEMPLATE/hardware-acceleration-support.yml index e50b21b9b3..1b7094fd7b 100644 --- a/.github/DISCUSSION_TEMPLATE/hardware-acceleration-support.yml +++ b/.github/DISCUSSION_TEMPLATE/hardware-acceleration-support.yml @@ -58,7 +58,7 @@ body: id: go2rtclogs attributes: label: Relevant go2rtc log output - description: Please copy and paste any relevant go2rtc log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. + description: Please copy and paste any relevant go2rtc log output. Include logs before and after your exact error when possible. Logs can be viewed via the Frigate UI, Docker, or the go2rtc dashboard. This will be automatically formatted into code, so no need for backticks. render: shell validations: required: true @@ -114,8 +114,10 @@ body: - type: textarea id: screenshots attributes: - label: Screenshots of the Frigate UI's System metrics pages and any other screenshots pertinent to your issue. - description: Drag and drop for images is possible in this field. + label: Screenshots of the Frigate UI's System metrics pages + description: Drag and drop for images is possible in this field. Please post screenshots of at least General and Cameras tabs. + validations: + required: true - type: textarea id: other attributes: diff --git a/.github/DISCUSSION_TEMPLATE/report-a-bug.yml b/.github/DISCUSSION_TEMPLATE/report-a-bug.yml index 472dffb73e..dba6d695e3 100644 --- a/.github/DISCUSSION_TEMPLATE/report-a-bug.yml +++ b/.github/DISCUSSION_TEMPLATE/report-a-bug.yml @@ -68,6 +68,14 @@ body: render: yaml validations: required: true + - type: textarea + id: docker + attributes: + label: docker-compose file or Docker CLI command + description: This will be automatically formatted into code, so no need for backticks. + render: yaml + validations: + required: true - type: textarea id: frigatelogs attributes: @@ -80,7 +88,7 @@ body: id: go2rtclogs attributes: label: Relevant go2rtc log output - description: Please copy and paste any relevant go2rtc log output. Include logs before and after your exact error when possible. This will be automatically formatted into code, so no need for backticks. + description: Please copy and paste any relevant go2rtc log output. Include logs before and after your exact error when possible. Logs can be viewed via the Frigate UI, Docker, or the go2rtc dashboard. This will be automatically formatted into code, so no need for backticks. render: shell validations: required: true @@ -128,8 +136,10 @@ body: - type: textarea id: screenshots attributes: - label: Screenshots of the Frigate UI's System metrics pages and any other screenshots pertinent to your issue. - description: Drag and drop for images is possible in this field. + label: Screenshots of the Frigate UI's System metrics pages + description: Drag and drop for images is possible in this field. Please post screenshots of all tabs. + validations: + required: true - type: textarea id: other attributes: From bf90daae2bd90e693dcc2f6c01feee5ad6cc9536 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sat, 24 Aug 2024 07:25:24 -0500 Subject: [PATCH 046/169] update actions for release (#13318) --- .github/actions/setup/action.yml | 2 +- .github/workflows/release.yml | 8 ++++---- .github/workflows/stale.yml | 26 +++++++++++++------------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 88ceab9355..793ea7d42e 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -5,7 +5,7 @@ inputs: required: true outputs: image-name: - value: ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ github.ref_name }}-${{ steps.create-short-sha.outputs.SHORT_SHA }} + value: ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ steps.create-short-sha.outputs.SHORT_SHA }} cache-name: value: ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:cache runs: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b51381956a..97d22202e1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,10 +23,10 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Create tag variables run: | - BRANCH=dev - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV + BUILD_TYPE=$([[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && echo "stable" || echo "beta") + echo "BUILD_TYPE=${BUILD_TYPE}" >> $GITHUB_ENV echo "BASE=ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}" >> $GITHUB_ENV - echo "BUILD_TAG=${BRANCH}-${GITHUB_SHA::7}" >> $GITHUB_ENV + echo "BUILD_TAG=${GITHUB_SHA::7}" >> $GITHUB_ENV echo "CLEAN_VERSION=$(echo ${GITHUB_REF##*/} | tr '[:upper:]' '[:lower:]' | sed 's/^[v]//')" >> $GITHUB_ENV - name: Tag and push the main image run: | @@ -39,7 +39,7 @@ jobs: done # stable tag - if [[ "${BRANCH}" == "master" ]]; then + if [[ "${BUILD_TYPE}" == "stable" ]]; then docker run --rm -v $HOME/.docker/config.json:/config.json quay.io/skopeo/stable:latest copy --authfile /config.json --multi-arch all docker://${PULL_TAG} docker://${STABLE_TAG} for variant in standard-arm64 tensorrt tensorrt-jp4 tensorrt-jp5 rk; do docker run --rm -v $HOME/.docker/config.json:/config.json quay.io/skopeo/stable:latest copy --authfile /config.json --multi-arch all docker://${PULL_TAG}-${variant} docker://${STABLE_TAG}-${variant} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 172eaeca6f..8e7e3223cb 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -25,17 +25,17 @@ jobs: - name: Print outputs run: echo ${{ join(steps.stale.outputs.*, ',') }} - clean_ghcr: - name: Delete outdated dev container images - runs-on: ubuntu-latest - steps: - - name: Delete old images - uses: snok/container-retention-policy@v2 - with: - image-names: dev-* - cut-off: 60 days ago UTC - keep-at-least: 5 - account-type: personal - token: ${{ secrets.GITHUB_TOKEN }} - token-type: github-token + # clean_ghcr: + # name: Delete outdated dev container images + # runs-on: ubuntu-latest + # steps: + # - name: Delete old images + # uses: snok/container-retention-policy@v2 + # with: + # image-names: dev-* + # cut-off: 60 days ago UTC + # keep-at-least: 5 + # account-type: personal + # token: ${{ secrets.GITHUB_TOKEN }} + # token-type: github-token From ce79898caedca74d82bce2cfd0a51569c1e38b23 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sat, 24 Aug 2024 07:44:15 -0500 Subject: [PATCH 047/169] fix default build (#13321) --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 860c8b4e44..5e360b5ff3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -229,7 +229,7 @@ jobs: run: echo "SHORT_SHA=${GITHUB_SHA::7}" >> $GITHUB_ENV - uses: int128/docker-manifest-create-action@v2 with: - tags: ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ github.ref_name }}-${{ env.SHORT_SHA }} + tags: ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ env.SHORT_SHA }} sources: | - ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ github.ref_name }}-${{ env.SHORT_SHA }}-amd64 - ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ github.ref_name }}-${{ env.SHORT_SHA }}-rpi + ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ env.SHORT_SHA }}-amd64 + ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ env.SHORT_SHA }}-rpi From 453a8d794e31ccc428a73c6ac590d56af871f4d8 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Sun, 25 Aug 2024 06:57:10 -0600 Subject: [PATCH 048/169] Add tooltip for icons in review event list (#13334) --- web/src/components/card/ReviewCard.tsx | 54 +++++++++++++++++++------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/web/src/components/card/ReviewCard.tsx b/web/src/components/card/ReviewCard.tsx index 33032d2b81..359dd6536d 100644 --- a/web/src/components/card/ReviewCard.tsx +++ b/web/src/components/card/ReviewCard.tsx @@ -32,6 +32,8 @@ import { Drawer, DrawerContent } from "../ui/drawer"; import axios from "axios"; import { toast } from "sonner"; import useKeyboardListener from "@/hooks/use-keyboard-listener"; +import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"; +import { capitalizeFirstLetter } from "@/utils/stringUtil"; type ReviewCardProps = { event: ReviewSegment; @@ -153,21 +155,43 @@ export default function ReviewCard({ }} />
-
- {event.data.objects.map((object) => { - return getIconForLabel( - object, - "size-3 text-primary dark:text-white", - ); - })} - {event.data.audio.map((audio) => { - return getIconForLabel( - audio, - "size-3 text-primary dark:text-white", - ); - })} -
{formattedDate}
-
+ + +
+ <> + {event.data.objects.map((object) => { + return getIconForLabel( + object, + "size-3 text-primary dark:text-white", + ); + })} + {event.data.audio.map((audio) => { + return getIconForLabel( + audio, + "size-3 text-primary dark:text-white", + ); + })} + +
{formattedDate}
+
+ + + {[ + ...new Set([ + ...(event.data.objects || []), + ...(event.data.sub_labels || []), + ...(event.data.audio || []), + ]), + ] + .filter( + (item) => item !== undefined && !item.includes("-verified"), + ) + .map((text) => capitalizeFirstLetter(text)) + .sort() + .join(", ") + .replaceAll("-verified", "")} + + Date: Mon, 26 Aug 2024 23:17:24 +0200 Subject: [PATCH 049/169] update go2rtc version in reference config (#13367) --- docs/docs/configuration/reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md index a90a3241e2..e5673157ac 100644 --- a/docs/docs/configuration/reference.md +++ b/docs/docs/configuration/reference.md @@ -466,7 +466,7 @@ snapshots: quality: 70 # Optional: Restream configuration -# Uses https://github.com/AlexxIT/go2rtc (v1.8.3) +# Uses https://github.com/AlexxIT/go2rtc (v1.9.2) go2rtc: # Optional: jsmpeg stream configuration for WebUI From ca0f6e4c0af7c1a1cc405ffe33a1a18cd3c8ff84 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Tue, 27 Aug 2024 20:14:22 -0500 Subject: [PATCH 050/169] Add portal the live player tooltip (#13389) --- web/src/components/player/LivePlayer.tsx | 31 +++++++++++++----------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/web/src/components/player/LivePlayer.tsx b/web/src/components/player/LivePlayer.tsx index 9a0b6f3db6..79de3b5a22 100644 --- a/web/src/components/player/LivePlayer.tsx +++ b/web/src/components/player/LivePlayer.tsx @@ -18,6 +18,7 @@ import Chip from "../indicators/Chip"; import { capitalizeFirstLetter } from "@/utils/stringUtil"; import { cn } from "@/lib/utils"; import { TbExclamationCircle } from "react-icons/tb"; +import { TooltipPortal } from "@radix-ui/react-tooltip"; type LivePlayerProps = { cameraRef?: (ref: HTMLDivElement | null) => void; @@ -258,20 +259,22 @@ export default function LivePlayer({
- - {[ - ...new Set([ - ...(objects || []).map(({ label, sub_label }) => - label.endsWith("verified") ? sub_label : label, - ), - ]), - ] - .filter((label) => label?.includes("-verified") == false) - .map((label) => capitalizeFirstLetter(label)) - .sort() - .join(", ") - .replaceAll("-verified", "")} - + + + {[ + ...new Set([ + ...(objects || []).map(({ label, sub_label }) => + label.endsWith("verified") ? sub_label : label, + ), + ]), + ] + .filter((label) => label?.includes("-verified") == false) + .map((label) => capitalizeFirstLetter(label)) + .sort() + .join(", ") + .replaceAll("-verified", "")} + +
)} From f4f3cfa91152952c1a96a9bc721fa3e1b270a9b8 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Wed, 28 Aug 2024 07:26:50 -0500 Subject: [PATCH 051/169] Don't allow periods in zone or camera group names (#13400) --- web/src/components/filter/CameraGroupSelector.tsx | 8 ++++++++ web/src/components/settings/ZoneEditPane.tsx | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/web/src/components/filter/CameraGroupSelector.tsx b/web/src/components/filter/CameraGroupSelector.tsx index f6a5c5fdf7..6f14b8eb96 100644 --- a/web/src/components/filter/CameraGroupSelector.tsx +++ b/web/src/components/filter/CameraGroupSelector.tsx @@ -551,6 +551,14 @@ export function CameraGroupEdit({ message: "Camera group name already exists.", }, ) + .refine( + (value: string) => { + return !value.includes("."); + }, + { + message: "Camera group name must not contain a period.", + }, + ) .refine((value: string) => value.toLowerCase() !== "default", { message: "Invalid camera group name.", }), diff --git a/web/src/components/settings/ZoneEditPane.tsx b/web/src/components/settings/ZoneEditPane.tsx index b80fa933e3..f1c23c7059 100644 --- a/web/src/components/settings/ZoneEditPane.tsx +++ b/web/src/components/settings/ZoneEditPane.tsx @@ -106,6 +106,14 @@ export default function ZoneEditPane({ { message: "Zone name already exists on this camera.", }, + ) + .refine( + (value: string) => { + return !value.includes("."); + }, + { + message: "Zone name must not contain a period.", + }, ), inertia: z.coerce .number() From 36cbffcc5e6c29ad5e2ddb0f83696777a71f6854 Mon Sep 17 00:00:00 2001 From: Jason Hunter Date: Fri, 21 Jun 2024 17:30:19 -0400 Subject: [PATCH 052/169] Semantic Search for Detections (#11899) * Initial re-implementation of semantic search * put docker-compose back and make reindex match docs * remove debug code and fix import * fix docs * manually build pysqlite3 as binaries are only available for x86-64 * update comment in build_pysqlite3.sh * only embed objects * better error handling when genai fails * ask ollama to pull requested model at startup * update ollama docs * address some PR review comments * fix lint * use IPC to write description, update docs for reindex * remove gemini-pro-vision from docs as it will be unavailable soon * fix OpenAI doc available models * fix api error in gemini and metadata for embeddings --- docker/main/Dockerfile | 13 ++ docker/main/build_pysqlite3.sh | 35 ++++ docker/main/requirements-wheels.txt | 7 + .../s6-rc.d/chroma-log/consumer-for | 1 + .../chroma-log/dependencies.d/log-prepare | 0 .../s6-rc.d/chroma-log/pipeline-name | 1 + .../etc/s6-overlay/s6-rc.d/chroma-log/run | 4 + .../etc/s6-overlay/s6-rc.d/chroma-log/type | 1 + .../s6-rc.d/chroma/dependencies.d/base | 0 .../etc/s6-overlay/s6-rc.d/chroma/finish | 28 +++ .../s6-overlay/s6-rc.d/chroma/producer-for | 1 + .../rootfs/etc/s6-overlay/s6-rc.d/chroma/run | 16 ++ .../s6-overlay/s6-rc.d/chroma/timeout-kill | 1 + .../rootfs/etc/s6-overlay/s6-rc.d/chroma/type | 1 + .../s6-rc.d/frigate/dependencies.d/chroma | 0 .../etc/s6-overlay/s6-rc.d/log-prepare/run | 2 +- docker/main/rootfs/usr/local/chroma | 14 ++ docs/docs/configuration/genai.md | 135 ++++++++++++ docs/docs/configuration/index.md | 5 + docs/docs/configuration/reference.md | 29 +++ docs/docs/configuration/semantic_search.md | 38 ++++ docs/sidebars.js | 4 + frigate/__main__.py | 7 +- frigate/api/app.py | 1 + frigate/app.py | 25 ++- frigate/comms/detections_updater.py | 101 ++------- frigate/comms/dispatcher.py | 7 +- frigate/comms/events_updater.py | 87 ++------ frigate/comms/zmq_proxy.py | 100 +++++++++ frigate/config.py | 46 ++++ frigate/const.py | 1 + frigate/embeddings/__init__.py | 67 ++++++ frigate/embeddings/embeddings.py | 122 +++++++++++ frigate/embeddings/functions/clip.py | 63 ++++++ frigate/embeddings/functions/minilm_l6_v2.py | 11 + frigate/embeddings/maintainer.py | 197 ++++++++++++++++++ frigate/events/audio.py | 2 +- frigate/events/cleanup.py | 19 +- frigate/events/external.py | 4 +- frigate/events/maintainer.py | 2 +- frigate/genai/__init__.py | 63 ++++++ frigate/genai/gemini.py | 49 +++++ frigate/genai/ollama.py | 41 ++++ frigate/genai/openai.py | 51 +++++ frigate/object_processing.py | 4 +- frigate/output/output.py | 4 +- frigate/record/maintainer.py | 2 +- frigate/review/maintainer.py | 2 +- 48 files changed, 1246 insertions(+), 168 deletions(-) create mode 100755 docker/main/build_pysqlite3.sh create mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/consumer-for create mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/dependencies.d/log-prepare create mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/pipeline-name create mode 100755 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/run create mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/type create mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/dependencies.d/base create mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/finish create mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/producer-for create mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/run create mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/timeout-kill create mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/type create mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/frigate/dependencies.d/chroma create mode 100755 docker/main/rootfs/usr/local/chroma create mode 100644 docs/docs/configuration/genai.md create mode 100644 docs/docs/configuration/semantic_search.md create mode 100644 frigate/comms/zmq_proxy.py create mode 100644 frigate/embeddings/__init__.py create mode 100644 frigate/embeddings/embeddings.py create mode 100644 frigate/embeddings/functions/clip.py create mode 100644 frigate/embeddings/functions/minilm_l6_v2.py create mode 100644 frigate/embeddings/maintainer.py create mode 100644 frigate/genai/__init__.py create mode 100644 frigate/genai/gemini.py create mode 100644 frigate/genai/ollama.py create mode 100644 frigate/genai/openai.py diff --git a/docker/main/Dockerfile b/docker/main/Dockerfile index ca14306f9b..5ae0418458 100644 --- a/docker/main/Dockerfile +++ b/docker/main/Dockerfile @@ -148,6 +148,8 @@ RUN apt-get -qq update \ gfortran openexr libatlas-base-dev libssl-dev\ libtbb2 libtbb-dev libdc1394-22-dev libopenexr-dev \ libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev \ + # sqlite3 dependencies + tclsh \ # scipy dependencies gcc gfortran libopenblas-dev liblapack-dev && \ rm -rf /var/lib/apt/lists/* @@ -161,6 +163,10 @@ RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \ COPY docker/main/requirements.txt /requirements.txt RUN pip3 install -r /requirements.txt +# Build pysqlite3 from source to support ChromaDB +COPY docker/main/build_pysqlite3.sh /build_pysqlite3.sh +RUN /build_pysqlite3.sh + COPY docker/main/requirements-wheels.txt /requirements-wheels.txt RUN pip3 wheel --wheel-dir=/wheels -r /requirements-wheels.txt @@ -188,6 +194,13 @@ ARG APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn ENV NVIDIA_VISIBLE_DEVICES=all ENV NVIDIA_DRIVER_CAPABILITIES="compute,video,utility" +# Turn off Chroma Telemetry: https://docs.trychroma.com/telemetry#opting-out +ENV ANONYMIZED_TELEMETRY=False +# Allow resetting the chroma database +ENV ALLOW_RESET=True +# Disable tokenizer parallelism warning +ENV TOKENIZERS_PARALLELISM=true + ENV PATH="/usr/lib/btbn-ffmpeg/bin:/usr/local/go2rtc/bin:/usr/local/tempio/bin:/usr/local/nginx/sbin:${PATH}" # Install dependencies diff --git a/docker/main/build_pysqlite3.sh b/docker/main/build_pysqlite3.sh new file mode 100755 index 0000000000..6375b33fa2 --- /dev/null +++ b/docker/main/build_pysqlite3.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -euxo pipefail + +SQLITE3_VERSION="96c92aba00c8375bc32fafcdf12429c58bd8aabfcadab6683e35bbb9cdebf19e" # 3.46.0 +PYSQLITE3_VERSION="0.5.3" + +# Fetch the source code for the latest release of Sqlite. +if [[ ! -d "sqlite" ]]; then + wget https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=${SQLITE3_VERSION} -O sqlite.tar.gz + tar xzf sqlite.tar.gz + cd sqlite/ + LIBS="-lm" ./configure --disable-tcl --enable-tempstore=always + make sqlite3.c + cd ../ + rm sqlite.tar.gz +fi + +# Grab the pysqlite3 source code. +if [[ ! -d "./pysqlite3" ]]; then + git clone https://github.com/coleifer/pysqlite3.git +fi + +cd pysqlite3/ +git checkout ${PYSQLITE3_VERSION} + +# Copy the sqlite3 source amalgamation into the pysqlite3 directory so we can +# create a self-contained extension module. +cp "../sqlite/sqlite3.c" ./ +cp "../sqlite/sqlite3.h" ./ + +# Create the wheel and put it in the /wheels dir. +sed -i "s|name='pysqlite3-binary'|name=PACKAGE_NAME|g" setup.py +python3 setup.py build_static +pip3 wheel . -w /wheels diff --git a/docker/main/requirements-wheels.txt b/docker/main/requirements-wheels.txt index 84c5a867c1..4b4e13850e 100644 --- a/docker/main/requirements-wheels.txt +++ b/docker/main/requirements-wheels.txt @@ -30,3 +30,10 @@ ws4py == 0.5.* unidecode == 1.3.* onnxruntime == 1.18.* openvino == 2024.1.* +# Embeddings +onnx_clip == 4.0.* +chromadb == 0.5.0 +# Generative AI +google-generativeai == 0.6.* +ollama == 0.2.* +openai == 1.30.* \ No newline at end of file diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/consumer-for b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/consumer-for new file mode 100644 index 0000000000..4b935d3cb5 --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/consumer-for @@ -0,0 +1 @@ +chroma \ No newline at end of file diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/dependencies.d/log-prepare b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/dependencies.d/log-prepare new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/pipeline-name b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/pipeline-name new file mode 100644 index 0000000000..71256e9ed9 --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/pipeline-name @@ -0,0 +1 @@ +chroma-pipeline \ No newline at end of file diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/run b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/run new file mode 100755 index 0000000000..2e47fd3ebe --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/run @@ -0,0 +1,4 @@ +#!/command/with-contenv bash +# shellcheck shell=bash + +exec logutil-service /dev/shm/logs/chroma diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/type b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/type new file mode 100644 index 0000000000..5883cff0cd --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/type @@ -0,0 +1 @@ +longrun diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/dependencies.d/base b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/dependencies.d/base new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/finish b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/finish new file mode 100644 index 0000000000..b6206b4ccf --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/finish @@ -0,0 +1,28 @@ +#!/command/with-contenv bash +# shellcheck shell=bash +# Take down the S6 supervision tree when the service exits + +set -o errexit -o nounset -o pipefail + +# Logs should be sent to stdout so that s6 can collect them + +declare exit_code_container +exit_code_container=$(cat /run/s6-linux-init-container-results/exitcode) +readonly exit_code_container +readonly exit_code_service="${1}" +readonly exit_code_signal="${2}" +readonly service="ChromaDB" + +echo "[INFO] Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})" + +if [[ "${exit_code_service}" -eq 256 ]]; then + if [[ "${exit_code_container}" -eq 0 ]]; then + echo $((128 + exit_code_signal)) >/run/s6-linux-init-container-results/exitcode + fi +elif [[ "${exit_code_service}" -ne 0 ]]; then + if [[ "${exit_code_container}" -eq 0 ]]; then + echo "${exit_code_service}" >/run/s6-linux-init-container-results/exitcode + fi +fi + +exec /run/s6/basedir/bin/halt diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/producer-for b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/producer-for new file mode 100644 index 0000000000..c17b71e87a --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/producer-for @@ -0,0 +1 @@ +chroma-log diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/run b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/run new file mode 100644 index 0000000000..bf28a56b4d --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/run @@ -0,0 +1,16 @@ +#!/command/with-contenv bash +# shellcheck shell=bash +# Start the Frigate service + +set -o errexit -o nounset -o pipefail + +# Logs should be sent to stdout so that s6 can collect them + +# Tell S6-Overlay not to restart this service +s6-svc -O . + +echo "[INFO] Starting ChromaDB..." + +# Replace the bash process with the Frigate process, redirecting stderr to stdout +exec 2>&1 +exec /usr/local/chroma run --path /config/chroma --host 0.0.0.0 diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/timeout-kill b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/timeout-kill new file mode 100644 index 0000000000..6f4f418441 --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/timeout-kill @@ -0,0 +1 @@ +120000 diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/type b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/type new file mode 100644 index 0000000000..5883cff0cd --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/type @@ -0,0 +1 @@ +longrun diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/frigate/dependencies.d/chroma b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/frigate/dependencies.d/chroma new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run index c493e320ee..0661f01c2f 100755 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run @@ -4,7 +4,7 @@ set -o errexit -o nounset -o pipefail -dirs=(/dev/shm/logs/frigate /dev/shm/logs/go2rtc /dev/shm/logs/nginx /dev/shm/logs/certsync) +dirs=(/dev/shm/logs/frigate /dev/shm/logs/go2rtc /dev/shm/logs/nginx /dev/shm/logs/certsync /dev/shm/logs/chroma) mkdir -p "${dirs[@]}" chown nobody:nogroup "${dirs[@]}" diff --git a/docker/main/rootfs/usr/local/chroma b/docker/main/rootfs/usr/local/chroma new file mode 100755 index 0000000000..5147db3877 --- /dev/null +++ b/docker/main/rootfs/usr/local/chroma @@ -0,0 +1,14 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*-s +__import__("pysqlite3") + +import re +import sys + +sys.modules["sqlite3"] = sys.modules.pop("pysqlite3") + +from chromadb.cli.cli import app + +if __name__ == "__main__": + sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0]) + sys.exit(app()) diff --git a/docs/docs/configuration/genai.md b/docs/docs/configuration/genai.md new file mode 100644 index 0000000000..ddd2e9e2ae --- /dev/null +++ b/docs/docs/configuration/genai.md @@ -0,0 +1,135 @@ +--- +id: genai +title: Generative AI +--- + +Generative AI can be used to automatically generate descriptions based on the thumbnails of your events. This helps with [semantic search](/configuration/semantic_search) in Frigate by providing detailed text descriptions as a basis of the search query. + +## Configuration + +Generative AI can be enabled for all cameras or only for specific cameras. There are currently 3 providers available to integrate with Frigate. + +If the provider you choose requires an API key, you may either directly paste it in your configuration, or store it in an environment variable prefixed with `FRIGATE_`. + +```yaml +genai: + enabled: True + provider: gemini + api_key: "{FRIGATE_GEMINI_API_KEY}" + model: gemini-1.5-flash + +cameras: + front_camera: ... + indoor_camera: + genai: # <- disable GenAI for your indoor camera + enabled: False +``` + +## Ollama + +[Ollama](https://ollama.com/) allows you to self-host large language models and keep everything running locally. It provides a nice API over [llama.cpp](https://github.com/ggerganov/llama.cpp). It is highly recommended to host this server on a machine with an Nvidia graphics card, or on a Apple silicon Mac for best performance. Most of the 7b parameter 4-bit vision models will fit inside 8GB of VRAM. There is also a [docker container](https://hub.docker.com/r/ollama/ollama) available. + +### Supported Models + +You must use a vision capable model with Frigate. Current model variants can be found [in their model library](https://ollama.com/library). At the time of writing, this includes `llava`, `llava-llama3`, `llava-phi3`, and `moondream`. + +:::note + +You should have at least 8 GB of RAM available (or VRAM if running on GPU) to run the 7B models, 16 GB to run the 13B models, and 32 GB to run the 33B models. + +::: + +### Configuration + +```yaml +genai: + enabled: True + provider: ollama + base_url: http://localhost:11434 + model: llava +``` + +## Google Gemini + +Google Gemini has a free tier allowing [15 queries per minute](https://ai.google.dev/pricing) to the API, which is more than sufficient for standard Frigate usage. + +### Supported Models + +You must use a vision capable model with Frigate. Current model variants can be found [in their documentation](https://ai.google.dev/gemini-api/docs/models/gemini). At the time of writing, this includes `gemini-1.5-pro` and `gemini-1.5-flash`. + +### Get API Key + +To start using Gemini, you must first get an API key from [Google AI Studio](https://aistudio.google.com). + +1. Accept the Terms of Service +2. Click "Get API Key" from the right hand navigation +3. Click "Create API key in new project" +4. Copy the API key for use in your config + +### Configuration + +```yaml +genai: + enabled: True + provider: gemini + api_key: "{FRIGATE_GEMINI_API_KEY}" + model: gemini-1.5-flash +``` + +## OpenAI + +OpenAI does not have a free tier for their API. With the release of gpt-4o, pricing has been reduced and each generation should cost fractions of a cent if you choose to go this route. + +### Supported Models + +You must use a vision capable model with Frigate. Current model variants can be found [in their documentation](https://platform.openai.com/docs/models). At the time of writing, this includes `gpt-4o` and `gpt-4-turbo`. + +### Get API Key + +To start using OpenAI, you must first [create an API key](https://platform.openai.com/api-keys) and [configure billing](https://platform.openai.com/settings/organization/billing/overview). + +### Configuration + +```yaml +genai: + enabled: True + provider: openai + api_key: "{FRIGATE_OPENAI_API_KEY}" + model: gpt-4o +``` + +## Custom Prompts + +Frigate sends multiple frames from the detection along with a prompt to your Generative AI provider asking it to generate a description. The default prompt is as follows: + +``` +Describe the {label} in the sequence of images with as much detail as possible. Do not describe the background. +``` + +:::tip + +Prompts can use variable replacements like `{label}`, `{sub_label}`, and `{camera}` to substitute information from the detection as part of the prompt. + +::: + +You are also able to define custom prompts in your configuration. + +```yaml +genai: + enabled: True + provider: ollama + base_url: http://localhost:11434 + model: llava + prompt: "Describe the {label} in these images from the {camera} security camera." + object_prompts: + person: "Describe the main person in these images (gender, age, clothing, activity, etc). Do not include where the activity is occurring (sidewalk, concrete, driveway, etc). If delivering a package, include the company the package is from." + car: "Label the primary vehicle in these images with just the name of the company if it is a delivery vehicle, or the color make and model." +``` + +### Experiment with prompts + +Providers also has a public facing chat interface for their models. Download a couple different thumbnails or snapshots from Frigate and try new things in the playground to get descriptions to your liking before updating the prompt in Frigate. + +- OpenAI - [ChatGPT](https://chatgpt.com) +- Gemini - [Google AI Studio](https://aistudio.google.com) +- Ollama - [Open WebUI](https://docs.openwebui.com/) diff --git a/docs/docs/configuration/index.md b/docs/docs/configuration/index.md index d1e382e40d..01fe97530c 100644 --- a/docs/docs/configuration/index.md +++ b/docs/docs/configuration/index.md @@ -56,6 +56,11 @@ go2rtc: password: "{FRIGATE_GO2RTC_RTSP_PASSWORD}" ``` +```yaml +genai: + api_key: "{FRIGATE_GENAI_API_KEY}" +``` + ## Common configuration examples Here are some common starter configuration examples. Refer to the [reference config](./reference.md) for detailed information about all the config values. diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md index e5673157ac..eb75ad2e58 100644 --- a/docs/docs/configuration/reference.md +++ b/docs/docs/configuration/reference.md @@ -465,6 +465,35 @@ snapshots: # Optional: quality of the encoded jpeg, 0-100 (default: shown below) quality: 70 +# Optional: Configuration for semantic search capability +semantic_search: + # Optional: Enable semantic search (default: shown below) + enabled: False + # Optional: Re-index embeddings database from historical events (default: shown below) + reindex: False + +# Optional: Configuration for AI generated event descriptions +# NOTE: Semantic Search must be enabled for this to do anything. +# WARNING: Depending on the provider, this will send thumbnails over the internet +# to Google or OpenAI's LLMs to generate descriptions. It can be overridden at +# the camera level (enabled: False) to enhance privacy for indoor cameras. +genai: + # Optional: Enable Google Gemini description generation (default: shown below) + enabled: False + # Required if enabled: Provider must be one of ollama, gemini, or openai + provider: ollama + # Required if provider is ollama. May also be used for an OpenAI API compatible backend with the openai provider. + base_url: http://localhost::11434 + # Required if gemini or openai + api_key: "{FRIGATE_GENAI_API_KEY}" + # Optional: The default prompt for generating descriptions. Can use replacement + # variables like "label", "sub_label", "camera" to make more dynamic. (default: shown below) + prompt: "Describe the {label} in the sequence of images with as much detail as possible. Do not describe the background." + # Optional: Object specific prompts to customize description results + # Format: {label}: {prompt} + object_prompts: + person: "My special person prompt." + # Optional: Restream configuration # Uses https://github.com/AlexxIT/go2rtc (v1.9.2) go2rtc: diff --git a/docs/docs/configuration/semantic_search.md b/docs/docs/configuration/semantic_search.md new file mode 100644 index 0000000000..d8ab61d413 --- /dev/null +++ b/docs/docs/configuration/semantic_search.md @@ -0,0 +1,38 @@ +--- +id: semantic_search +title: Using Semantic Search +--- + +Semantic search works by embedding images and/or text into a vector representation identified by numbers. Frigate has support for two such models which both run locally: [OpenAI CLIP](https://openai.com/research/clip) and [all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2). Embeddings are then saved to a local instance of [ChromaDB](https://trychroma.com). + +## Configuration + +Semantic Search is a global configuration setting. + +```yaml +semantic_search: + enabled: True + reindex: False +``` + +:::tip + +The embeddings database can be re-indexed from the existing detections in your database by adding `reindex: True` to your `semantic_search` configuration. Depending on the number of detections you have, it can take up to 30 minutes to complete and may max out your CPU while indexing. Make sure to set the config back to `False` before restarting Frigate again. + +::: + +### OpenAI CLIP + +This model is able to embed both images and text into the same vector space, which allows `image -> image` and `text -> image` similarity searches. Frigate uses this model on detections to encode the thumbnail image and store it in Chroma. When searching detections via text in the search box, frigate will perform a `text -> image` similarity search against this embedding. When clicking "FIND SIMILAR" next to a detection, Frigate will perform an `image -> image` similarity search to retrieve the closest matching thumbnails. + +### all-MiniLM-L6-v2 + +This is a sentence embedding model that has been fine tuned on over 1 billion sentence pairs. This model is used to embed detection descriptions and perform searches against them. Descriptions can be created and/or modified on the search page when clicking on the info icon next to a detection. See [the Generative AI docs](/configuration/genai.md) for more information on how to automatically generate event descriptions. + +## Usage Tips + +1. Semantic search is used in conjunction with the other filters available on the search page. Use a combination of traditional filtering and semantic search for the best results. +2. The comparison between text and image embedding distances generally means that results matching `description` will appear first, even if a `thumbnail` embedding may be a better match. Play with the "Search Type" filter to help find what you are looking for. +3. Make your search language and tone closely match your descriptions. If you are using thumbnail search, phrase your query as an image caption. +4. Semantic search on thumbnails tends to return better results when matching large subjects that take up most of the frame. Small things like "cat" tend to not work well. +5. Experiment! Find a detection you want to test and start typing keywords to see what works for you. diff --git a/docs/sidebars.js b/docs/sidebars.js index da41564cad..1e1a270464 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -29,6 +29,10 @@ module.exports = { "configuration/object_detectors", "configuration/audio_detectors", ], + "Semantic Search": [ + "configuration/semantic_search", + "configuration/genai", + ], Cameras: [ "configuration/cameras", "configuration/review", diff --git a/frigate/__main__.py b/frigate/__main__.py index 8442069082..7106f02097 100644 --- a/frigate/__main__.py +++ b/frigate/__main__.py @@ -1,9 +1,12 @@ import faulthandler +import sys import threading from flask import cli -from frigate.app import FrigateApp +# Hotsawp the sqlite3 module for Chroma compatibility +__import__("pysqlite3") +sys.modules["sqlite3"] = sys.modules.pop("pysqlite3") faulthandler.enable() @@ -12,6 +15,8 @@ cli.show_server_banner = lambda *x: None if __name__ == "__main__": + from frigate.app import FrigateApp + frigate_app = FrigateApp() frigate_app.start() diff --git a/frigate/api/app.py b/frigate/api/app.py index 139b10d5b6..5fec51c03b 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -454,6 +454,7 @@ def logs(service: str): "frigate": "/dev/shm/logs/frigate/current", "go2rtc": "/dev/shm/logs/go2rtc/current", "nginx": "/dev/shm/logs/nginx/current", + "chroma": "/dev/shm/logs/chroma/current", } service_location = log_locations.get(service) diff --git a/frigate/app.py b/frigate/app.py index 7e845f44ab..840686f0a7 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -22,11 +22,11 @@ from frigate.api.app import create_app from frigate.api.auth import hash_password from frigate.comms.config_updater import ConfigPublisher -from frigate.comms.detections_updater import DetectionProxy from frigate.comms.dispatcher import Communicator, Dispatcher from frigate.comms.inter_process import InterProcessCommunicator from frigate.comms.mqtt import MqttClient from frigate.comms.ws import WebSocketClient +from frigate.comms.zmq_proxy import ZmqProxy from frigate.config import FrigateConfig from frigate.const import ( CACHE_DIR, @@ -37,6 +37,8 @@ MODEL_CACHE_DIR, RECORD_DIR, ) +from frigate.embeddings import manage_embeddings +from frigate.embeddings.embeddings import Embeddings from frigate.events.audio import listen_to_audio from frigate.events.cleanup import EventCleanup from frigate.events.external import ExternalEventProcessor @@ -316,7 +318,21 @@ def init_review_segment_manager(self) -> None: self.review_segment_process = review_segment_process review_segment_process.start() self.processes["review_segment"] = review_segment_process.pid or 0 - logger.info(f"Recording process started: {review_segment_process.pid}") + logger.info(f"Review process started: {review_segment_process.pid}") + + def init_embeddings_manager(self) -> None: + # Create a client for other processes to use + self.embeddings = Embeddings() + embedding_process = mp.Process( + target=manage_embeddings, + name="embeddings_manager", + args=(self.config,), + ) + embedding_process.daemon = True + self.embedding_process = embedding_process + embedding_process.start() + self.processes["embeddings"] = embedding_process.pid or 0 + logger.info(f"Embedding process started: {embedding_process.pid}") def bind_database(self) -> None: """Bind db to the main process.""" @@ -362,7 +378,7 @@ def init_external_event_processor(self) -> None: def init_inter_process_communicator(self) -> None: self.inter_process_communicator = InterProcessCommunicator() self.inter_config_updater = ConfigPublisher() - self.inter_detection_proxy = DetectionProxy() + self.inter_zmq_proxy = ZmqProxy() def init_web_server(self) -> None: self.flask_app = create_app( @@ -678,6 +694,7 @@ def start(self) -> None: self.init_onvif() self.init_recording_manager() self.init_review_segment_manager() + self.init_embeddings_manager() self.init_go2rtc() self.bind_database() self.check_db_data_migrations() @@ -797,7 +814,7 @@ def stop(self) -> None: # Stop Communicators self.inter_process_communicator.stop() self.inter_config_updater.stop() - self.inter_detection_proxy.stop() + self.inter_zmq_proxy.stop() while len(self.detection_shms) > 0: shm = self.detection_shms.pop() diff --git a/frigate/comms/detections_updater.py b/frigate/comms/detections_updater.py index af7b7b65d5..a60bd0699a 100644 --- a/frigate/comms/detections_updater.py +++ b/frigate/comms/detections_updater.py @@ -1,14 +1,9 @@ """Facilitates communication between processes.""" -import threading from enum import Enum from typing import Optional -import zmq - -SOCKET_CONTROL = "inproc://control.detections_updater" -SOCKET_PUB = "ipc:///tmp/cache/detect_pub" -SOCKET_SUB = "ipc:///tmp/cache/detect_sub" +from .zmq_proxy import Publisher, Subscriber class DetectionTypeEnum(str, Enum): @@ -18,85 +13,31 @@ class DetectionTypeEnum(str, Enum): audio = "audio" -class DetectionProxyRunner(threading.Thread): - def __init__(self, context: zmq.Context[zmq.Socket]) -> None: - threading.Thread.__init__(self) - self.name = "detection_proxy" - self.context = context - - def run(self) -> None: - """Run the proxy.""" - control = self.context.socket(zmq.REP) - control.connect(SOCKET_CONTROL) - incoming = self.context.socket(zmq.XSUB) - incoming.bind(SOCKET_PUB) - outgoing = self.context.socket(zmq.XPUB) - outgoing.bind(SOCKET_SUB) - - zmq.proxy_steerable( - incoming, outgoing, None, control - ) # blocking, will unblock terminate message is received - incoming.close() - outgoing.close() - - -class DetectionProxy: - """Proxies video and audio detections.""" - - def __init__(self) -> None: - self.context = zmq.Context() - self.control = self.context.socket(zmq.REQ) - self.control.bind(SOCKET_CONTROL) - self.runner = DetectionProxyRunner(self.context) - self.runner.start() - - def stop(self) -> None: - self.control.send("TERMINATE".encode()) # tell the proxy to stop - self.runner.join() - self.context.destroy() - - -class DetectionPublisher: +class DetectionPublisher(Publisher): """Simplifies receiving video and audio detections.""" - def __init__(self, topic: DetectionTypeEnum) -> None: - self.topic = topic - self.context = zmq.Context() - self.socket = self.context.socket(zmq.PUB) - self.socket.connect(SOCKET_PUB) - - def send_data(self, payload: any) -> None: - """Publish detection.""" - self.socket.send_string(self.topic.value, flags=zmq.SNDMORE) - self.socket.send_json(payload) - - def stop(self) -> None: - self.socket.close() - self.context.destroy() - - -class DetectionSubscriber: - """Simplifies receiving video and audio detections.""" + topic_base = "detection/" def __init__(self, topic: DetectionTypeEnum) -> None: - self.context = zmq.Context() - self.socket = self.context.socket(zmq.SUB) - self.socket.setsockopt_string(zmq.SUBSCRIBE, topic.value) - self.socket.connect(SOCKET_SUB) + topic = topic.value + super().__init__(topic) - def get_data(self, timeout: float = None) -> Optional[tuple[str, any]]: - """Returns detections or None if no update.""" - try: - has_update, _, _ = zmq.select([self.socket], [], [], timeout) - if has_update: - topic = DetectionTypeEnum[self.socket.recv_string(flags=zmq.NOBLOCK)] - return (topic, self.socket.recv_json()) - except zmq.ZMQError: - pass +class DetectionSubscriber(Subscriber): + """Simplifies receiving video and audio detections.""" - return (None, None) + topic_base = "detection/" - def stop(self) -> None: - self.socket.close() - self.context.destroy() + def __init__(self, topic: DetectionTypeEnum) -> None: + topic = topic.value + super().__init__(topic) + + def check_for_update( + self, timeout: float = None + ) -> Optional[tuple[DetectionTypeEnum, any]]: + return super().check_for_update(timeout) + + def _return_object(self, topic: str, payload: any) -> any: + if payload is None: + return (None, None) + return (DetectionTypeEnum[topic[len(self.topic_base) :]], payload) diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py index 26922f284d..8e176657da 100644 --- a/frigate/comms/dispatcher.py +++ b/frigate/comms/dispatcher.py @@ -14,9 +14,10 @@ INSERT_PREVIEW, REQUEST_REGION_GRID, UPDATE_CAMERA_ACTIVITY, + UPDATE_EVENT_DESCRIPTION, UPSERT_REVIEW_SEGMENT, ) -from frigate.models import Previews, Recordings, ReviewSegment +from frigate.models import Event, Previews, Recordings, ReviewSegment from frigate.ptz.onvif import OnvifCommandEnum, OnvifController from frigate.types import PTZMetricsTypes from frigate.util.object import get_camera_regions_grid @@ -128,6 +129,10 @@ def _receive(self, topic: str, payload: str) -> Optional[Any]: ).execute() elif topic == UPDATE_CAMERA_ACTIVITY: self.camera_activity = payload + elif topic == UPDATE_EVENT_DESCRIPTION: + event: Event = Event.get(Event.id == payload["id"]) + event.data["description"] = payload["description"] + event.save() elif topic == "onConnect": camera_status = self.camera_activity.copy() diff --git a/frigate/comms/events_updater.py b/frigate/comms/events_updater.py index dd8caf8a30..7a5772273d 100644 --- a/frigate/comms/events_updater.py +++ b/frigate/comms/events_updater.py @@ -1,100 +1,51 @@ """Facilitates communication between processes.""" -import zmq - from frigate.events.types import EventStateEnum, EventTypeEnum -SOCKET_PUSH_PULL = "ipc:///tmp/cache/events" -SOCKET_PUSH_PULL_END = "ipc:///tmp/cache/events_ended" +from .zmq_proxy import Publisher, Subscriber -class EventUpdatePublisher: +class EventUpdatePublisher(Publisher): """Publishes events (objects, audio, manual).""" + topic_base = "event/" + def __init__(self) -> None: - self.context = zmq.Context() - self.socket = self.context.socket(zmq.PUSH) - self.socket.connect(SOCKET_PUSH_PULL) + super().__init__("update") def publish( self, payload: tuple[EventTypeEnum, EventStateEnum, str, dict[str, any]] ) -> None: - """There is no communication back to the processes.""" - self.socket.send_json(payload) - - def stop(self) -> None: - self.socket.close() - self.context.destroy() + super().publish(payload) -class EventUpdateSubscriber: +class EventUpdateSubscriber(Subscriber): """Receives event updates.""" - def __init__(self) -> None: - self.context = zmq.Context() - self.socket = self.context.socket(zmq.PULL) - self.socket.bind(SOCKET_PUSH_PULL) - - def check_for_update( - self, timeout=1 - ) -> tuple[EventTypeEnum, EventStateEnum, str, dict[str, any]]: - """Returns events or None if no update.""" - try: - has_update, _, _ = zmq.select([self.socket], [], [], timeout) - - if has_update: - return self.socket.recv_json() - except zmq.ZMQError: - pass + topic_base = "event/" - return None - - def stop(self) -> None: - self.socket.close() - self.context.destroy() + def __init__(self) -> None: + super().__init__("update") -class EventEndPublisher: +class EventEndPublisher(Publisher): """Publishes events that have ended.""" + topic_base = "event/" + def __init__(self) -> None: - self.context = zmq.Context() - self.socket = self.context.socket(zmq.PUSH) - self.socket.connect(SOCKET_PUSH_PULL_END) + super().__init__("finalized") def publish( self, payload: tuple[EventTypeEnum, EventStateEnum, str, dict[str, any]] ) -> None: - """There is no communication back to the processes.""" - self.socket.send_json(payload) + super().publish(payload) - def stop(self) -> None: - self.socket.close() - self.context.destroy() - -class EventEndSubscriber: +class EventEndSubscriber(Subscriber): """Receives events that have ended.""" + topic_base = "event/" + def __init__(self) -> None: - self.context = zmq.Context() - self.socket = self.context.socket(zmq.PULL) - self.socket.bind(SOCKET_PUSH_PULL_END) - - def check_for_update( - self, timeout=1 - ) -> tuple[EventTypeEnum, EventStateEnum, str, dict[str, any]]: - """Returns events ended or None if no update.""" - try: - has_update, _, _ = zmq.select([self.socket], [], [], timeout) - - if has_update: - return self.socket.recv_json() - except zmq.ZMQError: - pass - - return None - - def stop(self) -> None: - self.socket.close() - self.context.destroy() + super().__init__("finalized") diff --git a/frigate/comms/zmq_proxy.py b/frigate/comms/zmq_proxy.py new file mode 100644 index 0000000000..b6012966f1 --- /dev/null +++ b/frigate/comms/zmq_proxy.py @@ -0,0 +1,100 @@ +"""Facilitates communication over zmq proxy.""" + +import threading +from typing import Optional + +import zmq + +SOCKET_PUB = "ipc:///tmp/cache/proxy_pub" +SOCKET_SUB = "ipc:///tmp/cache/proxy_sub" + + +class ZmqProxyRunner(threading.Thread): + def __init__(self, context: zmq.Context[zmq.Socket]) -> None: + threading.Thread.__init__(self) + self.name = "detection_proxy" + self.context = context + + def run(self) -> None: + """Run the proxy.""" + incoming = self.context.socket(zmq.XSUB) + incoming.bind(SOCKET_PUB) + outgoing = self.context.socket(zmq.XPUB) + outgoing.bind(SOCKET_SUB) + + # Blocking: This will unblock (via exception) when we destroy the context + # The incoming and outgoing sockets will be closed automatically + # when the context is destroyed as well. + try: + zmq.proxy(incoming, outgoing) + except zmq.ZMQError: + pass + + +class ZmqProxy: + """Proxies video and audio detections.""" + + def __init__(self) -> None: + self.context = zmq.Context() + self.runner = ZmqProxyRunner(self.context) + self.runner.start() + + def stop(self) -> None: + # destroying the context will tell the proxy to stop + self.context.destroy() + self.runner.join() + + +class Publisher: + """Publishes messages.""" + + topic_base: str = "" + + def __init__(self, topic: str = "") -> None: + self.topic = f"{self.topic_base}{topic}" + self.context = zmq.Context() + self.socket = self.context.socket(zmq.PUB) + self.socket.connect(SOCKET_PUB) + + def publish(self, payload: any, sub_topic: str = "") -> None: + """Publish message.""" + self.socket.send_string(f"{self.topic}{sub_topic}", flags=zmq.SNDMORE) + self.socket.send_json(payload) + + def stop(self) -> None: + self.socket.close() + self.context.destroy() + + +class Subscriber: + """Receives messages.""" + + topic_base: str = "" + + def __init__(self, topic: str = "") -> None: + self.topic = f"{self.topic_base}{topic}" + self.context = zmq.Context() + self.socket = self.context.socket(zmq.SUB) + self.socket.setsockopt_string(zmq.SUBSCRIBE, self.topic) + self.socket.connect(SOCKET_SUB) + + def check_for_update(self, timeout: float = 1) -> Optional[tuple[str, any]]: + """Returns message or None if no update.""" + try: + has_update, _, _ = zmq.select([self.socket], [], [], timeout) + + if has_update: + topic = self.socket.recv_string(flags=zmq.NOBLOCK) + payload = self.socket.recv_json() + return self._return_object(topic, payload) + except zmq.ZMQError: + pass + + return self._return_object("", None) + + def stop(self) -> None: + self.socket.close() + self.context.destroy() + + def _return_object(self, topic: str, payload: any) -> any: + return payload diff --git a/frigate/config.py b/frigate/config.py index c62ccd9f1b..20a540919c 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -730,6 +730,38 @@ class ReviewConfig(FrigateBaseModel): ) +class SemanticSearchConfig(FrigateBaseModel): + enabled: bool = Field(default=True, title="Enable semantic search.") + reindex: Optional[bool] = Field( + default=False, title="Reindex all detections on startup." + ) + + +class GenAIProviderEnum(str, Enum): + openai = "openai" + gemini = "gemini" + ollama = "ollama" + + +class GenAIConfig(FrigateBaseModel): + enabled: bool = Field(default=False, title="Enable GenAI.") + provider: GenAIProviderEnum = Field( + default=GenAIProviderEnum.openai, title="GenAI provider." + ) + base_url: Optional[str] = Field(None, title="Provider base url.") + api_key: Optional[str] = Field(None, title="Provider API key.") + model: str = Field(default="gpt-4o", title="GenAI model.") + prompt: str = Field( + default="Describe the {label} in the sequence of images with as much detail as possible. Do not describe the background.", + title="Default caption prompt.", + ) + object_prompts: Dict[str, str] = Field(default={}, title="Object specific prompts.") + + +class GenAICameraConfig(FrigateBaseModel): + enabled: bool = Field(default=False, title="Enable GenAI for camera.") + + class AudioConfig(FrigateBaseModel): enabled: bool = Field(default=False, title="Enable audio events.") max_not_heard: int = Field( @@ -1011,6 +1043,9 @@ class CameraConfig(FrigateBaseModel): review: ReviewConfig = Field( default_factory=ReviewConfig, title="Review configuration." ) + genai: GenAICameraConfig = Field( + default_factory=GenAICameraConfig, title="Generative AI configuration." + ) audio: AudioConfig = Field( default_factory=AudioConfig, title="Audio events configuration." ) @@ -1363,6 +1398,12 @@ class FrigateConfig(FrigateBaseModel): review: ReviewConfig = Field( default_factory=ReviewConfig, title="Review configuration." ) + semantic_search: SemanticSearchConfig = Field( + default_factory=SemanticSearchConfig, title="Semantic search configuration." + ) + genai: GenAIConfig = Field( + default_factory=GenAIConfig, title="Generative AI configuration." + ) audio: AudioConfig = Field( default_factory=AudioConfig, title="Global Audio events configuration." ) @@ -1397,6 +1438,10 @@ def runtime_config(self, plus_api: PlusApi = None) -> FrigateConfig: config.mqtt.user = config.mqtt.user.format(**FRIGATE_ENV_VARS) config.mqtt.password = config.mqtt.password.format(**FRIGATE_ENV_VARS) + # GenAI substitution + if config.genai.api_key: + config.genai.api_key = config.genai.api_key.format(**FRIGATE_ENV_VARS) + # set default min_score for object attributes for attribute in ALL_ATTRIBUTE_LABELS: if not config.objects.filters.get(attribute): @@ -1418,6 +1463,7 @@ def runtime_config(self, plus_api: PlusApi = None) -> FrigateConfig: "live": ..., "objects": ..., "review": ..., + "genai": {"enabled"}, "motion": ..., "detect": ..., "ffmpeg": ..., diff --git a/frigate/const.py b/frigate/const.py index b55187744c..7cdb2b6725 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -81,6 +81,7 @@ UPSERT_REVIEW_SEGMENT = "upsert_review_segment" CLEAR_ONGOING_REVIEW_SEGMENTS = "clear_ongoing_review_segments" UPDATE_CAMERA_ACTIVITY = "update_camera_activity" +UPDATE_EVENT_DESCRIPTION = "update_event_description" # Stats Values diff --git a/frigate/embeddings/__init__.py b/frigate/embeddings/__init__.py new file mode 100644 index 0000000000..41af73c012 --- /dev/null +++ b/frigate/embeddings/__init__.py @@ -0,0 +1,67 @@ +"""ChromaDB embeddings database.""" + +import logging +import multiprocessing as mp +import signal +import sys +import threading +from types import FrameType +from typing import Optional + +from playhouse.sqliteq import SqliteQueueDatabase +from setproctitle import setproctitle + +from frigate.config import FrigateConfig +from frigate.models import Event +from frigate.util.services import listen + +logger = logging.getLogger(__name__) + + +def manage_embeddings(config: FrigateConfig) -> None: + # Only initialize embeddings if semantic search is enabled + if not config.semantic_search.enabled: + return + + stop_event = mp.Event() + + def receiveSignal(signalNumber: int, frame: Optional[FrameType]) -> None: + stop_event.set() + + signal.signal(signal.SIGTERM, receiveSignal) + signal.signal(signal.SIGINT, receiveSignal) + + threading.current_thread().name = "process:embeddings_manager" + setproctitle("frigate.embeddings_manager") + listen() + + # Configure Frigate DB + db = SqliteQueueDatabase( + config.database.path, + pragmas={ + "auto_vacuum": "FULL", # Does not defragment database + "cache_size": -512 * 1000, # 512MB of cache + "synchronous": "NORMAL", # Safe when using WAL https://www.sqlite.org/pragma.html#pragma_synchronous + }, + timeout=max(60, 10 * len([c for c in config.cameras.values() if c.enabled])), + ) + models = [Event] + db.bind(models) + + # Hotsawp the sqlite3 module for Chroma compatibility + __import__("pysqlite3") + sys.modules["sqlite3"] = sys.modules.pop("pysqlite3") + from .embeddings import Embeddings + from .maintainer import EmbeddingMaintainer + + embeddings = Embeddings() + + # Check if we need to re-index events + if config.semantic_search.reindex: + embeddings.reindex() + + maintainer = EmbeddingMaintainer( + config, + stop_event, + ) + maintainer.start() diff --git a/frigate/embeddings/embeddings.py b/frigate/embeddings/embeddings.py new file mode 100644 index 0000000000..c7a688d120 --- /dev/null +++ b/frigate/embeddings/embeddings.py @@ -0,0 +1,122 @@ +"""ChromaDB embeddings database.""" + +import base64 +import io +import logging +import time + +import numpy as np +from chromadb import Collection +from chromadb import HttpClient as ChromaClient +from chromadb.config import Settings +from PIL import Image +from playhouse.shortcuts import model_to_dict + +from frigate.models import Event + +from .functions.clip import ClipEmbedding +from .functions.minilm_l6_v2 import MiniLMEmbedding + +logger = logging.getLogger(__name__) + + +def get_metadata(event: Event) -> dict: + """Extract valid event metadata.""" + event_dict = model_to_dict(event) + return ( + { + k: v + for k, v in event_dict.items() + if k not in ["id", "thumbnail"] + and v is not None + and isinstance(v, (str, int, float, bool)) + } + | { + k: v + for k, v in event_dict["data"].items() + if k not in ["description"] + and v is not None + and isinstance(v, (str, int, float, bool)) + } + | { + # Metadata search doesn't support $contains + # and an event can have multiple zones, so + # we need to create a key for each zone + f"{k}_{x}": True + for k, v in event_dict.items() + if isinstance(v, list) and len(v) > 0 + for x in v + if isinstance(x, str) + } + ) + + +class Embeddings: + """ChromaDB embeddings database.""" + + def __init__(self) -> None: + self.client: ChromaClient = ChromaClient( + host="127.0.0.1", + settings=Settings(anonymized_telemetry=False), + ) + + @property + def thumbnail(self) -> Collection: + return self.client.get_or_create_collection( + name="event_thumbnail", embedding_function=ClipEmbedding() + ) + + @property + def description(self) -> Collection: + return self.client.get_or_create_collection( + name="event_description", embedding_function=MiniLMEmbedding() + ) + + def reindex(self) -> None: + """Reindex all event embeddings.""" + logger.info("Indexing event embeddings...") + self.client.reset() + + st = time.time() + + thumbnails = {"ids": [], "images": [], "metadatas": []} + descriptions = {"ids": [], "documents": [], "metadatas": []} + + events = Event.select().where( + (Event.has_clip == True | Event.has_snapshot == True) + & Event.thumbnail.is_null(False) + ) + + event: Event + for event in events.iterator(): + metadata = get_metadata(event) + thumbnail = base64.b64decode(event.thumbnail) + img = np.array(Image.open(io.BytesIO(thumbnail)).convert("RGB")) + thumbnails["ids"].append(event.id) + thumbnails["images"].append(img) + thumbnails["metadatas"].append(metadata) + if event.data.get("description") is not None: + descriptions["ids"].append(event.id) + descriptions["documents"].append(event.data["description"]) + descriptions["metadatas"].append(metadata) + + if len(thumbnails["ids"]) > 0: + self.thumbnail.upsert( + images=thumbnails["images"], + metadatas=thumbnails["metadatas"], + ids=thumbnails["ids"], + ) + + if len(descriptions["ids"]) > 0: + self.description.upsert( + documents=descriptions["documents"], + metadatas=descriptions["metadatas"], + ids=descriptions["ids"], + ) + + logger.info( + "Embedded %d thumbnails and %d descriptions in %s seconds", + len(thumbnails["ids"]), + len(descriptions["ids"]), + time.time() - st, + ) diff --git a/frigate/embeddings/functions/clip.py b/frigate/embeddings/functions/clip.py new file mode 100644 index 0000000000..867938aff4 --- /dev/null +++ b/frigate/embeddings/functions/clip.py @@ -0,0 +1,63 @@ +"""CLIP Embeddings for Frigate.""" + +import os +from typing import Tuple, Union + +import onnxruntime as ort +from chromadb import EmbeddingFunction, Embeddings +from chromadb.api.types import ( + Documents, + Images, + is_document, + is_image, +) +from onnx_clip import OnnxClip + +from frigate.const import MODEL_CACHE_DIR + + +class Clip(OnnxClip): + """Override load models to download to cache directory.""" + + @staticmethod + def _load_models( + model: str, + silent: bool, + ) -> Tuple[ort.InferenceSession, ort.InferenceSession]: + """ + These models are a part of the container. Treat as as such. + """ + if model == "ViT-B/32": + IMAGE_MODEL_FILE = "clip_image_model_vitb32.onnx" + TEXT_MODEL_FILE = "clip_text_model_vitb32.onnx" + elif model == "RN50": + IMAGE_MODEL_FILE = "clip_image_model_rn50.onnx" + TEXT_MODEL_FILE = "clip_text_model_rn50.onnx" + else: + raise ValueError(f"Unexpected model {model}. No `.onnx` file found.") + + models = [] + for model_file in [IMAGE_MODEL_FILE, TEXT_MODEL_FILE]: + path = os.path.join(MODEL_CACHE_DIR, "clip", model_file) + models.append(OnnxClip._load_model(path, silent)) + + return models[0], models[1] + + +class ClipEmbedding(EmbeddingFunction): + """Embedding function for CLIP model used in Chroma.""" + + def __init__(self, model: str = "ViT-B/32"): + """Initialize CLIP Embedding function.""" + self.model = Clip(model) + + def __call__(self, input: Union[Documents, Images]) -> Embeddings: + embeddings: Embeddings = [] + for item in input: + if is_image(item): + result = self.model.get_image_embeddings([item]) + embeddings.append(result[0, :].tolist()) + elif is_document(item): + result = self.model.get_text_embeddings([item]) + embeddings.append(result[0, :].tolist()) + return embeddings diff --git a/frigate/embeddings/functions/minilm_l6_v2.py b/frigate/embeddings/functions/minilm_l6_v2.py new file mode 100644 index 0000000000..f90060fdb3 --- /dev/null +++ b/frigate/embeddings/functions/minilm_l6_v2.py @@ -0,0 +1,11 @@ +"""Embedding function for ONNX MiniLM-L6 model used in Chroma.""" + +from chromadb.utils.embedding_functions import ONNXMiniLM_L6_V2 + +from frigate.const import MODEL_CACHE_DIR + + +class MiniLMEmbedding(ONNXMiniLM_L6_V2): + """Override DOWNLOAD_PATH to download to cache directory.""" + + DOWNLOAD_PATH = f"{MODEL_CACHE_DIR}/all-MiniLM-L6-v2" diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py new file mode 100644 index 0000000000..aac85c01a6 --- /dev/null +++ b/frigate/embeddings/maintainer.py @@ -0,0 +1,197 @@ +"""Maintain embeddings in Chroma.""" + +import base64 +import io +import logging +import threading +from multiprocessing.synchronize import Event as MpEvent +from typing import Optional + +import cv2 +import numpy as np +from peewee import DoesNotExist +from PIL import Image + +from frigate.comms.events_updater import EventEndSubscriber, EventUpdateSubscriber +from frigate.comms.inter_process import InterProcessRequestor +from frigate.config import FrigateConfig +from frigate.const import UPDATE_EVENT_DESCRIPTION +from frigate.events.types import EventTypeEnum +from frigate.genai import get_genai_client +from frigate.models import Event +from frigate.util.image import SharedMemoryFrameManager, calculate_region + +from .embeddings import Embeddings, get_metadata + +logger = logging.getLogger(__name__) + + +class EmbeddingMaintainer(threading.Thread): + """Handle embedding queue and post event updates.""" + + def __init__( + self, + config: FrigateConfig, + stop_event: MpEvent, + ) -> None: + threading.Thread.__init__(self) + self.name = "embeddings_maintainer" + self.config = config + self.embeddings = Embeddings() + self.event_subscriber = EventUpdateSubscriber() + self.event_end_subscriber = EventEndSubscriber() + self.frame_manager = SharedMemoryFrameManager() + # create communication for updating event descriptions + self.requestor = InterProcessRequestor() + self.stop_event = stop_event + self.tracked_events = {} + self.genai_client = get_genai_client(config.genai) + + def run(self) -> None: + """Maintain a Chroma vector database for semantic search.""" + while not self.stop_event.is_set(): + self._process_updates() + self._process_finalized() + + self.event_subscriber.stop() + self.event_end_subscriber.stop() + self.requestor.stop() + logger.info("Exiting embeddings maintenance...") + + def _process_updates(self) -> None: + """Process event updates""" + update = self.event_subscriber.check_for_update() + + if update is None: + return + + source_type, _, camera, data = update + + if not camera or source_type != EventTypeEnum.tracked_object: + return + + camera_config = self.config.cameras[camera] + if data["id"] not in self.tracked_events: + self.tracked_events[data["id"]] = [] + + # Create our own thumbnail based on the bounding box and the frame time + try: + frame_id = f"{camera}{data['frame_time']}" + yuv_frame = self.frame_manager.get(frame_id, camera_config.frame_shape_yuv) + data["thumbnail"] = self._create_thumbnail(yuv_frame, data["box"]) + self.tracked_events[data["id"]].append(data) + self.frame_manager.close(frame_id) + except FileNotFoundError: + pass + + def _process_finalized(self) -> None: + """Process the end of an event.""" + while True: + ended = self.event_end_subscriber.check_for_update() + + if ended == None: + break + + event_id, camera, updated_db = ended + camera_config = self.config.cameras[camera] + + if updated_db: + try: + event: Event = Event.get(Event.id == event_id) + except DoesNotExist: + continue + + # Skip the event if not an object + if event.data.get("type") != "object": + continue + + # Extract valid event metadata + metadata = get_metadata(event) + thumbnail = base64.b64decode(event.thumbnail) + + # Embed the thumbnail + self._embed_thumbnail(event_id, thumbnail, metadata) + + if ( + camera_config.genai.enabled + and self.genai_client is not None + and event.data.get("description") is None + ): + # Generate the description. Call happens in a thread since it is network bound. + threading.Thread( + target=self._embed_description, + name=f"_embed_description_{event.id}", + daemon=True, + args=( + event, + [ + data["thumbnail"] + for data in self.tracked_events[event_id] + ] + if len(self.tracked_events.get(event_id, [])) > 0 + else [thumbnail], + metadata, + ), + ).start() + + # Delete tracked events based on the event_id + if event_id in self.tracked_events: + del self.tracked_events[event_id] + + def _create_thumbnail(self, yuv_frame, box, height=500) -> Optional[bytes]: + """Return jpg thumbnail of a region of the frame.""" + frame = cv2.cvtColor(yuv_frame, cv2.COLOR_YUV2BGR_I420) + region = calculate_region( + frame.shape, box[0], box[1], box[2], box[3], height, multiplier=1.4 + ) + frame = frame[region[1] : region[3], region[0] : region[2]] + width = int(height * frame.shape[1] / frame.shape[0]) + frame = cv2.resize(frame, dsize=(width, height), interpolation=cv2.INTER_AREA) + ret, jpg = cv2.imencode(".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), 100]) + + if ret: + return jpg.tobytes() + + return None + + def _embed_thumbnail(self, event_id: str, thumbnail: bytes, metadata: dict) -> None: + """Embed the thumbnail for an event.""" + + # Encode the thumbnail + img = np.array(Image.open(io.BytesIO(thumbnail)).convert("RGB")) + self.embeddings.thumbnail.upsert( + images=[img], + metadatas=[metadata], + ids=[event_id], + ) + + def _embed_description( + self, event: Event, thumbnails: list[bytes], metadata: dict + ) -> None: + """Embed the description for an event.""" + + description = self.genai_client.generate_description(thumbnails, metadata) + + if description is None: + logger.debug("Failed to generate description for %s", event.id) + return + + # fire and forget description update + self.requestor.send_data( + UPDATE_EVENT_DESCRIPTION, + {"id": event.id, "description": description}, + ) + + # Encode the description + self.embeddings.description.upsert( + documents=[description], + metadatas=[metadata], + ids=[event.id], + ) + + logger.debug( + "Generated description for %s (%d images): %s", + event.id, + len(thumbnails), + description, + ) diff --git a/frigate/events/audio.py b/frigate/events/audio.py index 471d403b81..6a7736bd36 100644 --- a/frigate/events/audio.py +++ b/frigate/events/audio.py @@ -223,7 +223,7 @@ def detect_audio(self, audio) -> None: audio_detections.append(label) # send audio detection data - self.detection_publisher.send_data( + self.detection_publisher.publish( ( self.config.name, datetime.datetime.now().timestamp(), diff --git a/frigate/events/cleanup.py b/frigate/events/cleanup.py index 88b4f0de66..12b00f620b 100644 --- a/frigate/events/cleanup.py +++ b/frigate/events/cleanup.py @@ -10,6 +10,7 @@ from frigate.config import FrigateConfig from frigate.const import CLIPS_DIR +from frigate.embeddings.embeddings import Embeddings from frigate.models import Event, Timeline logger = logging.getLogger(__name__) @@ -26,6 +27,7 @@ def __init__(self, config: FrigateConfig, stop_event: MpEvent): self.name = "event_cleanup" self.config = config self.stop_event = stop_event + self.embeddings = Embeddings() self.camera_keys = list(self.config.cameras.keys()) self.removed_camera_labels: list[str] = None self.camera_labels: dict[str, dict[str, any]] = {} @@ -197,9 +199,20 @@ def run(self) -> None: self.expire(EventCleanupType.snapshots) # drop events from db where has_clip and has_snapshot are false - delete_query = Event.delete().where( - Event.has_clip == False, Event.has_snapshot == False + events = ( + Event.select() + .where(Event.has_clip == False, Event.has_snapshot == False) + .iterator() ) - delete_query.execute() + events_to_delete = [e.id for e in events] + if len(events_to_delete) > 0: + chunk_size = 50 + for i in range(0, len(events_to_delete), chunk_size): + chunk = events_to_delete[i : i + chunk_size] + Event.delete().where(Event.id << chunk).execute() + + if self.config.semantic_search.enabled: + self.embeddings.thumbnail.delete(ids=chunk) + self.embeddings.description.delete(ids=chunk) logger.info("Exiting event cleanup...") diff --git a/frigate/events/external.py b/frigate/events/external.py index 46ce6f12c5..00f7cee4f5 100644 --- a/frigate/events/external.py +++ b/frigate/events/external.py @@ -86,7 +86,7 @@ def create_manual_event( if source_type == "api": self.event_camera[event_id] = camera - self.detection_updater.send_data( + self.detection_updater.publish( ( camera, now, @@ -115,7 +115,7 @@ def finish_manual_event(self, event_id: str, end_time: float) -> None: ) if event_id in self.event_camera: - self.detection_updater.send_data( + self.detection_updater.publish( ( self.event_camera[event_id], end_time, diff --git a/frigate/events/maintainer.py b/frigate/events/maintainer.py index 0ef3ffaaca..e83194ede3 100644 --- a/frigate/events/maintainer.py +++ b/frigate/events/maintainer.py @@ -237,7 +237,7 @@ def handle_object_detection( if event_type == EventStateEnum.end: del self.events_in_process[event_data["id"]] - self.event_end_publisher.publish((event_data["id"], camera)) + self.event_end_publisher.publish((event_data["id"], camera, updated_db)) def handle_external_detection( self, event_type: EventStateEnum, event_data: Event diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py new file mode 100644 index 0000000000..3761fa62fc --- /dev/null +++ b/frigate/genai/__init__.py @@ -0,0 +1,63 @@ +"""Generative AI module for Frigate.""" + +import importlib +import os +from typing import Optional + +from frigate.config import GenAIConfig, GenAIProviderEnum + +PROVIDERS = {} + + +def register_genai_provider(key: GenAIProviderEnum): + """Register a GenAI provider.""" + + def decorator(cls): + PROVIDERS[key] = cls + return cls + + return decorator + + +class GenAIClient: + """Generative AI client for Frigate.""" + + def __init__(self, genai_config: GenAIConfig, timeout: int = 60) -> None: + self.genai_config: GenAIConfig = genai_config + self.timeout = timeout + self.provider = self._init_provider() + + def generate_description( + self, thumbnails: list[bytes], metadata: dict[str, any] + ) -> Optional[str]: + """Generate a description for the frame.""" + prompt = self.genai_config.object_prompts.get( + metadata["label"], self.genai_config.prompt + ).format(**metadata) + return self._send(prompt, thumbnails) + + def _init_provider(self): + """Initialize the client.""" + return None + + def _send(self, prompt: str, images: list[bytes]) -> Optional[str]: + """Submit a request to the provider.""" + return None + + +def get_genai_client(genai_config: GenAIConfig) -> Optional[GenAIClient]: + """Get the GenAI client.""" + if genai_config.enabled: + load_providers() + provider = PROVIDERS.get(genai_config.provider) + if provider: + return provider(genai_config) + return None + + +def load_providers(): + package_dir = os.path.dirname(__file__) + for filename in os.listdir(package_dir): + if filename.endswith(".py") and filename != "__init__.py": + module_name = f"frigate.genai.{filename[:-3]}" + importlib.import_module(module_name) diff --git a/frigate/genai/gemini.py b/frigate/genai/gemini.py new file mode 100644 index 0000000000..f5b7bd2df1 --- /dev/null +++ b/frigate/genai/gemini.py @@ -0,0 +1,49 @@ +"""Gemini Provider for Frigate AI.""" + +from typing import Optional + +import google.generativeai as genai +from google.api_core.exceptions import GoogleAPICallError + +from frigate.config import GenAIProviderEnum +from frigate.genai import GenAIClient, register_genai_provider + + +@register_genai_provider(GenAIProviderEnum.gemini) +class GeminiClient(GenAIClient): + """Generative AI client for Frigate using Gemini.""" + + provider: genai.GenerativeModel + + def _init_provider(self): + """Initialize the client.""" + genai.configure(api_key=self.genai_config.api_key) + return genai.GenerativeModel(self.genai_config.model) + + def _send(self, prompt: str, images: list[bytes]) -> Optional[str]: + """Submit a request to Gemini.""" + data = [ + { + "mime_type": "image/jpeg", + "data": img, + } + for img in images + ] + [prompt] + try: + response = self.provider.generate_content( + data, + generation_config=genai.types.GenerationConfig( + candidate_count=1, + ), + request_options=genai.types.RequestOptions( + timeout=self.timeout, + ), + ) + except GoogleAPICallError: + return None + try: + description = response.text.strip() + except ValueError: + # No description was generated + return None + return description diff --git a/frigate/genai/ollama.py b/frigate/genai/ollama.py new file mode 100644 index 0000000000..09bcad0c53 --- /dev/null +++ b/frigate/genai/ollama.py @@ -0,0 +1,41 @@ +"""Ollama Provider for Frigate AI.""" + +import logging +from typing import Optional + +from httpx import TimeoutException +from ollama import Client as ApiClient +from ollama import ResponseError + +from frigate.config import GenAIProviderEnum +from frigate.genai import GenAIClient, register_genai_provider + +logger = logging.getLogger(__name__) + + +@register_genai_provider(GenAIProviderEnum.ollama) +class OllamaClient(GenAIClient): + """Generative AI client for Frigate using Ollama.""" + + provider: ApiClient + + def _init_provider(self): + """Initialize the client.""" + client = ApiClient(host=self.genai_config.base_url, timeout=self.timeout) + response = client.pull(self.genai_config.model) + if response["status"] != "success": + logger.error("Failed to pull %s model from Ollama", self.genai_config.model) + return None + return client + + def _send(self, prompt: str, images: list[bytes]) -> Optional[str]: + """Submit a request to Ollama""" + try: + result = self.provider.generate( + self.genai_config.model, + prompt, + images=images, + ) + return result["response"].strip() + except (TimeoutException, ResponseError): + return None diff --git a/frigate/genai/openai.py b/frigate/genai/openai.py new file mode 100644 index 0000000000..d0178df8bc --- /dev/null +++ b/frigate/genai/openai.py @@ -0,0 +1,51 @@ +"""OpenAI Provider for Frigate AI.""" + +import base64 +from typing import Optional + +from httpx import TimeoutException +from openai import OpenAI + +from frigate.config import GenAIProviderEnum +from frigate.genai import GenAIClient, register_genai_provider + + +@register_genai_provider(GenAIProviderEnum.openai) +class OpenAIClient(GenAIClient): + """Generative AI client for Frigate using OpenAI.""" + + provider: OpenAI + + def _init_provider(self): + """Initialize the client.""" + return OpenAI(api_key=self.genai_config.api_key) + + def _send(self, prompt: str, images: list[bytes]) -> Optional[str]: + """Submit a request to OpenAI.""" + encoded_images = [base64.b64encode(image).decode("utf-8") for image in images] + try: + result = self.provider.chat.completions.create( + model=self.genai_config.model, + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{image}", + "detail": "low", + }, + } + for image in encoded_images + ] + + [prompt], + }, + ], + timeout=self.timeout, + ) + except TimeoutException: + return None + if len(result.choices) > 0: + return result.choices[0].message.content.strip() + return None diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 7ac0b7276c..dcf6014fcc 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -1187,7 +1187,7 @@ def run(self): ] # publish info on this frame - self.detection_publisher.send_data( + self.detection_publisher.publish( ( camera, frame_time, @@ -1274,7 +1274,7 @@ def run(self): if not update: break - event_id, camera = update + event_id, camera, _ = update self.camera_states[camera].finished(event_id) self.requestor.stop() diff --git a/frigate/output/output.py b/frigate/output/output.py index e458d3242e..e0e7d0cac0 100644 --- a/frigate/output/output.py +++ b/frigate/output/output.py @@ -80,7 +80,7 @@ def receiveSignal(signalNumber, frame): websocket_thread.start() while not stop_event.is_set(): - (topic, data) = detection_subscriber.get_data(timeout=1) + (topic, data) = detection_subscriber.check_for_update(timeout=1) if not topic: continue @@ -134,7 +134,7 @@ def receiveSignal(signalNumber, frame): move_preview_frames("clips") while True: - (topic, data) = detection_subscriber.get_data(timeout=0) + (topic, data) = detection_subscriber.check_for_update(timeout=0) if not topic: break diff --git a/frigate/record/maintainer.py b/frigate/record/maintainer.py index 50ead905c4..2d12e2c320 100644 --- a/frigate/record/maintainer.py +++ b/frigate/record/maintainer.py @@ -470,7 +470,7 @@ def run(self) -> None: stale_frame_count_threshold = 10 # empty the object recordings info queue while True: - (topic, data) = self.detection_subscriber.get_data( + (topic, data) = self.detection_subscriber.check_for_update( timeout=QUEUE_READ_TIMEOUT ) diff --git a/frigate/review/maintainer.py b/frigate/review/maintainer.py index dfc7259b23..fa2678d9b7 100644 --- a/frigate/review/maintainer.py +++ b/frigate/review/maintainer.py @@ -424,7 +424,7 @@ def run(self) -> None: camera_name = updated_topic.rpartition("/")[-1] self.config.cameras[camera_name].record = updated_record_config - (topic, data) = self.detection_subscriber.get_data(timeout=1) + (topic, data) = self.detection_subscriber.check_for_update(timeout=1) if not topic: continue From 9e825811f21fd067e5fd06a46e80a56bc9c85afe Mon Sep 17 00:00:00 2001 From: Jason Hunter Date: Sun, 23 Jun 2024 09:13:02 -0400 Subject: [PATCH 053/169] Semantic Search API (#12105) * initial event search api implementation * fix lint * fix tests * move chromadb imports and pysqlite hotswap to fix tests * remove unused import * switch default limit to 50 * fix events accidently pulling inside chroma results loop --- frigate/__main__.py | 7 +- frigate/api/app.py | 3 + frigate/api/event.py | 244 ++++++++++++++++++++++++++++++- frigate/app.py | 9 +- frigate/embeddings/__init__.py | 38 ++++- frigate/embeddings/embeddings.py | 23 ++- frigate/embeddings/util.py | 47 ++++++ frigate/test/test_http.py | 11 ++ 8 files changed, 359 insertions(+), 23 deletions(-) create mode 100644 frigate/embeddings/util.py diff --git a/frigate/__main__.py b/frigate/__main__.py index 7106f02097..8442069082 100644 --- a/frigate/__main__.py +++ b/frigate/__main__.py @@ -1,12 +1,9 @@ import faulthandler -import sys import threading from flask import cli -# Hotsawp the sqlite3 module for Chroma compatibility -__import__("pysqlite3") -sys.modules["sqlite3"] = sys.modules.pop("pysqlite3") +from frigate.app import FrigateApp faulthandler.enable() @@ -15,8 +12,6 @@ cli.show_server_banner = lambda *x: None if __name__ == "__main__": - from frigate.app import FrigateApp - frigate_app = FrigateApp() frigate_app.start() diff --git a/frigate/api/app.py b/frigate/api/app.py index 5fec51c03b..0e3b0fecde 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -23,6 +23,7 @@ from frigate.api.review import ReviewBp from frigate.config import FrigateConfig from frigate.const import CONFIG_DIR +from frigate.embeddings import EmbeddingsContext from frigate.events.external import ExternalEventProcessor from frigate.models import Event, Timeline from frigate.plus import PlusApi @@ -52,6 +53,7 @@ def create_app( frigate_config, database: SqliteQueueDatabase, + embeddings: EmbeddingsContext, detected_frames_processor, storage_maintainer: StorageMaintainer, onvif: OnvifController, @@ -79,6 +81,7 @@ def _db_close(exc): database.close() app.frigate_config = frigate_config + app.embeddings = embeddings app.detected_frames_processor = detected_frames_processor app.storage_maintainer = storage_maintainer app.onvif = onvif diff --git a/frigate/api/event.py b/frigate/api/event.py index 267d13bc91..0ecb9ddbd4 100644 --- a/frigate/api/event.py +++ b/frigate/api/event.py @@ -1,5 +1,7 @@ """Event apis.""" +import base64 +import io import logging import os from datetime import datetime @@ -8,6 +10,7 @@ from urllib.parse import unquote import cv2 +import numpy as np from flask import ( Blueprint, current_app, @@ -15,13 +18,16 @@ make_response, request, ) -from peewee import DoesNotExist, fn, operator +from peewee import JOIN, DoesNotExist, fn, operator +from PIL import Image from playhouse.shortcuts import model_to_dict from frigate.const import ( CLIPS_DIR, ) -from frigate.models import Event, Timeline +from frigate.embeddings import EmbeddingsContext +from frigate.embeddings.embeddings import get_metadata +from frigate.models import Event, ReviewSegment, Timeline from frigate.object_processing import TrackedObject from frigate.util.builtin import get_tz_modifiers @@ -245,6 +251,189 @@ def events(): return jsonify(list(events)) +@EventBp.route("/events/search") +def events_search(): + query = request.args.get("query", type=str) + search_type = request.args.get("search_type", "text", type=str) + include_thumbnails = request.args.get("include_thumbnails", default=1, type=int) + limit = request.args.get("limit", 50, type=int) + + # Filters + cameras = request.args.get("cameras", "all", type=str) + labels = request.args.get("labels", "all", type=str) + zones = request.args.get("zones", "all", type=str) + after = request.args.get("after", type=float) + before = request.args.get("before", type=float) + + if not query: + return make_response( + jsonify( + { + "success": False, + "message": "A search query must be supplied", + } + ), + 400, + ) + + if not current_app.frigate_config.semantic_search.enabled: + return make_response( + jsonify( + { + "success": False, + "message": "Semantic search is not enabled", + } + ), + 400, + ) + + context: EmbeddingsContext = current_app.embeddings + + selected_columns = [ + Event.id, + Event.camera, + Event.label, + Event.sub_label, + Event.zones, + Event.start_time, + Event.end_time, + Event.data, + ReviewSegment.thumb_path, + ] + + if include_thumbnails: + selected_columns.append(Event.thumbnail) + + # Build the where clause for the embeddings query + embeddings_filters = [] + + if cameras != "all": + camera_list = cameras.split(",") + embeddings_filters.append({"camera": {"$in": camera_list}}) + + if labels != "all": + label_list = labels.split(",") + embeddings_filters.append({"label": {"$in": label_list}}) + + if zones != "all": + filtered_zones = zones.split(",") + zone_filters = [{f"zones_{zone}": {"$eq": True}} for zone in filtered_zones] + if len(zone_filters) > 1: + embeddings_filters.append({"$or": zone_filters}) + else: + embeddings_filters.append(zone_filters[0]) + + if after: + embeddings_filters.append({"start_time": {"$gt": after}}) + + if before: + embeddings_filters.append({"start_time": {"$lt": before}}) + + where = None + if len(embeddings_filters) > 1: + where = {"$and": embeddings_filters} + elif len(embeddings_filters) == 1: + where = embeddings_filters[0] + + thumb_ids = {} + desc_ids = {} + + if search_type == "thumbnail": + # Grab the ids of events that match the thumbnail image embeddings + try: + search_event: Event = Event.get(Event.id == query) + except DoesNotExist: + return make_response( + jsonify( + { + "success": False, + "message": "Event not found", + } + ), + 404, + ) + thumbnail = base64.b64decode(search_event.thumbnail) + img = np.array(Image.open(io.BytesIO(thumbnail)).convert("RGB")) + thumb_result = context.embeddings.thumbnail.query( + query_images=[img], + n_results=limit, + where=where, + ) + thumb_ids = dict(zip(thumb_result["ids"][0], thumb_result["distances"][0])) + else: + thumb_result = context.embeddings.thumbnail.query( + query_texts=[query], + n_results=limit, + where=where, + ) + # Do a rudimentary normalization of the difference in distances returned by CLIP and MiniLM. + thumb_ids = dict( + zip( + thumb_result["ids"][0], + context.thumb_stats.normalize(thumb_result["distances"][0]), + ) + ) + desc_result = context.embeddings.description.query( + query_texts=[query], + n_results=limit, + where=where, + ) + desc_ids = dict( + zip( + desc_result["ids"][0], + context.desc_stats.normalize(desc_result["distances"][0]), + ) + ) + + results = {} + for event_id in thumb_ids.keys() | desc_ids: + min_distance = min( + i + for i in (thumb_ids.get(event_id), desc_ids.get(event_id)) + if i is not None + ) + results[event_id] = { + "distance": min_distance, + "source": "thumbnail" + if min_distance == thumb_ids.get(event_id) + else "description", + } + + if not results: + return jsonify([]) + + # Get the event data + events = ( + Event.select(*selected_columns) + .join( + ReviewSegment, + JOIN.LEFT_OUTER, + on=(fn.json_extract(ReviewSegment.data, "$.detections").contains(Event.id)), + ) + .where(Event.id << list(results.keys())) + .dicts() + .iterator() + ) + events = list(events) + + events = [ + {k: v for k, v in event.items() if k != "data"} + | { + k: v + for k, v in event["data"].items() + if k in ["type", "score", "top_score", "description"] + } + | { + "search_distance": results[event["id"]]["distance"], + "search_source": results[event["id"]]["source"], + } + for event in events + ] + events = sorted(events, key=lambda x: x["search_distance"])[:limit] + + return jsonify(events) + + @EventBp.route("/events/summary") def events_summary(): tz_name = request.args.get("timezone", default="utc", type=str) @@ -604,6 +793,52 @@ def set_sub_label(id): ) +@EventBp.route("/events//description", methods=("POST",)) +def set_description(id): + try: + event: Event = Event.get(Event.id == id) + except DoesNotExist: + return make_response( + jsonify({"success": False, "message": "Event " + id + " not found"}), 404 + ) + + json: dict[str, any] = request.get_json(silent=True) or {} + new_description = json.get("description") + + if new_description is None or len(new_description) == 0: + return make_response( + jsonify( + { + "success": False, + "message": "description cannot be empty", + } + ), + 400, + ) + + event.data["description"] = new_description + event.save() + + # If semantic search is enabled, update the index + if current_app.frigate_config.semantic_search.enabled: + context: EmbeddingsContext = current_app.embeddings + context.embeddings.description.upsert( + documents=[new_description], + metadatas=[get_metadata(event)], + ids=[id], + ) + + return make_response( + jsonify( + { + "success": True, + "message": "Event " + id + " description set to " + new_description, + } + ), + 200, + ) + + @EventBp.route("/events/", methods=("DELETE",)) def delete_event(id): try: @@ -625,6 +860,11 @@ def delete_event(id): event.delete_instance() Timeline.delete().where(Timeline.source_id == id).execute() + # If semantic search is enabled, update the index + if current_app.frigate_config.semantic_search.enabled: + context: EmbeddingsContext = current_app.embeddings + context.embeddings.thumbnail.delete(ids=[id]) + context.embeddings.description.delete(ids=[id]) return make_response( jsonify({"success": True, "message": "Event " + id + " deleted"}), 200 ) diff --git a/frigate/app.py b/frigate/app.py index 840686f0a7..ef9360354d 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -37,8 +37,7 @@ MODEL_CACHE_DIR, RECORD_DIR, ) -from frigate.embeddings import manage_embeddings -from frigate.embeddings.embeddings import Embeddings +from frigate.embeddings import EmbeddingsContext, manage_embeddings from frigate.events.audio import listen_to_audio from frigate.events.cleanup import EventCleanup from frigate.events.external import ExternalEventProcessor @@ -322,7 +321,7 @@ def init_review_segment_manager(self) -> None: def init_embeddings_manager(self) -> None: # Create a client for other processes to use - self.embeddings = Embeddings() + self.embeddings = EmbeddingsContext() embedding_process = mp.Process( target=manage_embeddings, name="embeddings_manager", @@ -384,6 +383,7 @@ def init_web_server(self) -> None: self.flask_app = create_app( self.config, self.db, + self.embeddings, self.detected_frames_processor, self.storage_maintainer, self.onvif_controller, @@ -811,6 +811,9 @@ def stop(self) -> None: self.frigate_watchdog.join() self.db.stop() + # Save embeddings stats to disk + self.embeddings.save_stats() + # Stop Communicators self.inter_process_communicator.stop() self.inter_config_updater.stop() diff --git a/frigate/embeddings/__init__.py b/frigate/embeddings/__init__.py index 41af73c012..b3ad228745 100644 --- a/frigate/embeddings/__init__.py +++ b/frigate/embeddings/__init__.py @@ -1,9 +1,9 @@ """ChromaDB embeddings database.""" +import json import logging import multiprocessing as mp import signal -import sys import threading from types import FrameType from typing import Optional @@ -12,9 +12,14 @@ from setproctitle import setproctitle from frigate.config import FrigateConfig +from frigate.const import CONFIG_DIR from frigate.models import Event from frigate.util.services import listen +from .embeddings import Embeddings +from .maintainer import EmbeddingMaintainer +from .util import ZScoreNormalization + logger = logging.getLogger(__name__) @@ -48,12 +53,6 @@ def receiveSignal(signalNumber: int, frame: Optional[FrameType]) -> None: models = [Event] db.bind(models) - # Hotsawp the sqlite3 module for Chroma compatibility - __import__("pysqlite3") - sys.modules["sqlite3"] = sys.modules.pop("pysqlite3") - from .embeddings import Embeddings - from .maintainer import EmbeddingMaintainer - embeddings = Embeddings() # Check if we need to re-index events @@ -65,3 +64,28 @@ def receiveSignal(signalNumber: int, frame: Optional[FrameType]) -> None: stop_event, ) maintainer.start() + + +class EmbeddingsContext: + def __init__(self): + self.embeddings = Embeddings() + self.thumb_stats = ZScoreNormalization() + self.desc_stats = ZScoreNormalization() + + # load stats from disk + try: + with open(f"{CONFIG_DIR}/.search_stats.json", "r") as f: + data = json.loads(f.read()) + self.thumb_stats.from_dict(data["thumb_stats"]) + self.desc_stats.from_dict(data["desc_stats"]) + except FileNotFoundError: + pass + + def save_stats(self): + """Write the stats to disk as JSON on exit.""" + contents = { + "thumb_stats": self.thumb_stats.to_dict(), + "desc_stats": self.desc_stats.to_dict(), + } + with open(f"{CONFIG_DIR}/.search_stats.json", "w") as f: + f.write(json.dumps(contents)) diff --git a/frigate/embeddings/embeddings.py b/frigate/embeddings/embeddings.py index c7a688d120..58dd707bbb 100644 --- a/frigate/embeddings/embeddings.py +++ b/frigate/embeddings/embeddings.py @@ -3,19 +3,32 @@ import base64 import io import logging +import sys import time import numpy as np -from chromadb import Collection -from chromadb import HttpClient as ChromaClient -from chromadb.config import Settings from PIL import Image from playhouse.shortcuts import model_to_dict from frigate.models import Event -from .functions.clip import ClipEmbedding -from .functions.minilm_l6_v2 import MiniLMEmbedding +# Hotsawp the sqlite3 module for Chroma compatibility +try: + from chromadb import Collection + from chromadb import HttpClient as ChromaClient + from chromadb.config import Settings + + from .functions.clip import ClipEmbedding + from .functions.minilm_l6_v2 import MiniLMEmbedding +except RuntimeError: + __import__("pysqlite3") + sys.modules["sqlite3"] = sys.modules.pop("pysqlite3") + from chromadb import Collection + from chromadb import HttpClient as ChromaClient + from chromadb.config import Settings + + from .functions.clip import ClipEmbedding + from .functions.minilm_l6_v2 import MiniLMEmbedding logger = logging.getLogger(__name__) diff --git a/frigate/embeddings/util.py b/frigate/embeddings/util.py new file mode 100644 index 0000000000..7550716c93 --- /dev/null +++ b/frigate/embeddings/util.py @@ -0,0 +1,47 @@ +"""Z-score normalization for search distance.""" + +import math + + +class ZScoreNormalization: + """Running Z-score normalization for search distance.""" + + def __init__(self): + self.n = 0 + self.mean = 0 + self.m2 = 0 + + @property + def variance(self): + return self.m2 / (self.n - 1) if self.n > 1 else 0.0 + + @property + def stddev(self): + return math.sqrt(self.variance) + + def normalize(self, distances: list[float]): + self._update(distances) + if self.stddev == 0: + return distances + return [(x - self.mean) / self.stddev for x in distances] + + def _update(self, distances: list[float]): + for x in distances: + self.n += 1 + delta = x - self.mean + self.mean += delta / self.n + delta2 = x - self.mean + self.m2 += delta * delta2 + + def to_dict(self): + return { + "n": self.n, + "mean": self.mean, + "m2": self.m2, + } + + def from_dict(self, data: dict): + self.n = data["n"] + self.mean = data["mean"] + self.m2 = data["m2"] + return self diff --git a/frigate/test/test_http.py b/frigate/test/test_http.py index f0cb927f4c..936dc80e5b 100644 --- a/frigate/test/test_http.py +++ b/frigate/test/test_http.py @@ -120,6 +120,7 @@ def test_get_event_list(self): None, None, None, + None, PlusApi(), None, ) @@ -156,6 +157,7 @@ def test_get_good_event(self): None, None, None, + None, PlusApi(), None, ) @@ -177,6 +179,7 @@ def test_get_bad_event(self): None, None, None, + None, PlusApi(), None, ) @@ -197,6 +200,7 @@ def test_delete_event(self): None, None, None, + None, PlusApi(), None, ) @@ -219,6 +223,7 @@ def test_event_retention(self): None, None, None, + None, PlusApi(), None, ) @@ -245,6 +250,7 @@ def test_event_time_filtering(self): None, None, None, + None, PlusApi(), None, ) @@ -283,6 +289,7 @@ def test_set_delete_sub_label(self): None, None, None, + None, PlusApi(), None, ) @@ -318,6 +325,7 @@ def test_sub_label_list(self): None, None, None, + None, PlusApi(), None, ) @@ -343,6 +351,7 @@ def test_config(self): None, None, None, + None, PlusApi(), None, ) @@ -360,6 +369,7 @@ def test_recordings(self): None, None, None, + None, PlusApi(), None, ) @@ -381,6 +391,7 @@ def test_stats(self): None, None, None, + None, PlusApi(), stats, ) From 0d7a1488972ea663abf26e2da4ec2e5b768c0995 Mon Sep 17 00:00:00 2001 From: Jason Hunter Date: Sun, 23 Jun 2024 15:27:21 -0400 Subject: [PATCH 054/169] reindex events in batches to reduce memory and cpu load (#12124) --- frigate/embeddings/embeddings.py | 94 +++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 33 deletions(-) diff --git a/frigate/embeddings/embeddings.py b/frigate/embeddings/embeddings.py index 58dd707bbb..c0dc51ea94 100644 --- a/frigate/embeddings/embeddings.py +++ b/frigate/embeddings/embeddings.py @@ -12,6 +12,9 @@ from frigate.models import Event +# Squelch posthog logging +logging.getLogger("chromadb.telemetry.product.posthog").setLevel(logging.CRITICAL) + # Hotsawp the sqlite3 module for Chroma compatibility try: from chromadb import Collection @@ -91,45 +94,70 @@ def reindex(self) -> None: self.client.reset() st = time.time() + totals = { + "thumb": 0, + "desc": 0, + } - thumbnails = {"ids": [], "images": [], "metadatas": []} - descriptions = {"ids": [], "documents": [], "metadatas": []} - - events = Event.select().where( - (Event.has_clip == True | Event.has_snapshot == True) - & Event.thumbnail.is_null(False) - ) - - event: Event - for event in events.iterator(): - metadata = get_metadata(event) - thumbnail = base64.b64decode(event.thumbnail) - img = np.array(Image.open(io.BytesIO(thumbnail)).convert("RGB")) - thumbnails["ids"].append(event.id) - thumbnails["images"].append(img) - thumbnails["metadatas"].append(metadata) - if event.data.get("description") is not None: - descriptions["ids"].append(event.id) - descriptions["documents"].append(event.data["description"]) - descriptions["metadatas"].append(metadata) - - if len(thumbnails["ids"]) > 0: - self.thumbnail.upsert( - images=thumbnails["images"], - metadatas=thumbnails["metadatas"], - ids=thumbnails["ids"], + batch_size = 100 + current_page = 1 + events = ( + Event.select() + .where( + (Event.has_clip == True | Event.has_snapshot == True) + & Event.thumbnail.is_null(False) ) + .order_by(Event.start_time.desc()) + .paginate(current_page, batch_size) + ) - if len(descriptions["ids"]) > 0: - self.description.upsert( - documents=descriptions["documents"], - metadatas=descriptions["metadatas"], - ids=descriptions["ids"], + while len(events) > 0: + thumbnails = {"ids": [], "images": [], "metadatas": []} + descriptions = {"ids": [], "documents": [], "metadatas": []} + + event: Event + for event in events: + metadata = get_metadata(event) + thumbnail = base64.b64decode(event.thumbnail) + img = np.array(Image.open(io.BytesIO(thumbnail)).convert("RGB")) + thumbnails["ids"].append(event.id) + thumbnails["images"].append(img) + thumbnails["metadatas"].append(metadata) + if event.data.get("description") is not None: + descriptions["ids"].append(event.id) + descriptions["documents"].append(event.data["description"]) + descriptions["metadatas"].append(metadata) + + if len(thumbnails["ids"]) > 0: + totals["thumb"] += len(thumbnails["ids"]) + self.thumbnail.upsert( + images=thumbnails["images"], + metadatas=thumbnails["metadatas"], + ids=thumbnails["ids"], + ) + + if len(descriptions["ids"]) > 0: + totals["desc"] += len(descriptions["ids"]) + self.description.upsert( + documents=descriptions["documents"], + metadatas=descriptions["metadatas"], + ids=descriptions["ids"], + ) + + current_page += 1 + events = ( + Event.select() + .where( + (Event.has_clip == True | Event.has_snapshot == True) + & Event.thumbnail.is_null(False) + ) + .order_by(Event.start_time.desc()) + .paginate(current_page, batch_size) ) logger.info( "Embedded %d thumbnails and %d descriptions in %s seconds", - len(thumbnails["ids"]), - len(descriptions["ids"]), + totals["thumb"], + totals["desc"], time.time() - st, ) From 9d7e499adbca20597179d5723c6d776bb7c9fe19 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Sun, 23 Jun 2024 14:58:00 -0600 Subject: [PATCH 055/169] Semantic Search Frontend (#12112) * Add basic search page * Abstract filters to separate components * Make searching functional * Add loading and no results indicators * Implement searching * Combine account and settings menus on mobile * Support using thumbnail for in progress detections * Fetch previews * Move recordings view and open recordings when search is selected * Implement detail pane * Implement saving of description * Implement similarity search * Fix clicking * Add date range picker * Fix * Fix iOS zoom bug * Mobile fixes * Use text area * Fix spacing for drawer * Fix fetching previews incorrectly --- web/src/App.tsx | 2 + web/src/components/card/AnimatedEventCard.tsx | 2 +- .../filter/CalendarFilterButton.tsx | 152 ++++++ .../components/filter/CamerasFilterButton.tsx | 177 +++++++ .../components/filter/ReviewFilterGroup.tsx | 254 +-------- .../components/filter/SearchFilterGroup.tsx | 458 ++++++++++++++++ web/src/components/indicators/Chip.tsx | 8 +- web/src/components/menu/GeneralSettings.tsx | 32 +- web/src/components/navigation/Bottombar.tsx | 2 - .../components/overlay/SearchDetailDialog.tsx | 167 ++++++ .../player/PreviewThumbnailPlayer.tsx | 496 +----------------- .../player/SearchThumbnailPlayer.tsx | 331 ++++++++++++ .../components/preview/ScrubbablePreview.tsx | 486 +++++++++++++++++ web/src/components/ui/calendar-range.tsx | 444 ++++++++++++++++ web/src/components/ui/textarea.tsx | 24 + web/src/hooks/use-date-utils.ts | 15 + web/src/hooks/use-navigation.ts | 28 +- web/src/pages/Events.tsx | 2 +- web/src/pages/Search.tsx | 201 +++++++ web/src/pages/SubmitPlus.tsx | 6 +- web/src/types/frigateConfig.ts | 4 + web/src/types/search.ts | 20 + .../{events => recording}/RecordingView.tsx | 0 web/src/views/search/SearchView.tsx | 126 +++++ 24 files changed, 2682 insertions(+), 755 deletions(-) create mode 100644 web/src/components/filter/CalendarFilterButton.tsx create mode 100644 web/src/components/filter/CamerasFilterButton.tsx create mode 100644 web/src/components/filter/SearchFilterGroup.tsx create mode 100644 web/src/components/overlay/SearchDetailDialog.tsx create mode 100644 web/src/components/player/SearchThumbnailPlayer.tsx create mode 100644 web/src/components/preview/ScrubbablePreview.tsx create mode 100644 web/src/components/ui/calendar-range.tsx create mode 100644 web/src/components/ui/textarea.tsx create mode 100644 web/src/pages/Search.tsx create mode 100644 web/src/types/search.ts rename web/src/views/{events => recording}/RecordingView.tsx (100%) create mode 100644 web/src/views/search/SearchView.tsx diff --git a/web/src/App.tsx b/web/src/App.tsx index 93e980f2c1..53491c6aa2 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -13,6 +13,7 @@ import { isPWA } from "./utils/isPWA"; const Live = lazy(() => import("@/pages/Live")); const Events = lazy(() => import("@/pages/Events")); +const Search = lazy(() => import("@/pages/Search")); const Exports = lazy(() => import("@/pages/Exports")); const SubmitPlus = lazy(() => import("@/pages/SubmitPlus")); const ConfigEditor = lazy(() => import("@/pages/ConfigEditor")); @@ -44,6 +45,7 @@ function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/web/src/components/card/AnimatedEventCard.tsx b/web/src/components/card/AnimatedEventCard.tsx index 85d6eb6d4b..7a2eba4b16 100644 --- a/web/src/components/card/AnimatedEventCard.tsx +++ b/web/src/components/card/AnimatedEventCard.tsx @@ -7,10 +7,10 @@ import { REVIEW_PADDING, ReviewSegment } from "@/types/review"; import { useNavigate } from "react-router-dom"; import { RecordingStartingPoint } from "@/types/record"; import axios from "axios"; -import { VideoPreview } from "../player/PreviewThumbnailPlayer"; import { isCurrentHour } from "@/utils/dateUtil"; import { useCameraPreviews } from "@/hooks/use-camera-previews"; import { baseUrl } from "@/api/baseUrl"; +import { VideoPreview } from "../preview/ScrubbablePreview"; import { useApiHost } from "@/api"; import { isDesktop, isSafari } from "react-device-detect"; import { usePersistence } from "@/hooks/use-persistence"; diff --git a/web/src/components/filter/CalendarFilterButton.tsx b/web/src/components/filter/CalendarFilterButton.tsx new file mode 100644 index 0000000000..9b630b8932 --- /dev/null +++ b/web/src/components/filter/CalendarFilterButton.tsx @@ -0,0 +1,152 @@ +import { + useFormattedRange, + useFormattedTimestamp, +} from "@/hooks/use-date-utils"; +import { ReviewSummary } from "@/types/review"; +import { Button } from "../ui/button"; +import { FaCalendarAlt } from "react-icons/fa"; +import ReviewActivityCalendar from "../overlay/ReviewActivityCalendar"; +import { DropdownMenuSeparator } from "../ui/dropdown-menu"; +import { Drawer, DrawerContent, DrawerTrigger } from "../ui/drawer"; +import { isMobile } from "react-device-detect"; +import { Popover, PopoverContent, PopoverTrigger } from "../ui/popover"; +import { DateRangePicker } from "../ui/calendar-range"; +import { DateRange } from "react-day-picker"; + +type CalendarFilterButtonProps = { + reviewSummary?: ReviewSummary; + day?: Date; + updateSelectedDay: (day?: Date) => void; +}; +export default function CalendarFilterButton({ + reviewSummary, + day, + updateSelectedDay, +}: CalendarFilterButtonProps) { + const selectedDate = useFormattedTimestamp( + day == undefined ? 0 : day?.getTime() / 1000 + 1, + "%b %-d", + ); + + const trigger = ( + + ); + const content = ( + <> + + +
+ +
+ + ); + + if (isMobile) { + return ( + + {trigger} + {content} + + ); + } + + return ( + + {trigger} + {content} + + ); +} + +type CalendarRangeFilterButtonProps = { + range?: DateRange; + defaultText: string; + updateSelectedRange: (range?: DateRange) => void; +}; +export function CalendarRangeFilterButton({ + range, + defaultText, + updateSelectedRange, +}: CalendarRangeFilterButtonProps) { + const selectedDate = useFormattedRange( + range?.from == undefined ? 0 : range.from.getTime() / 1000 + 1, + range?.to == undefined ? 0 : range.to.getTime() / 1000 - 1, + "%b %-d", + ); + + const trigger = ( + + ); + const content = ( + <> + updateSelectedRange(range.range)} + /> + +
+ +
+ + ); + + if (isMobile) { + return ( + + {trigger} + {content} + + ); + } + + return ( + + {trigger} + {content} + + ); +} diff --git a/web/src/components/filter/CamerasFilterButton.tsx b/web/src/components/filter/CamerasFilterButton.tsx new file mode 100644 index 0000000000..b1878bf12b --- /dev/null +++ b/web/src/components/filter/CamerasFilterButton.tsx @@ -0,0 +1,177 @@ +import { Button } from "../ui/button"; +import { CameraGroupConfig } from "@/types/frigateConfig"; +import { useState } from "react"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuLabel, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from "../ui/dropdown-menu"; +import { isMobile } from "react-device-detect"; +import { Drawer, DrawerContent, DrawerTrigger } from "../ui/drawer"; +import FilterSwitch from "./FilterSwitch"; +import { FaVideo } from "react-icons/fa"; + +type CameraFilterButtonProps = { + allCameras: string[]; + groups: [string, CameraGroupConfig][]; + selectedCameras: string[] | undefined; + updateCameraFilter: (cameras: string[] | undefined) => void; +}; +export function CamerasFilterButton({ + allCameras, + groups, + selectedCameras, + updateCameraFilter, +}: CameraFilterButtonProps) { + const [open, setOpen] = useState(false); + const [currentCameras, setCurrentCameras] = useState( + selectedCameras, + ); + + const trigger = ( + + ); + const content = ( + <> + {isMobile && ( + <> + + Cameras + + + + )} +
+ { + if (isChecked) { + setCurrentCameras(undefined); + } + }} + /> + {groups.length > 0 && ( + <> + + {groups.map(([name, conf]) => { + return ( +
setCurrentCameras([...conf.cameras])} + > + {name} +
+ ); + })} + + )} + +
+ {allCameras.map((item) => ( + { + if (isChecked) { + const updatedCameras = currentCameras + ? [...currentCameras] + : []; + + updatedCameras.push(item); + setCurrentCameras(updatedCameras); + } else { + const updatedCameras = currentCameras + ? [...currentCameras] + : []; + + // can not deselect the last item + if (updatedCameras.length > 1) { + updatedCameras.splice(updatedCameras.indexOf(item), 1); + setCurrentCameras(updatedCameras); + } + } + }} + /> + ))} +
+
+ +
+ + +
+ + ); + + if (isMobile) { + return ( + { + if (!open) { + setCurrentCameras(selectedCameras); + } + + setOpen(open); + }} + > + {trigger} + + {content} + + + ); + } + + return ( + { + if (!open) { + setCurrentCameras(selectedCameras); + } + + setOpen(open); + }} + > + {trigger} + {content} + + ); +} diff --git a/web/src/components/filter/ReviewFilterGroup.tsx b/web/src/components/filter/ReviewFilterGroup.tsx index d01ea384f5..d73e77464a 100644 --- a/web/src/components/filter/ReviewFilterGroup.tsx +++ b/web/src/components/filter/ReviewFilterGroup.tsx @@ -1,36 +1,24 @@ import { Button } from "../ui/button"; import { Popover, PopoverContent, PopoverTrigger } from "../ui/popover"; import useSWR from "swr"; -import { CameraGroupConfig, FrigateConfig } from "@/types/frigateConfig"; +import { FrigateConfig } from "@/types/frigateConfig"; import { useCallback, useMemo, useState } from "react"; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuLabel, - DropdownMenuSeparator, - DropdownMenuTrigger, -} from "../ui/dropdown-menu"; +import { DropdownMenuSeparator } from "../ui/dropdown-menu"; import { ReviewFilter, ReviewSeverity, ReviewSummary } from "@/types/review"; import { getEndOfDayTimestamp } from "@/utils/dateUtil"; -import { useFormattedTimestamp } from "@/hooks/use-date-utils"; -import { - FaCalendarAlt, - FaCheckCircle, - FaFilter, - FaRunning, - FaVideo, -} from "react-icons/fa"; +import { FaCheckCircle, FaFilter, FaRunning } from "react-icons/fa"; import { isDesktop, isMobile } from "react-device-detect"; import { Drawer, DrawerContent, DrawerTrigger } from "../ui/drawer"; import { Switch } from "../ui/switch"; import { Label } from "../ui/label"; -import ReviewActivityCalendar from "../overlay/ReviewActivityCalendar"; import MobileReviewSettingsDrawer, { DrawerFeatures, } from "../overlay/MobileReviewSettingsDrawer"; import useOptimisticState from "@/hooks/use-optimistic-state"; import FilterSwitch from "./FilterSwitch"; import { FilterList } from "@/types/filter"; +import CalendarFilterButton from "./CalendarFilterButton"; +import { CamerasFilterButton } from "./CamerasFilterButton"; const REVIEW_FILTERS = [ "cameras", @@ -210,6 +198,7 @@ export default function ReviewFilterGroup({ ? undefined : new Date(filter.after * 1000) } + defaultText="Last 24 Hours" updateSelectedDay={onUpdateSelectedDay} /> )} @@ -260,169 +249,6 @@ export default function ReviewFilterGroup({ ); } -type CameraFilterButtonProps = { - allCameras: string[]; - groups: [string, CameraGroupConfig][]; - selectedCameras: string[] | undefined; - updateCameraFilter: (cameras: string[] | undefined) => void; -}; -export function CamerasFilterButton({ - allCameras, - groups, - selectedCameras, - updateCameraFilter, -}: CameraFilterButtonProps) { - const [open, setOpen] = useState(false); - const [currentCameras, setCurrentCameras] = useState( - selectedCameras, - ); - - const trigger = ( - - ); - const content = ( - <> - {isMobile && ( - <> - - Cameras - - - - )} -
- { - if (isChecked) { - setCurrentCameras(undefined); - } - }} - /> - {groups.length > 0 && ( - <> - - {groups.map(([name, conf]) => { - return ( -
setCurrentCameras([...conf.cameras])} - > - {name} -
- ); - })} - - )} - -
- {allCameras.map((item) => ( - { - if (isChecked) { - const updatedCameras = currentCameras - ? [...currentCameras] - : []; - - updatedCameras.push(item); - setCurrentCameras(updatedCameras); - } else { - const updatedCameras = currentCameras - ? [...currentCameras] - : []; - - // can not deselect the last item - if (updatedCameras.length > 1) { - updatedCameras.splice(updatedCameras.indexOf(item), 1); - setCurrentCameras(updatedCameras); - } - } - }} - /> - ))} -
-
- -
- - -
- - ); - - if (isMobile) { - return ( - { - if (!open) { - setCurrentCameras(selectedCameras); - } - - setOpen(open); - }} - > - {trigger} - - {content} - - - ); - } - - return ( - { - if (!open) { - setCurrentCameras(selectedCameras); - } - - setOpen(open); - }} - > - {trigger} - {content} - - ); -} - type ShowReviewedFilterProps = { showReviewed: boolean; setShowReviewed: (reviewed: boolean) => void; @@ -466,74 +292,6 @@ function ShowReviewFilter({ ); } -type CalendarFilterButtonProps = { - reviewSummary?: ReviewSummary; - day?: Date; - updateSelectedDay: (day?: Date) => void; -}; -function CalendarFilterButton({ - reviewSummary, - day, - updateSelectedDay, -}: CalendarFilterButtonProps) { - const selectedDate = useFormattedTimestamp( - day == undefined ? 0 : day?.getTime() / 1000 + 1, - "%b %-d", - ); - - const trigger = ( - - ); - const content = ( - <> - - -
- -
- - ); - - if (isMobile) { - return ( - - {trigger} - {content} - - ); - } - - return ( - - {trigger} - {content} - - ); -} - type GeneralFilterButtonProps = { allLabels: string[]; selectedLabels: string[] | undefined; diff --git a/web/src/components/filter/SearchFilterGroup.tsx b/web/src/components/filter/SearchFilterGroup.tsx new file mode 100644 index 0000000000..e27a65a473 --- /dev/null +++ b/web/src/components/filter/SearchFilterGroup.tsx @@ -0,0 +1,458 @@ +import { Button } from "../ui/button"; +import { Popover, PopoverContent, PopoverTrigger } from "../ui/popover"; +import useSWR from "swr"; +import { FrigateConfig } from "@/types/frigateConfig"; +import { useCallback, useMemo, useState } from "react"; +import { DropdownMenuSeparator } from "../ui/dropdown-menu"; +import { getEndOfDayTimestamp } from "@/utils/dateUtil"; +import { FaFilter } from "react-icons/fa"; +import { isDesktop, isMobile } from "react-device-detect"; +import { Drawer, DrawerContent, DrawerTrigger } from "../ui/drawer"; +import { Switch } from "../ui/switch"; +import { Label } from "../ui/label"; +import MobileReviewSettingsDrawer, { + DrawerFeatures, +} from "../overlay/MobileReviewSettingsDrawer"; +import FilterSwitch from "./FilterSwitch"; +import { FilterList } from "@/types/filter"; +import { CalendarRangeFilterButton } from "./CalendarFilterButton"; +import { CamerasFilterButton } from "./CamerasFilterButton"; +import { SearchFilter } from "@/types/search"; +import { DateRange } from "react-day-picker"; + +const SEARCH_FILTERS = ["cameras", "date", "general"] as const; +type SearchFilters = (typeof SEARCH_FILTERS)[number]; +const DEFAULT_REVIEW_FILTERS: SearchFilters[] = ["cameras", "date", "general"]; + +type SearchFilterGroupProps = { + filters?: SearchFilters[]; + filter?: SearchFilter; + filterList?: FilterList; + onUpdateFilter: (filter: SearchFilter) => void; +}; + +export default function SearchFilterGroup({ + filters = DEFAULT_REVIEW_FILTERS, + filter, + filterList, + onUpdateFilter, +}: SearchFilterGroupProps) { + const { data: config } = useSWR("config"); + + const allLabels = useMemo(() => { + if (filterList?.labels) { + return filterList.labels; + } + + if (!config) { + return []; + } + + const labels = new Set(); + const cameras = filter?.cameras || Object.keys(config.cameras); + + cameras.forEach((camera) => { + if (camera == "birdseye") { + return; + } + const cameraConfig = config.cameras[camera]; + cameraConfig.objects.track.forEach((label) => { + labels.add(label); + }); + + if (cameraConfig.audio.enabled_in_config) { + cameraConfig.audio.listen.forEach((label) => { + labels.add(label); + }); + } + }); + + return [...labels].sort(); + }, [config, filterList, filter]); + + const allZones = useMemo(() => { + if (filterList?.zones) { + return filterList.zones; + } + + if (!config) { + return []; + } + + const zones = new Set(); + const cameras = filter?.cameras || Object.keys(config.cameras); + + cameras.forEach((camera) => { + if (camera == "birdseye") { + return; + } + const cameraConfig = config.cameras[camera]; + cameraConfig.review.alerts.required_zones.forEach((zone) => { + zones.add(zone); + }); + cameraConfig.review.detections.required_zones.forEach((zone) => { + zones.add(zone); + }); + }); + + return [...zones].sort(); + }, [config, filterList, filter]); + + const filterValues = useMemo( + () => ({ + cameras: Object.keys(config?.cameras || {}), + labels: Object.values(allLabels || {}), + zones: Object.values(allZones || {}), + }), + [config, allLabels, allZones], + ); + + const groups = useMemo(() => { + if (!config) { + return []; + } + + return Object.entries(config.camera_groups).sort( + (a, b) => a[1].order - b[1].order, + ); + }, [config]); + + const mobileSettingsFeatures = useMemo(() => { + const features: DrawerFeatures[] = []; + + if (filters.includes("date")) { + features.push("calendar"); + } + + if (filters.includes("general")) { + features.push("filter"); + } + + return features; + }, [filters]); + + // handle updating filters + + const onUpdateSelectedRange = useCallback( + (range?: DateRange) => { + onUpdateFilter({ + ...filter, + after: + range?.from == undefined ? undefined : range.from.getTime() / 1000, + before: + range?.to == undefined ? undefined : getEndOfDayTimestamp(range.to), + }); + }, + [filter, onUpdateFilter], + ); + + return ( +
+ {filters.includes("cameras") && ( + { + onUpdateFilter({ ...filter, cameras: newCameras }); + }} + /> + )} + {isDesktop && filters.includes("date") && ( + + )} + {isDesktop && filters.includes("general") && ( + { + onUpdateFilter({ ...filter, labels: newLabels }); + }} + updateZoneFilter={(newZones) => + onUpdateFilter({ ...filter, zones: newZones }) + } + /> + )} + {isMobile && mobileSettingsFeatures.length > 0 && ( + {}} + setRange={() => {}} + /> + )} +
+ ); +} + +type GeneralFilterButtonProps = { + allLabels: string[]; + selectedLabels: string[] | undefined; + allZones: string[]; + selectedZones?: string[]; + updateLabelFilter: (labels: string[] | undefined) => void; + updateZoneFilter: (zones: string[] | undefined) => void; +}; +function GeneralFilterButton({ + allLabels, + selectedLabels, + allZones, + selectedZones, + updateLabelFilter, + updateZoneFilter, +}: GeneralFilterButtonProps) { + const [open, setOpen] = useState(false); + const [currentLabels, setCurrentLabels] = useState( + selectedLabels, + ); + const [currentZones, setCurrentZones] = useState( + selectedZones, + ); + + const trigger = ( + + ); + const content = ( + setOpen(false)} + /> + ); + + if (isMobile) { + return ( + { + if (!open) { + setCurrentLabels(selectedLabels); + } + + setOpen(open); + }} + > + {trigger} + + {content} + + + ); + } + + return ( + { + if (!open) { + setCurrentLabels(selectedLabels); + } + + setOpen(open); + }} + > + {trigger} + {content} + + ); +} + +type GeneralFilterContentProps = { + allLabels: string[]; + selectedLabels: string[] | undefined; + currentLabels: string[] | undefined; + allZones?: string[]; + selectedZones?: string[]; + currentZones?: string[]; + updateLabelFilter: (labels: string[] | undefined) => void; + setCurrentLabels: (labels: string[] | undefined) => void; + updateZoneFilter?: (zones: string[] | undefined) => void; + setCurrentZones?: (zones: string[] | undefined) => void; + onClose: () => void; +}; +export function GeneralFilterContent({ + allLabels, + selectedLabels, + currentLabels, + allZones, + selectedZones, + currentZones, + updateLabelFilter, + setCurrentLabels, + updateZoneFilter, + setCurrentZones, + onClose, +}: GeneralFilterContentProps) { + return ( + <> +
+
+ + { + if (isChecked) { + setCurrentLabels(undefined); + } + }} + /> +
+
+ {allLabels.map((item) => ( + { + if (isChecked) { + const updatedLabels = currentLabels ? [...currentLabels] : []; + + updatedLabels.push(item); + setCurrentLabels(updatedLabels); + } else { + const updatedLabels = currentLabels ? [...currentLabels] : []; + + // can not deselect the last item + if (updatedLabels.length > 1) { + updatedLabels.splice(updatedLabels.indexOf(item), 1); + setCurrentLabels(updatedLabels); + } + } + }} + /> + ))} +
+ + {allZones && setCurrentZones && ( + <> + +
+ + { + if (isChecked) { + setCurrentZones(undefined); + } + }} + /> +
+
+ {allZones.map((item) => ( + { + if (isChecked) { + const updatedZones = currentZones + ? [...currentZones] + : []; + + updatedZones.push(item); + setCurrentZones(updatedZones); + } else { + const updatedZones = currentZones + ? [...currentZones] + : []; + + // can not deselect the last item + if (updatedZones.length > 1) { + updatedZones.splice(updatedZones.indexOf(item), 1); + setCurrentZones(updatedZones); + } + } + }} + /> + ))} +
+ + )} +
+ +
+ + +
+ + ); +} diff --git a/web/src/components/indicators/Chip.tsx b/web/src/components/indicators/Chip.tsx index e484f2f5aa..8c8d373c89 100644 --- a/web/src/components/indicators/Chip.tsx +++ b/web/src/components/indicators/Chip.tsx @@ -39,7 +39,13 @@ export default function Chip({ className, !isIOS && "z-10", )} - onClick={onClick} + onClick={(e) => { + e.stopPropagation(); + + if (onClick) { + onClick(); + } + }} > {children}
diff --git a/web/src/components/menu/GeneralSettings.tsx b/web/src/components/menu/GeneralSettings.tsx index 34637d57ef..c1c46d87bf 100644 --- a/web/src/components/menu/GeneralSettings.tsx +++ b/web/src/components/menu/GeneralSettings.tsx @@ -3,6 +3,7 @@ import { LuGithub, LuLifeBuoy, LuList, + LuLogOut, LuMoon, LuPenSquare, LuRotateCw, @@ -56,7 +57,7 @@ import { TooltipTrigger, } from "@/components/ui/tooltip"; import ActivityIndicator from "../indicators/activity-indicator"; -import { isDesktop } from "react-device-detect"; +import { isDesktop, isMobile } from "react-device-detect"; import { Drawer, DrawerContent, DrawerTrigger } from "../ui/drawer"; import { Dialog, @@ -68,11 +69,18 @@ import { import { TooltipPortal } from "@radix-ui/react-tooltip"; import { cn } from "@/lib/utils"; import { baseUrl } from "@/api/baseUrl"; +import useSWR from "swr"; type GeneralSettingsProps = { className?: string; }; export default function GeneralSettings({ className }: GeneralSettingsProps) { + const { data: profile } = useSWR("profile"); + const { data: config } = useSWR("config"); + const logoutUrl = config?.proxy?.logout_url || "/api/logout"; + + // settings + const { theme, colorScheme, setTheme, setColorScheme } = useTheme(); const [restartDialogOpen, setRestartDialogOpen] = useState(false); const [restartingSheetOpen, setRestartingSheetOpen] = useState(false); @@ -154,6 +162,28 @@ export default function GeneralSettings({ className }: GeneralSettingsProps) { } >
+ {isMobile && ( + <> + + Current User: {profile?.username || "anonymous"} + + + + + + Logout + + + + )} System diff --git a/web/src/components/navigation/Bottombar.tsx b/web/src/components/navigation/Bottombar.tsx index b6432bd6cb..d82f69c600 100644 --- a/web/src/components/navigation/Bottombar.tsx +++ b/web/src/components/navigation/Bottombar.tsx @@ -7,7 +7,6 @@ import { useFrigateStats } from "@/api/ws"; import { useContext, useEffect, useMemo } from "react"; import useStats from "@/hooks/use-stats"; import GeneralSettings from "../menu/GeneralSettings"; -import AccountSettings from "../menu/AccountSettings"; import useNavigation from "@/hooks/use-navigation"; import { StatusBarMessagesContext, @@ -35,7 +34,6 @@ function Bottombar() { ))} -
); diff --git a/web/src/components/overlay/SearchDetailDialog.tsx b/web/src/components/overlay/SearchDetailDialog.tsx new file mode 100644 index 0000000000..23580bdbf6 --- /dev/null +++ b/web/src/components/overlay/SearchDetailDialog.tsx @@ -0,0 +1,167 @@ +import { isDesktop, isIOS } from "react-device-detect"; +import { Sheet, SheetContent } from "../ui/sheet"; +import { Drawer, DrawerContent } from "../ui/drawer"; +import { SearchResult } from "@/types/search"; +import useSWR from "swr"; +import { FrigateConfig } from "@/types/frigateConfig"; +import { useFormattedTimestamp } from "@/hooks/use-date-utils"; +import { getIconForLabel } from "@/utils/iconUtil"; +import { useApiHost } from "@/api"; +import { Button } from "../ui/button"; +import { useCallback, useEffect, useState } from "react"; +import axios from "axios"; +import { toast } from "sonner"; +import { Textarea } from "../ui/textarea"; + +type SearchDetailDialogProps = { + search?: SearchResult; + setSearch: (search: SearchResult | undefined) => void; + setSimilarity?: () => void; +}; +export default function SearchDetailDialog({ + search, + setSearch, + setSimilarity, +}: SearchDetailDialogProps) { + const { data: config } = useSWR("config", { + revalidateOnFocus: false, + }); + + const apiHost = useApiHost(); + + // data + + const [desc, setDesc] = useState(search?.description); + + // we have to make sure the current selected search item stays in sync + useEffect(() => setDesc(search?.description), [search]); + + const formattedDate = useFormattedTimestamp( + search?.start_time ?? 0, + config?.ui.time_format == "24hour" + ? "%b %-d %Y, %H:%M" + : "%b %-d %Y, %I:%M %p", + ); + + // api + + const updateDescription = useCallback(() => { + if (!search) { + return; + } + + axios + .post(`events/${search.id}/description`, { description: desc }) + .then((resp) => { + if (resp.status == 200) { + toast.success("Successfully saved description", { + position: "top-center", + }); + } + }) + .catch(() => { + toast.error("Failed to update the description", { + position: "top-center", + }); + setDesc(search.description); + }); + }, [desc, search]); + + // content + + const Overlay = isDesktop ? Sheet : Drawer; + const Content = isDesktop ? SheetContent : DrawerContent; + + return ( + { + if (!open) { + setSearch(undefined); + } + }} + > + + {search && ( +
+
+
+
+
Label
+
+ {getIconForLabel(search.label, "size-4 text-white")} + {search.label} +
+
+
+
Score
+
+ {Math.round(search.score * 100)}% +
+
+
+
Camera
+
+ {search.camera.replaceAll("_", " ")} +
+
+
+
Timestamp
+
{formattedDate}
+
+
+
+ + +
+
+
+
Description
+