Semantic Search for Events

.github/workflows/pull_request.yml

@@ -40,9 +40,9 @@ jobs:
           node-version: 16.x
       - run: npm install
         working-directory: ./web
-      - name: Lint
-        run: npm run lint
-        working-directory: ./web
+      # - name: Lint
+      #   run: npm run lint
+      #   working-directory: ./web
 
   web_test:
     name: Web - Test
@@ -51,12 +51,12 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@master
         with:
-          node-version: 16.x
+          node-version: 20.x
       - run: npm install
         working-directory: ./web
-      - name: Test
-        run: npm run test
-        working-directory: ./web
+      # - name: Test
+      #   run: npm run test
+      #   working-directory: ./web
 
   python_checks:
     runs-on: ubuntu-latest

.gitignore

@@ -8,13 +8,15 @@ config/*
 !config/*.example
 models
 *.mp4
-*.ts
 *.db
 *.csv
 frigate/version.py
 web/build
 web/node_modules
 web/coverage
+web-new/build
+web-new/node_modules
+web-new/coverage
 core
 !/web/**/*.ts
 .idea/*

docker/main/Dockerfile

@@ -8,7 +8,7 @@ ARG SLIM_BASE=debian:11-slim
 
 FROM ${BASE_IMAGE} AS base
 
-FROM --platform=${BUILDPLATFORM} debian:11 AS base_host
+FROM --platform=${BUILDPLATFORM} ${BASE_IMAGE} AS base_host
 
 FROM ${SLIM_BASE} AS slim-base
 
@@ -176,6 +176,13 @@ ARG APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES="compute,video,utility"
 
+# Turn off Chroma Telemetry: https://docs.trychroma.com/telemetry#opting-out
+ENV ANONYMIZED_TELEMETRY=False
+# Allow resetting the chroma database
+ENV ALLOW_RESET=True
+# Disable tokenizer parallelism warning
+ENV TOKENIZERS_PARALLELISM=true
+
 ENV PATH="/usr/lib/btbn-ffmpeg/bin:/usr/local/go2rtc/bin:/usr/local/nginx/sbin:${PATH}"
 
 # Install dependencies
@@ -237,13 +244,13 @@ CMD ["sleep", "infinity"]
 
 # Frigate web build
 # This should be architecture agnostic, so speed up the build on multiarch by not using QEMU.
-FROM --platform=$BUILDPLATFORM node:16 AS web-build
+FROM --platform=$BUILDPLATFORM node:20 AS web-build
 
 WORKDIR /work
-COPY web/package.json web/package-lock.json ./
+COPY web-old/package.json web-old/package-lock.json ./
 RUN npm install
 
-COPY web/ ./
+COPY web-old/ ./
 RUN npm run build \
     && mv dist/BASE_PATH/monacoeditorwork/* dist/assets/ \
     && rm -rf dist/BASE_PATH

docker/main/build_nginx.sh

@@ -5,7 +5,6 @@ set -euxo pipefail
 NGINX_VERSION="1.25.3"
 VOD_MODULE_VERSION="1.31"
 SECURE_TOKEN_MODULE_VERSION="1.5"
-RTMP_MODULE_VERSION="1.2.2"
 
 cp /etc/apt/sources.list /etc/apt/sources.list.d/sources-src.list
 sed -i 's|deb http|deb-src http|g' /etc/apt/sources.list.d/sources-src.list
@@ -49,10 +48,6 @@ mkdir /tmp/nginx-secure-token-module
 wget https://github.com/kaltura/nginx-secure-token-module/archive/refs/tags/${SECURE_TOKEN_MODULE_VERSION}.tar.gz
 tar -zxf ${SECURE_TOKEN_MODULE_VERSION}.tar.gz -C /tmp/nginx-secure-token-module --strip-components=1
 rm ${SECURE_TOKEN_MODULE_VERSION}.tar.gz
-mkdir /tmp/nginx-rtmp-module
-wget -nv https://github.com/arut/nginx-rtmp-module/archive/refs/tags/v${RTMP_MODULE_VERSION}.tar.gz
-tar -zxf v${RTMP_MODULE_VERSION}.tar.gz -C /tmp/nginx-rtmp-module --strip-components=1
-rm v${RTMP_MODULE_VERSION}.tar.gz
 
 cd /tmp/nginx
 
@@ -63,7 +58,6 @@ cd /tmp/nginx
     --with-threads \
     --add-module=../nginx-vod-module \
     --add-module=../nginx-secure-token-module \
-    --add-module=../nginx-rtmp-module \
     --with-cc-opt="-O3 -Wno-error=implicit-fallthrough"
 
 make CC="ccache gcc" -j$(nproc) && make install

docker/main/requirements-wheels.txt

@@ -27,3 +27,9 @@ unidecode == 1.3.*
 # Openvino Library - Custom built with MYRIAD support
 openvino @ https://github.com/NateMeyer/openvino-wheels/releases/download/multi-arch_2022.3.1/openvino-2022.3.1-1-cp39-cp39-manylinux_2_31_x86_64.whl; platform_machine == 'x86_64'
 openvino @ https://github.com/NateMeyer/openvino-wheels/releases/download/multi-arch_2022.3.1/openvino-2022.3.1-1-cp39-cp39-linux_aarch64.whl; platform_machine == 'aarch64'
+# Embeddings
+onnxruntime == 1.16.*
+onnx_clip == 4.0.*
+pysqlite3-binary == 0.5.2
+chromadb == 0.4.20
+google-generativeai == 0.3.*

docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/consumer-for

@@ -0,0 +1 @@
+chroma

docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/pipeline-name

@@ -0,0 +1 @@
+chroma-pipeline

docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/run

@@ -0,0 +1,4 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+
+exec logutil-service /dev/shm/logs/chroma

docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/type

@@ -0,0 +1 @@
+longrun

docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/finish

@@ -0,0 +1,28 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+# Take down the S6 supervision tree when the service exits
+
+set -o errexit -o nounset -o pipefail
+
+# Logs should be sent to stdout so that s6 can collect them
+
+declare exit_code_container
+exit_code_container=$(cat /run/s6-linux-init-container-results/exitcode)
+readonly exit_code_container
+readonly exit_code_service="${1}"
+readonly exit_code_signal="${2}"
+readonly service="ChromaDB"
+
+echo "[INFO] Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})"
+
+if [[ "${exit_code_service}" -eq 256 ]]; then
+  if [[ "${exit_code_container}" -eq 0 ]]; then
+    echo $((128 + exit_code_signal)) >/run/s6-linux-init-container-results/exitcode
+  fi
+elif [[ "${exit_code_service}" -ne 0 ]]; then
+  if [[ "${exit_code_container}" -eq 0 ]]; then
+    echo "${exit_code_service}" >/run/s6-linux-init-container-results/exitcode
+  fi
+fi
+
+exec /run/s6/basedir/bin/halt

docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/producer-for

@@ -0,0 +1 @@
+chroma-log

docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/run

@@ -0,0 +1,16 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+# Start the Frigate service
+
+set -o errexit -o nounset -o pipefail
+
+# Logs should be sent to stdout so that s6 can collect them
+
+# Tell S6-Overlay not to restart this service
+s6-svc -O .
+
+echo "[INFO] Starting ChromaDB..."
+
+# Replace the bash process with the Frigate process, redirecting stderr to stdout
+exec 2>&1
+exec /usr/local/chroma run --path /config/chroma --host 0.0.0.0

docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/timeout-kill

@@ -0,0 +1 @@
+120000

docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/type

@@ -0,0 +1 @@
+longrun

docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run

@@ -4,7 +4,7 @@
 
 set -o errexit -o nounset -o pipefail
 
-dirs=(/dev/shm/logs/frigate /dev/shm/logs/go2rtc /dev/shm/logs/nginx)
+dirs=(/dev/shm/logs/frigate /dev/shm/logs/go2rtc /dev/shm/logs/nginx /dev/shm/logs/chroma)
 
 mkdir -p "${dirs[@]}"
 chown nobody:nogroup "${dirs[@]}"

docker/main/rootfs/usr/local/chroma

@@ -0,0 +1,14 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-s
+__import__("pysqlite3")
+
+import re
+import sys
+
+sys.modules["sqlite3"] = sys.modules.pop("pysqlite3")
+
+from chromadb.cli.cli import app
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
+    sys.exit(app())

docker/main/rootfs/usr/local/nginx/conf/nginx.conf

@@ -275,18 +275,3 @@ http {
         }
     }
 }
-
-rtmp {
-    server {
-        listen 1935;
-        chunk_size 4096;
-        allow publish 127.0.0.1;
-        deny publish all;
-        allow play all;
-        application live {
-            live on;
-            record off;
-            meta copy;
-        }
-    }
-}

docs/docs/configuration/camera_specific.md

@@ -69,16 +69,12 @@ cameras:
     ffmpeg:
       output_args:
         record: -f segment -segment_time 10 -segment_format mp4 -reset_timestamps 1 -strftime 1 -c:v copy -tag:v hvc1 -bsf:v hevc_mp4toannexb -c:a aac
-        rtmp: -c:v copy -c:a aac -f flv
 
       inputs:
         - path: rtsp://user:password@camera-ip:554/H264/ch1/main/av_stream # <----- Update for your camera
           roles:
             - detect
             - record
-            - rtmp
-    rtmp:
-      enabled: False # <-- RTMP should be disabled if your stream is not H264
     detect:
       width: # <- optional, by default Frigate tries to automatically detect resolution
       height: # <- optional, by default Frigate tries to automatically detect resolution
@@ -156,13 +152,12 @@ go2rtc:
 
 [See the go2rtc docs for more information](https://github.com/AlexxIT/go2rtc/tree/v1.8.4#source-rtsp)
 
-In the Unifi 2.0 update Unifi Protect Cameras had a change in audio sample rate which causes issues for ffmpeg. The input rate needs to be set for record and rtmp if used directly with unifi protect.
+In the Unifi 2.0 update Unifi Protect Cameras had a change in audio sample rate which causes issues for ffmpeg. The input rate needs to be set for record if used directly with unifi protect.
 
 ```yaml
 ffmpeg:
   output_args:
     record: preset-record-ubiquiti
-    rtmp: preset-rtmp-ubiquiti # recommend using go2rtc instead
 ```
 
 ### TP-Link VIGI Cameras

docs/docs/configuration/cameras.md

@@ -16,7 +16,6 @@ Each role can only be assigned to one input per camera. The options for roles ar
 | `detect` | Main feed for object detection. [docs](object_detectors.md)                              |
 | `record` | Saves segments of the video feed based on configuration settings. [docs](record.md)      |
 | `audio`  | Feed for audio based detection. [docs](audio_detectors.md)                               |
-| `rtmp`   | Deprecated: Broadcast as an RTMP feed for other services to consume. [docs](restream.md) |
 
 ```yaml
 mqtt:
@@ -29,7 +28,6 @@ cameras:
         - path: rtsp://viewer:{FRIGATE_RTSP_PASSWORD}@10.0.10.10:554/cam/realmonitor?channel=1&subtype=2
           roles:
             - detect
-            - rtmp # <- deprecated, recommend using restream instead
         - path: rtsp://viewer:{FRIGATE_RTSP_PASSWORD}@10.0.10.10:554/live
           roles:
             - record

docs/docs/configuration/index.md

@@ -206,8 +206,6 @@ ffmpeg:
     detect: -threads 2 -f rawvideo -pix_fmt yuv420p
     # Optional: output args for record streams (default: shown below)
     record: preset-record-generic
-    # Optional: output args for rtmp streams (default: shown below)
-    rtmp: preset-rtmp-generic
   # Optional: Time in seconds to wait before ffmpeg retries connecting to the camera. (default: shown below)
   # If set too low, frigate will retry a connection to the camera's stream too frequently, using up the limited streams some cameras can allow at once
   # If set too high, then if a ffmpeg crash or camera stream timeout occurs, you could potentially lose up to a maximum of retry_interval second(s) of footage
@@ -369,6 +367,11 @@ record:
     # The -r (framerate) dictates how smooth the output video is.
     # So the args would be -vf setpts=0.02*PTS -r 30 in that case.
     timelapse_args: "-vf setpts=0.04*PTS -r 30"
+  # Optional: Recording Preview Settings
+  preview:
+    # Optional: Quality of recording preview (default: shown below).
+    # Options are: very_low, low, medium, high, very_high
+    quality: medium
   # Optional: Event recording settings
   events:
     # Optional: Number of seconds before the event to include (default: shown below)
@@ -425,13 +428,6 @@ snapshots:
   # Optional: quality of the encoded jpeg, 0-100 (default: shown below)
   quality: 70
 
-# Optional: RTMP configuration
-# NOTE: RTMP is deprecated in favor of restream
-# NOTE: Can be overridden at the camera level
-rtmp:
-  # Optional: Enable the RTMP stream (default: False)
-  enabled: False
-
 # Optional: Restream configuration
 # Uses https://github.com/AlexxIT/go2rtc (v1.8.3)
 go2rtc:
@@ -488,14 +484,13 @@ cameras:
         # Required: the path to the stream
         # NOTE: path may include environment variables or docker secrets, which must begin with 'FRIGATE_' and be referenced in {}
         - path: rtsp://viewer:{FRIGATE_RTSP_PASSWORD}@10.0.10.10:554/cam/realmonitor?channel=1&subtype=2
-          # Required: list of roles for this stream. valid values are: audio,detect,record,rtmp
-          # NOTICE: In addition to assigning the audio, record, and rtmp roles,
+          # Required: list of roles for this stream. valid values are: audio,detect,record
+          # NOTICE: In addition to assigning the audio and record roles,
           # they must also be enabled in the camera config.
           roles:
             - audio
             - detect
             - record
-            - rtmp
           # Optional: stream specific global args (default: inherit)
           # global_args:
           # Optional: stream specific hwaccel args (default: inherit)

docs/docs/configuration/restream.md

@@ -38,10 +38,6 @@ go2rtc:
 
 **NOTE:** This does not apply to localhost requests, there is no need to provide credentials when using the restream as a source for frigate cameras.
 
-## RTMP (Deprecated)
-
-In previous Frigate versions RTMP was used for re-streaming. RTMP has disadvantages however including being incompatible with H.265, high bitrates, and certain audio codecs. RTMP is deprecated and it is recommended to move to the new restream role.
-
 ## Reduce Connections To Camera
 
 Some cameras only support one active connection or you may just want to have a single connection open to the camera. The RTSP restream allows this to be possible.

docs/docs/guides/configuring_go2rtc.md

@@ -9,7 +9,7 @@ Use of the bundled go2rtc is optional. You can still configure FFmpeg to connect
 
 - WebRTC or MSE for live viewing with higher resolutions and frame rates than the jsmpeg stream which is limited to the detect stream
 - Live stream support for cameras in Home Assistant Integration
-- RTSP (instead of RTMP) relay for use with other consumers to reduce the number of connections to your camera streams
+- RTSP relay for use with other consumers to reduce the number of connections to your camera streams
 
 # Setup a go2rtc stream
 

docs/docs/integrations/api.md

@@ -174,6 +174,8 @@ Events from the database. Accepts the following query string parameters:
 | `is_submitted`       | int   | Filter events that are submitted to Frigate+ (0 or 1) |
 | `min_length`         | float | Minimum length of the event                           |
 | `max_length`         | float | Maximum length of the event                           |
+| `search`             | str   | Search query for semantic search                      |
+| `like`               | str   | Event ID for thumbnail similarity search              |
 
 ### `GET /api/timeline`
 
@@ -225,7 +227,17 @@ Sub labels must be 100 characters or shorter.
 ```json
 {
   "subLabel": "some_string",
-  "subLabelScore": 0.79,
+  "subLabelScore": 0.79
+}
+```
+
+### `POST /api/events/<id>/description`
+
+Set a description for an event to add details about what is occurring. Used during semantic search.
+
+```json
+{
+  "description": "This is a black and white dog walking on the sidewalk."
 }
 ```
 
@@ -298,7 +310,7 @@ It is also possible to export this recording as a timelapse.
 
 ```json
 {
-  "playback": "realtime", // playback factor: realtime or timelapse_25x
+  "playback": "realtime" // playback factor: realtime or timelapse_25x
 }
 ```
 

docs/docs/integrations/home-assistant.md

@@ -124,10 +124,6 @@ https://HA_URL/api/frigate/notifications/<event-id>/clip.mp4
 
 <a name="streams"></a>
 
-## RTMP stream
-
-RTMP is deprecated and it is recommended to switch to use RTSP restreams.
-
 ## RTSP stream
 
 In order for the live streams to function they need to be accessible on the RTSP