From 703bd6abb47f7888ebc8fde5a7ca754f5c0d8b93 Mon Sep 17 00:00:00 2001 From: Jukka Pirinen <9322054+jukkapirinen@users.noreply.github.com> Date: Fri, 15 Nov 2024 20:15:31 +0200 Subject: [PATCH 01/13] Allow underscore in device_name --- blueprints/event_summary.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index 0521c82..6279cd2 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -170,7 +170,7 @@ variables: {% set ns = namespace(device_names=[]) %} {% for device_id in notify_devices %} {% set device_name = device_attr(device_id, "name") %} - {% set sanitized_name = "mobile_app_" + device_name | lower | regex_replace("[^a-z0-9 ]", "") | replace(" ", "_") %} + {% set sanitized_name = "mobile_app_" + device_name | lower | regex_replace("[^a-z0-9_ ]", "") | replace(" ", "_") %} {% set ns.device_names = ns.device_names + [sanitized_name] %} {% endfor %} {{ ns.device_names }} @@ -388,4 +388,4 @@ action: clickAction: !input tap_navigate #Android tag: "{{tag}}" group: "{{group}}" - interruption-level: passive \ No newline at end of file + interruption-level: passive From fad0750dcae1777fde08ffe92db5c89e5baa88fe Mon Sep 17 00:00:00 2001 From: Jukka Pirinen <9322054+jukkapirinen@users.noreply.github.com> Date: Fri, 15 Nov 2024 20:26:53 +0200 Subject: [PATCH 02/13] Allow hyphen in device_name and convert to underscore --- blueprints/event_summary.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index 6279cd2..8a28143 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -170,7 +170,7 @@ variables: {% set ns = namespace(device_names=[]) %} {% for device_id in notify_devices %} {% set device_name = device_attr(device_id, "name") %} - {% set sanitized_name = "mobile_app_" + device_name | lower | regex_replace("[^a-z0-9_ ]", "") | replace(" ", "_") %} + {% set sanitized_name = "mobile_app_" + device_name | lower | regex_replace("[^a-z0-9_- ]", "") | replace(" ", "_") | replace("-", "_") %} {% set ns.device_names = ns.device_names + [sanitized_name] %} {% endfor %} {{ ns.device_names }} From 9d7d4eb33bebb97d7c65255d3a649190666101dc Mon Sep 17 00:00:00 2001 From: Jukka Pirinen <9322054+jukkapirinen@users.noreply.github.com> Date: Fri, 15 Nov 2024 20:34:20 +0200 Subject: [PATCH 03/13] Escape hyphen in device_name regexp --- blueprints/event_summary.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index 8a28143..d204a21 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -170,7 +170,7 @@ variables: {% set ns = namespace(device_names=[]) %} {% for device_id in notify_devices %} {% set device_name = device_attr(device_id, "name") %} - {% set sanitized_name = "mobile_app_" + device_name | lower | regex_replace("[^a-z0-9_- ]", "") | replace(" ", "_") | replace("-", "_") %} + {% set sanitized_name = "mobile_app_" + device_name | lower | regex_replace("[^a-z0-9_\- ]", "") | replace(" ", "_") | replace("-", "_") %} {% set ns.device_names = ns.device_names + [sanitized_name] %} {% endfor %} {{ ns.device_names }} From 8350a4cd1c2c699c7f8fdc034018ef9bfe230f73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Fr=C3=B6hlich?= <85313672+valentinfrlch@users.noreply.github.com> Date: Mon, 25 Nov 2024 20:15:26 +0100 Subject: [PATCH 04/13] Update roadmap --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5437d18..f910132 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ logger: > These are planned features and ideas. They are subject to change and may not be implemented in the order listed or at all. 1. **HACS**: Include in HACS default repository +2. **Providers**: Support for Azure For features added in previous versions, check the changelogs in the release notes. From 0db6c45bf9fd34c9f22caf0491fb36a040e3ee33 Mon Sep 17 00:00:00 2001 From: Ryan Steckler Date: Mon, 9 Dec 2024 20:00:58 -0800 Subject: [PATCH 05/13] Configurable retries on fetching Frigate clips --- custom_components/llmvision/__init__.py | 8 ++++++- custom_components/llmvision/const.py | 2 ++ custom_components/llmvision/media_handlers.py | 6 ++--- custom_components/llmvision/services.yaml | 24 +++++++++++++++++++ 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/custom_components/llmvision/__init__.py b/custom_components/llmvision/__init__.py index 3d94311..5fb9c01 100644 --- a/custom_components/llmvision/__init__.py +++ b/custom_components/llmvision/__init__.py @@ -24,6 +24,8 @@ IMAGE_ENTITY, VIDEO_FILE, EVENT_ID, + FRIGATE_RETRY_ATTEMPTS, + FRIGATE_RETRY_SECONDS, INTERVAL, DURATION, MAX_FRAMES, @@ -227,6 +229,8 @@ def __init__(self, data_call): "\n") if data_call.data.get(EVENT_ID) else None self.interval = int(data_call.data.get(INTERVAL, 2)) self.duration = int(data_call.data.get(DURATION, 10)) + self.frigate_retry_attempts = int(data_call.data.get(FRIGATE_RETRY_ATTEMPTS, 2)) + self.frigate_retry_seconds = int(data_call.data.get(FRIGATE_RETRY_SECONDS, 1)) self.max_frames = int(data_call.data.get(MAX_FRAMES, 3)) self.target_width = data_call.data.get(TARGET_WIDTH, 3840) self.temperature = float(data_call.data.get(TEMPERATURE, 0.3)) @@ -285,7 +289,9 @@ async def video_analyzer(data_call): max_frames=call.max_frames, target_width=call.target_width, include_filename=call.include_filename, - expose_images=call.expose_images + expose_images=call.expose_images, + frigate_retry_attempts=call.frigate_retry_attempts, + frigate_retry_seconds=call.frigate_retry_seconds ) response = await client.make_request(call) await _remember(hass, call, start, response) diff --git a/custom_components/llmvision/const.py b/custom_components/llmvision/const.py index 4bbd7f8..b554ff9 100644 --- a/custom_components/llmvision/const.py +++ b/custom_components/llmvision/const.py @@ -31,6 +31,8 @@ EVENT_ID = 'event_id' INTERVAL = 'interval' DURATION = 'duration' +FRIGATE_RETRY_ATTEMPTS = 'frigate_retry_attempts' +FRIGATE_RETRY_SECONDS = 'frigate_retry_seconds' MAX_FRAMES = 'max_frames' DETAIL = 'detail' TEMPERATURE = 'temperature' diff --git a/custom_components/llmvision/media_handlers.py b/custom_components/llmvision/media_handlers.py index f37ab8c..69a03ab 100644 --- a/custom_components/llmvision/media_handlers.py +++ b/custom_components/llmvision/media_handlers.py @@ -294,7 +294,7 @@ async def add_images(self, image_entities, image_paths, target_width, include_fi raise ServiceValidationError(f"Error: {e}") return self.client - async def add_videos(self, video_paths, event_ids, max_frames, target_width, include_filename, expose_images): + async def add_videos(self, video_paths, event_ids, max_frames, target_width, include_filename, expose_images, frigate_retry_attempts, frigate_retry_seconds): """Wrapper for client.add_frame for videos""" tmp_clips_dir = f"/config/custom_components/{DOMAIN}/tmp_clips" tmp_frames_dir = f"/config/custom_components/{DOMAIN}/tmp_frames" @@ -306,8 +306,8 @@ async def add_videos(self, video_paths, event_ids, max_frames, target_width, inc try: base_url = get_url(self.hass) frigate_url = base_url + "/api/frigate/notifications/" + event_id + "/clip.mp4" - clip_data = await self.client._fetch(frigate_url) - + clip_data = await self.client._fetch(frigate_url, max_retries=frigate_retry_attempts, retry_delay=frigate_retry_seconds) + if not clip_data: raise ServiceValidationError( f"Failed to fetch frigate clip {event_id}") diff --git a/custom_components/llmvision/services.yaml b/custom_components/llmvision/services.yaml index 78ad818..d40bfec 100644 --- a/custom_components/llmvision/services.yaml +++ b/custom_components/llmvision/services.yaml @@ -160,6 +160,30 @@ video_analyzer: selector: text: multiline: true + frigate_retry_attempts: + name: Frigate Retry Attempts + description: How many times to retry fetching the video clip from Frigate. Clips are not always available from Frigate as soon as the event has ended. + Slower machines or longer clips may need additional attempts. Increase this if you see errors fetching the clips from Frigate in your automation traces. + required: false + example: 2 + default: 2 + selector: + number: + min: 1 + max: 10 + step: 1 + frigate_retry_seconds: + name: Frigate Retry Seconds + description: How long to wait between retries to fetch the video clip from Frigate. Clips are not always available from Frigate as soon as the event has ended. + Slower machines or longer clips may need additional attempts. Increase this if you see errors fetching the clips from Frigate in your automation traces. + required: false + example: 1 + default: 1 + selector: + number: + min: 1 + max: 10 + step: 1 max_frames: name: Max Frames description: How many frames to analyze. Picks frames with the most movement. From 0f4d94fbbc7ad0f63d98491e024390a5aaf4cd0a Mon Sep 17 00:00:00 2001 From: Ryan Steckler Date: Mon, 9 Dec 2024 21:48:08 -0800 Subject: [PATCH 06/13] fixed ffmpeg command --- custom_components/llmvision/media_handlers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/custom_components/llmvision/media_handlers.py b/custom_components/llmvision/media_handlers.py index f37ab8c..9cab83f 100644 --- a/custom_components/llmvision/media_handlers.py +++ b/custom_components/llmvision/media_handlers.py @@ -347,8 +347,9 @@ async def add_videos(self, video_paths, event_ids, max_frames, target_width, inc ffmpeg_cmd = [ "ffmpeg", "-i", video_path, - "-vf", f"fps=1/{interval},select='eq(n\\,0)+not(mod(n\\,{interval}))'", os.path.join( - tmp_frames_dir, "frame%04d.jpg") + "-vf", f"fps=fps='source_fps',select='eq(n\\,0)+not(mod(n\\,{interval}))'", + "-fps_mode", "passthrough", + os.path.join(tmp_frames_dir, "frame%04d.jpg") ] # Run ffmpeg command await self.hass.loop.run_in_executor(None, os.system, " ".join(ffmpeg_cmd)) From 297c453a0f2bc317aa196da4bee8a6e7174019d2 Mon Sep 17 00:00:00 2001 From: Michael Rappazzo Date: Tue, 10 Dec 2024 13:06:34 -0500 Subject: [PATCH 07/13] event_summary blueprint: adjust the device name sanitization The device name sanitize code was changed from "remove all non-conforming characters and then convert some special ones to '_'" to "convert some special characters to '_' and then remove all other non-conforming characters. In the new version, both of these operations were changed to use regex replace only. Note that in the "special characters" expression `[' -]` the '-' does not need to be escaped because it is the last item in the regex character class. The "non-conforming characters" regex was changed to `[^a-z0-9_]`, which reflects that the special characters no longer need to be included. An important addition to the "special characters" is the single-quote, which was previously omitted. --- blueprints/event_summary.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index d204a21..ac4915e 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -170,7 +170,7 @@ variables: {% set ns = namespace(device_names=[]) %} {% for device_id in notify_devices %} {% set device_name = device_attr(device_id, "name") %} - {% set sanitized_name = "mobile_app_" + device_name | lower | regex_replace("[^a-z0-9_\- ]", "") | replace(" ", "_") | replace("-", "_") %} + {% set sanitized_name = "mobile_app_" + device_name | lower | regex_replace("[' -]", "_") | regex_replace("[^a-z0-9_]", "") %} {% set ns.device_names = ns.device_names + [sanitized_name] %} {% endfor %} {{ ns.device_names }} From 476820c54e8a1f8ab8358c8d5d7c22bf06d9e6b4 Mon Sep 17 00:00:00 2001 From: Michael Rappazzo Date: Mon, 16 Dec 2024 15:19:38 -0500 Subject: [PATCH 08/13] blueprint: re-implement cooldown with singlemode execution and a delay Force the automation to use single mode with suppressed warnings (see https://www.home-assistant.io/docs/automation/modes/). To implement the cooldown, add a delay as the last action in the actions. Also remove the cooldown from the automation conditions. The main difference here is that no new triggers will arrive while the automation is running. The condition is smaller now, and is limited to the content of the automation, and not managing how the automation is run. With this narrowing, it opens up the possibility of adding other conditions to the blueprint config in the future. --- blueprints/event_summary.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index ac4915e..faabe94 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -230,6 +230,10 @@ variables: Use "critical" only for possible burglaries and similar events. "time-sensitive" could be a courier at the front door or an event of similar importance. Reply with these replies exactly. +max_exceeded: silent + +mode: single + trigger: - platform: mqtt topic: "frigate/events" @@ -247,9 +251,7 @@ condition: - condition: template value_template: > {% if mode == 'Frigate' %} - {{ trigger.payload_json["type"] == "end" and (state_attr(this.entity_id, 'last_triggered') is none or (now() - state_attr(this.entity_id, 'last_triggered')).total_seconds() / 60 > cooldown) and ('camera.' + trigger.payload_json['after']['camera']|lower) in camera_entities_list }} - {% else %} - {{ state_attr(this.entity_id, 'last_triggered') is none or (now() - state_attr(this.entity_id, 'last_triggered')).total_seconds() / 60 > cooldown }} + {{ trigger.payload_json["type"] == "end" and ('camera.' + trigger.payload_json['after']['camera']|lower) in camera_entities_list }} {% endif %} @@ -389,3 +391,5 @@ action: tag: "{{tag}}" group: "{{group}}" interruption-level: passive + +- delay: 00:{{cooldown|int}}:00 From 1f14fea14ddc7a34a23abb52130677bd6f3c562e Mon Sep 17 00:00:00 2001 From: Michael Rappazzo Date: Sat, 21 Dec 2024 09:29:23 -0500 Subject: [PATCH 09/13] blueprint: reformat the variable descriptions Include a newline in the description for variables which describe their applicability for frigate mode, camera mode, or both. Also, add a suggestion for the 'tap naviagation' to link directly to the input video or image. --- blueprints/event_summary.yaml | 42 ++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index ac4915e..35d7088 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -2,7 +2,7 @@ blueprint: name: AI Event Summary (LLM Vision v1.3.1) author: valentinfrlch description: > - AI-powered security event summaries for frigate or camera entities. + AI-powered security event summaries for frigate or camera entities. Sends a notification with a preview to your phone that is updated dynamically when the AI summary is available. domain: automation source_url: https://github.com/valentinfrlch/ha-llmvision/blob/main/blueprints/event_summary.yaml @@ -42,7 +42,10 @@ blueprint: integration: mobile_app camera_entities: name: Camera Entities - description: (Camera and Frigate mode) List of camera entities to monitor + description: >- + (Camera and Frigate mode) + + List of camera entities to monitor default: [] selector: entity: @@ -51,14 +54,20 @@ blueprint: domain: camera trigger_state: name: Trigger State - description: (Camera mode only) Trigger the automation when your cameras change to this state. + description: >- + (Camera mode only) + + Trigger the automation when your cameras change to this state. default: 'recording' selector: text: multiline: false motion_sensors: name: Motion Sensor - description: (Camera mode only) Set if your cameras don't change state. Use the same order used for camera entities. + description: >- + (Camera mode only) + + Set if your cameras don't change state. Use the same order used for camera entities. default: [] selector: entity: @@ -67,7 +76,10 @@ blueprint: domain: binary_sensor preview_mode: name: Preview Mode - description: (Camera mode only) Choose between a live preview or a snapshot of the event + description: >- + (Camera mode only) + + Choose between a live preview or a snapshot of the event default: 'Live Preview' selector: select: @@ -84,14 +96,21 @@ blueprint: max: 60 tap_navigate: name: Tap Navigate - description: Path to navigate to when notification is opened (e.g. /lovelace/cameras) + description: >- + Path to navigate to when notification is opened (e.g. /lovelace/cameras). + + To have use the same input which was sent to the ai engine, use + `{{video if video != '''' else image}}` default: "/lovelace/0" selector: text: multiline: false duration: name: Duration - description: (Camera mode only) How long to record before analyzing (in seconds) + description: >- + (Camera mode only) + + How long to record before analyzing (in seconds) default: 5 selector: number: @@ -99,7 +118,10 @@ blueprint: max: 60 max_frames: name: Max Frames - description: (Camera and Frigate mode) How many frames to analyze. Picks frames with the most movement. + description: >- + (Camera and Frigate mode) + + How many frames to analyze. Picks frames with the most movement. default: 3 selector: number: @@ -293,7 +315,7 @@ action: max_tokens: 3 temperature: 0.1 response_variable: importance - + # Cancel automation if event not deemed important - choose: - conditions: @@ -365,7 +387,7 @@ action: temperature: !input temperature expose_images: "{{true if preview_mode == 'Snapshot'}}" response_variable: response - + - choose: - conditions: From 6d754f2a27a68a3842e3d92dd91ed106c28da335 Mon Sep 17 00:00:00 2001 From: Michael Rappazzo Date: Sat, 21 Dec 2024 08:12:30 -0500 Subject: [PATCH 10/13] blueprint: add a filter for frigate object detection The frigate payload includes a label which indicates what the detected object is. This can be useful to separate llm-vision automations to react differently for different objects. For example, if a bird is detected, the ai instructions could include something like, "if you see a bird, try to identify what species it is". This discription doesn't make much sense if the detected object is a person, and including separate instructions to the ai makes it more confusing on both the human and ai side of things. Allowing for separate instructions by object will make for easier to understand automations. --- blueprints/event_summary.yaml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index faabe94..128fa9f 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -49,6 +49,17 @@ blueprint: multiple: true filter: domain: camera + object_type: + name: Included Object Type(s) + description: >- + (Frigate mode only) + + Only run if frigate labels the object as one of these. (person, dog, bird, etc) + default: [] + selector: + text: + multiline: false + multiple: true trigger_state: name: Trigger State description: (Camera mode only) Trigger the automation when your cameras change to this state. @@ -175,6 +186,7 @@ variables: {% endfor %} {{ ns.device_names }} camera_entities_list: !input camera_entities + object_types_list: !input object_type motion_sensors_list: !input motion_sensors camera_entity: > {% if mode == 'Camera' %} @@ -251,7 +263,10 @@ condition: - condition: template value_template: > {% if mode == 'Frigate' %} - {{ trigger.payload_json["type"] == "end" and ('camera.' + trigger.payload_json['after']['camera']|lower) in camera_entities_list }} + {{ trigger.payload_json["type"] == "end" + and ('camera.' + trigger.payload_json['after']['camera']|lower) in camera_entities_list + and ((object_types_list|length) == 0 or ((trigger.payload_json['after']['label']|lower) in object_types_list)) + }} {% endif %} From 25084c14b13953438480a59cf79c905d8d342f35 Mon Sep 17 00:00:00 2001 From: Moz <97829410+SleepyMoz@users.noreply.github.com> Date: Mon, 23 Dec 2024 06:54:26 +0000 Subject: [PATCH 11/13] Fixed a bug with importing to HA --- blueprints/event_summary.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index e2e9b9c..380bccd 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -429,4 +429,4 @@ action: group: "{{group}}" interruption-level: passive -- delay: 00:{{cooldown|int}}:00 +- delay: '00:{{cooldown|int}}:00' From 2299060e049b9772324015fe0d097a3ae292a3b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Fr=C3=B6hlich?= <85313672+valentinfrlch@users.noreply.github.com> Date: Mon, 23 Dec 2024 08:33:10 +0100 Subject: [PATCH 12/13] Update event_summary.yaml Added two spaces in front of "- delay" --- blueprints/event_summary.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index 380bccd..b422e13 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -429,4 +429,4 @@ action: group: "{{group}}" interruption-level: passive -- delay: '00:{{cooldown|int}}:00' + - delay: '00:{{cooldown|int}}:00' From d7335e04a3da96e11cab61a1aaf45a9dbc4f47e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Fr=C3=B6hlich?= <85313672+valentinfrlch@users.noreply.github.com> Date: Mon, 23 Dec 2024 14:51:57 +0100 Subject: [PATCH 13/13] (Blueprint) Camera mode hotfix Fixes #141 --- blueprints/event_summary.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index b422e13..ea5304e 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -289,6 +289,8 @@ condition: and ('camera.' + trigger.payload_json['after']['camera']|lower) in camera_entities_list and ((object_types_list|length) == 0 or ((trigger.payload_json['after']['label']|lower) in object_types_list)) }} + {%else%} + true {% endif %}