diff --git a/docs/docs/frigate/installation.md b/docs/docs/frigate/installation.md index 98f55f229e..200b48e60a 100644 --- a/docs/docs/frigate/installation.md +++ b/docs/docs/frigate/installation.md @@ -73,23 +73,23 @@ Users of the Snapcraft build of Docker cannot use storage locations outside your Frigate utilizes shared memory to store frames during processing. The default `shm-size` provided by Docker is **64MB**. -The default shm size of **64MB** is fine for setups with **2 cameras** detecting at **720p**. If Frigate is exiting with "Bus error" messages, it is likely because you have too many high resolution cameras and you need to specify a higher shm size, using [`--shm-size`](https://docs.docker.com/engine/reference/run/#runtime-constraints-on-resources) (or [`service.shm_size`](https://docs.docker.com/compose/compose-file/compose-file-v2/#shm_size) in docker-compose). +The default shm size of **128MB** is fine for setups with **2 cameras** detecting at **720p**. If Frigate is exiting with "Bus error" messages, it is likely because you have too many high resolution cameras and you need to specify a higher shm size, using [`--shm-size`](https://docs.docker.com/engine/reference/run/#runtime-constraints-on-resources) (or [`service.shm_size`](https://docs.docker.com/compose/compose-file/compose-file-v2/#shm_size) in docker-compose). -The Frigate container also stores logs in shm, which can take up to **30MB**, so make sure to take this into account in your math as well. +The Frigate container also stores logs in shm, which can take up to **40MB**, so make sure to take this into account in your math as well. -You can calculate the necessary shm size for each camera with the following formula using the resolution specified for detect: +You can calculate the **minimum** shm size for each camera with the following formula using the resolution specified for detect: ```console # Replace and -$ python -c 'print("{:.2f}MB".format(( * * 1.5 * 9 + 270480) / 1048576))' +$ python -c 'print("{:.2f}MB".format(( * * 1.5 * 10 + 270480) / 1048576))' # Example for 1280x720 -$ python -c 'print("{:.2f}MB".format((1280 * 720 * 1.5 * 9 + 270480) / 1048576))' -12.12MB +$ python -c 'print("{:.2f}MB".format((1280 * 720 * 1.5 * 10 + 270480) / 1048576))' +13.44MB # Example for eight cameras detecting at 1280x720, including logs -$ python -c 'print("{:.2f}MB".format(((1280 * 720 * 1.5 * 9 + 270480) / 1048576) * 8 + 30))' -126.99MB +$ python -c 'print("{:.2f}MB".format(((1280 * 720 * 1.5 * 10 + 270480) / 1048576) * 8 + 40))' +136.99MB ``` The shm size cannot be set per container for Home Assistant add-ons. However, this is probably not required since by default Home Assistant Supervisor allocates `/dev/shm` with half the size of your total memory. If your machine has 8GB of memory, chances are that Frigate will have access to up to 4GB without any additional configuration. diff --git a/frigate/app.py b/frigate/app.py index 9149f98546..68e5c872cb 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -537,7 +537,7 @@ def start_camera_capture_processes(self) -> None: capture_process = mp.Process( target=capture_camera, name=f"camera_capture:{name}", - args=(name, config, self.camera_metrics[name]), + args=(name, config, self.shm_frame_count, self.camera_metrics[name]), ) capture_process.daemon = True self.camera_metrics[name]["capture_process"] = capture_process @@ -601,19 +601,34 @@ def start_watchdog(self) -> None: self.frigate_watchdog.start() def check_shm(self) -> None: - available_shm = round(shutil.disk_usage("/dev/shm").total / pow(2, 20), 1) - min_req_shm = 30 - - for _, camera in self.config.cameras.items(): - min_req_shm += round( - (camera.detect.width * camera.detect.height * 1.5 * 9 + 270480) - / 1048576, - 1, - ) + total_shm = round(shutil.disk_usage("/dev/shm").total / pow(2, 20), 1) + + # required for log files + nginx cache + min_req_shm = 40 + 10 + + if self.config.birdseye.restream: + min_req_shm += 8 + + available_shm = total_shm - min_req_shm + cam_total_frame_size = 0 + + for camera in self.config.cameras.values(): + if camera.enabled: + cam_total_frame_size += round( + (camera.detect.width * camera.detect.height * 1.5 + 270480) + / 1048576, + 1, + ) + + self.shm_frame_count = min(50, int(available_shm / (cam_total_frame_size))) + + logger.debug( + f"Calculated total camera size {available_shm} / {cam_total_frame_size} :: {self.shm_frame_count} frames for each camera in SHM" + ) - if available_shm < min_req_shm: + if self.shm_frame_count < 10: logger.warning( - f"The current SHM size of {available_shm}MB is too small, recommend increasing it to at least {min_req_shm}MB." + f"The current SHM size of {total_shm}MB is too small, recommend increasing it to at least {round(min_req_shm + cam_total_frame_size)}MB." ) def init_auth(self) -> None: @@ -718,6 +733,7 @@ def start(self) -> None: self.init_historical_regions() self.start_detected_frames_processor() self.start_camera_processors() + self.check_shm() self.start_camera_capture_processes() self.start_audio_processors() self.start_storage_maintainer() @@ -729,7 +745,6 @@ def start(self) -> None: self.start_event_cleanup() self.start_record_cleanup() self.start_watchdog() - self.check_shm() self.init_auth() # Flask only listens for SIGINT, so we need to catch SIGTERM and send SIGINT diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index aac85c01a6..a60663e7dc 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -78,9 +78,11 @@ def _process_updates(self) -> None: try: frame_id = f"{camera}{data['frame_time']}" yuv_frame = self.frame_manager.get(frame_id, camera_config.frame_shape_yuv) - data["thumbnail"] = self._create_thumbnail(yuv_frame, data["box"]) - self.tracked_events[data["id"]].append(data) - self.frame_manager.close(frame_id) + + if yuv_frame is not None: + data["thumbnail"] = self._create_thumbnail(yuv_frame, data["box"]) + self.tracked_events[data["id"]].append(data) + self.frame_manager.close(frame_id) except FileNotFoundError: pass diff --git a/frigate/object_detection.py b/frigate/object_detection.py index 207637ed47..22ff84f894 100644 --- a/frigate/object_detection.py +++ b/frigate/object_detection.py @@ -118,6 +118,7 @@ def receiveSignal(signalNumber, frame): ) if input_frame is None: + logger.warning(f"Failed to get frame {connection_id} from SHM") continue # detect and send the output diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 5b7a1bedfd..3616d6b503 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -147,7 +147,7 @@ def compute_score(self): """get median of scores for object.""" return median(self.score_history) - def update(self, current_frame_time, obj_data): + def update(self, current_frame_time: float, obj_data, has_valid_frame: bool): thumb_update = False significant_change = False autotracker_update = False @@ -168,7 +168,7 @@ def update(self, current_frame_time, obj_data): self.false_positive = self._is_false_positive() self.active = self.is_active() - if not self.false_positive: + if not self.false_positive and has_valid_frame: # determine if this frame is a better thumbnail if self.thumbnail_data is None or is_better_thumbnail( self.obj_data["label"], @@ -668,10 +668,14 @@ def on(self, event_type: str, callback: Callable[[dict], None]): def update(self, frame_time, current_detections, motion_boxes, regions): # get the new frame frame_id = f"{self.name}{frame_time}" + current_frame = self.frame_manager.get( frame_id, self.camera_config.frame_shape_yuv ) + if current_frame is None: + logger.debug(f"Failed to get frame {frame_id} from SHM") + tracked_objects = self.tracked_objects.copy() current_ids = set(current_detections.keys()) previous_ids = set(tracked_objects.keys()) @@ -695,14 +699,14 @@ def update(self, frame_time, current_detections, motion_boxes, regions): for id in updated_ids: updated_obj = tracked_objects[id] thumb_update, significant_update, autotracker_update = updated_obj.update( - frame_time, current_detections[id] + frame_time, current_detections[id], current_frame is not None ) if autotracker_update or significant_update: for c in self.callbacks["autotrack"]: c(self.name, updated_obj, frame_time) - if thumb_update: + if thumb_update and current_frame is not None: # ensure this frame is stored in the cache if ( updated_obj.thumbnail_data["frame_time"] == frame_time @@ -886,12 +890,16 @@ def update(self, frame_time, current_detections, motion_boxes, regions): with self.current_frame_lock: self.tracked_objects = tracked_objects - self.current_frame_time = frame_time self.motion_boxes = motion_boxes self.regions = regions - self._current_frame = current_frame - if self.previous_frame_id is not None: - self.frame_manager.close(self.previous_frame_id) + + if current_frame is not None: + self.current_frame_time = frame_time + self._current_frame = current_frame + + if self.previous_frame_id is not None: + self.frame_manager.close(self.previous_frame_id) + self.previous_frame_id = frame_id diff --git a/frigate/output/birdseye.py b/frigate/output/birdseye.py index 6e0e2bc22e..e22628ecd2 100644 --- a/frigate/output/birdseye.py +++ b/frigate/output/birdseye.py @@ -357,16 +357,14 @@ def copy_to_position(self, position, camera=None, frame_time=None): frame = None channel_dims = None else: - try: - frame = self.frame_manager.get( - f"{camera}{frame_time}", self.config.cameras[camera].frame_shape_yuv - ) - except FileNotFoundError: - # TODO: better frame management would prevent this edge case - logger.warning( - f"Unable to copy frame {camera}{frame_time} to birdseye." - ) + frame = self.frame_manager.get( + f"{camera}{frame_time}", self.config.cameras[camera].frame_shape_yuv + ) + + if frame is None: + logger.debug(f"Unable to copy frame {camera}{frame_time} to birdseye.") return + channel_dims = self.cameras[camera]["channel_dims"] copy_yuv_to_position( diff --git a/frigate/output/output.py b/frigate/output/output.py index b29144b009..f351e967d3 100644 --- a/frigate/output/output.py +++ b/frigate/output/output.py @@ -45,7 +45,6 @@ def receiveSignal(signalNumber, frame): signal.signal(signal.SIGINT, receiveSignal) frame_manager = SharedMemoryFrameManager() - previous_frames = {} # start a websocket server on 8082 WebSocketWSGIHandler.http_version = "1.1" @@ -99,6 +98,10 @@ def receiveSignal(signalNumber, frame): frame = frame_manager.get(frame_id, config.cameras[camera].frame_shape_yuv) + if frame is None: + logger.debug(f"Failed to get frame {frame_id} from SHM") + continue + # send camera frame to ffmpeg process if websockets are connected if any( ws.environ["PATH_INFO"].endswith(camera) for ws in websocket_server.manager @@ -128,10 +131,6 @@ def receiveSignal(signalNumber, frame): ) preview_write_times[camera] = frame_time - # delete frames after they have been used for output - if camera in previous_frames: - frame_manager.delete(f"{camera}{previous_frames[camera]}") - # if another camera generated a preview, # check for any cameras that are currently offline # and need to generate a preview @@ -141,7 +140,7 @@ def receiveSignal(signalNumber, frame): preview_recorders[camera].flag_offline(frame_time) preview_write_times[camera] = frame_time - previous_frames[camera] = frame_time + frame_manager.close(frame_id) move_preview_frames("clips") @@ -161,7 +160,7 @@ def receiveSignal(signalNumber, frame): frame_id = f"{camera}{frame_time}" frame = frame_manager.get(frame_id, config.cameras[camera].frame_shape_yuv) - frame_manager.delete(frame_id) + frame_manager.close(frame_id) detection_subscriber.stop() diff --git a/frigate/ptz/autotrack.py b/frigate/ptz/autotrack.py index 44082a52eb..92c4141a72 100644 --- a/frigate/ptz/autotrack.py +++ b/frigate/ptz/autotrack.py @@ -99,6 +99,10 @@ def motion_estimator(self, detections, frame_time, camera): frame_id, self.camera_config.frame_shape_yuv ) + if yuv_frame is None: + self.coord_transformations = None + return None + frame = cv2.cvtColor(yuv_frame, cv2.COLOR_YUV2GRAY_I420) # mask out detections for better motion estimation diff --git a/frigate/review/maintainer.py b/frigate/review/maintainer.py index fa2678d9b7..bea62256d5 100644 --- a/frigate/review/maintainer.py +++ b/frigate/review/maintainer.py @@ -292,6 +292,11 @@ def update_existing_segment( yuv_frame = self.frame_manager.get( frame_id, camera_config.frame_shape_yuv ) + + if yuv_frame is None: + logger.debug(f"Failed to get frame {frame_id} from SHM") + return + self.update_segment( segment, camera_config, yuv_frame, active_objects, prev_data ) @@ -305,6 +310,11 @@ def update_existing_segment( yuv_frame = self.frame_manager.get( frame_id, camera_config.frame_shape_yuv ) + + if yuv_frame is None: + logger.debug(f"Failed to get frame {frame_id} from SHM") + return + segment.save_full_frame(camera_config, yuv_frame) self.frame_manager.close(frame_id) self.update_segment(segment, camera_config, None, [], prev_data) @@ -401,6 +411,11 @@ def check_if_new_segment( yuv_frame = self.frame_manager.get( frame_id, camera_config.frame_shape_yuv ) + + if yuv_frame is None: + logger.debug(f"Failed to get frame {frame_id} from SHM") + return + self.active_review_segments[camera].update_frame( camera_config, yuv_frame, active_objects ) diff --git a/frigate/util/image.py b/frigate/util/image.py index 3962d9600e..7c470a2a0c 100644 --- a/frigate/util/image.py +++ b/frigate/util/image.py @@ -687,31 +687,46 @@ def delete(self, name): class SharedMemoryFrameManager(FrameManager): def __init__(self): - self.shm_store = {} + self.shm_store: dict[str, shared_memory.SharedMemory] = {} - def create(self, name, size) -> AnyStr: + def create(self, name: str, size) -> AnyStr: shm = shared_memory.SharedMemory(name=name, create=True, size=size) self.shm_store[name] = shm return shm.buf - def get(self, name, shape): - if name in self.shm_store: - shm = self.shm_store[name] - else: - shm = shared_memory.SharedMemory(name=name) - self.shm_store[name] = shm - return np.ndarray(shape, dtype=np.uint8, buffer=shm.buf) + def get(self, name: str, shape) -> Optional[np.ndarray]: + try: + if name in self.shm_store: + shm = self.shm_store[name] + else: + shm = shared_memory.SharedMemory(name=name) + self.shm_store[name] = shm + return np.ndarray(shape, dtype=np.uint8, buffer=shm.buf) + except FileNotFoundError: + return None - def close(self, name): + def close(self, name: str): if name in self.shm_store: self.shm_store[name].close() del self.shm_store[name] - def delete(self, name): + def delete(self, name: str): if name in self.shm_store: self.shm_store[name].close() - self.shm_store[name].unlink() + + try: + self.shm_store[name].unlink() + except FileNotFoundError: + pass + del self.shm_store[name] + else: + try: + shm = shared_memory.SharedMemory(name=name) + shm.close() + shm.unlink() + except FileNotFoundError: + pass def create_mask(frame_shape, mask): diff --git a/frigate/video.py b/frigate/video.py index 1c74575dc3..80749c6546 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -94,7 +94,8 @@ def start_or_restart_ffmpeg( def capture_frames( ffmpeg_process, - camera_name, + config: CameraConfig, + shm_frame_count: int, frame_shape, frame_manager: FrameManager, frame_queue, @@ -108,27 +109,40 @@ def capture_frames( frame_rate.start() skipped_eps = EventsPerSecond() skipped_eps.start() + + shm_frames: list[str] = [] + while True: fps.value = frame_rate.eps() skipped_fps.value = skipped_eps.eps() - current_frame.value = datetime.datetime.now().timestamp() - frame_name = f"{camera_name}{current_frame.value}" + frame_name = f"{config.name}{current_frame.value}" frame_buffer = frame_manager.create(frame_name, frame_size) try: frame_buffer[:] = ffmpeg_process.stdout.read(frame_size) + + # update frame cache and cleanup existing frames + shm_frames.append(frame_name) + + if len(shm_frames) > shm_frame_count: + expired_frame_name = shm_frames.pop(0) + frame_manager.delete(expired_frame_name) except Exception: + # always delete the frame + frame_manager.delete(frame_name) + # shutdown has been initiated if stop_event.is_set(): break - logger.error(f"{camera_name}: Unable to read frames from ffmpeg process.") + + logger.error(f"{config.name}: Unable to read frames from ffmpeg process.") if ffmpeg_process.poll() is not None: logger.error( - f"{camera_name}: ffmpeg process is not running. exiting capture thread..." + f"{config.name}: ffmpeg process is not running. exiting capture thread..." ) - frame_manager.delete(frame_name) break + continue frame_rate.update() @@ -137,12 +151,14 @@ def capture_frames( try: # add to the queue frame_queue.put(current_frame.value, False) - # close the frame frame_manager.close(frame_name) except queue.Full: # if the queue is full, skip this frame skipped_eps.update() - frame_manager.delete(frame_name) + + # clear out frames + for frame in shm_frames: + frame_manager.delete(frame) class CameraWatchdog(threading.Thread): @@ -150,6 +166,7 @@ def __init__( self, camera_name, config: CameraConfig, + shm_frame_count: int, frame_queue, camera_fps, skipped_fps, @@ -160,6 +177,7 @@ def __init__( self.logger = logging.getLogger(f"watchdog.{camera_name}") self.camera_name = camera_name self.config = config + self.shm_frame_count = shm_frame_count self.capture_thread = None self.ffmpeg_detect_process = None self.logpipe = LogPipe(f"ffmpeg.{self.camera_name}.detect") @@ -282,7 +300,8 @@ def start_ffmpeg_detect(self): ) self.ffmpeg_pid.value = self.ffmpeg_detect_process.pid self.capture_thread = CameraCapture( - self.camera_name, + self.config, + self.shm_frame_count, self.ffmpeg_detect_process, self.frame_shape, self.frame_queue, @@ -321,7 +340,8 @@ def get_latest_segment_datetime(self, latest_segment: datetime.datetime) -> int: class CameraCapture(threading.Thread): def __init__( self, - camera_name, + config: CameraConfig, + shm_frame_count: int, ffmpeg_process, frame_shape, frame_queue, @@ -330,8 +350,9 @@ def __init__( stop_event, ): threading.Thread.__init__(self) - self.name = f"capture:{camera_name}" - self.camera_name = camera_name + self.name = f"capture:{config.name}" + self.config = config + self.shm_frame_count = shm_frame_count self.frame_shape = frame_shape self.frame_queue = frame_queue self.fps = fps @@ -345,7 +366,8 @@ def __init__( def run(self): capture_frames( self.ffmpeg_process, - self.camera_name, + self.config, + self.shm_frame_count, self.frame_shape, self.frame_manager, self.frame_queue, @@ -356,7 +378,7 @@ def run(self): ) -def capture_camera(name, config: CameraConfig, process_info): +def capture_camera(name, config: CameraConfig, shm_frame_count: int, process_info): stop_event = mp.Event() def receiveSignal(signalNumber, frame): @@ -373,6 +395,7 @@ def receiveSignal(signalNumber, frame): camera_watchdog = CameraWatchdog( name, config, + shm_frame_count, frame_queue, process_info["camera_fps"], process_info["skipped_fps"], @@ -567,7 +590,7 @@ def process_frames( ) if frame is None: - logger.info(f"{camera_name}: frame {frame_time} is not in memory store.") + logger.debug(f"{camera_name}: frame {frame_time} is not in memory store.") continue # look for motion if enabled