Ongoing review segments (#10924)

* Update review maintainer to save events when ongoing

* Handle previews for in progress review items

* Reset DB items in app

* Handle in progress review items

* Scroll back down to selected event item

* Handle undefined end time

* Formatting

* remove unused

* Make export handles have full resolution

* reduce preview thumbnail props

* fix missing return

Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>

---------

Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>

frigate/api/media.py

@@ -1333,7 +1333,9 @@ def review_preview(id: str):
 
     padding = 8
     start_ts = review.start_time - padding
-    end_ts = review.end_time + padding
+    end_ts = (
+        review.end_time + padding if review.end_time else datetime.now().timestamp()
+    )
     return preview_gif(review.camera, start_ts, end_ts)
 
 
@@ -1344,8 +1346,15 @@ def preview_thumbnail(file_name: str):
     safe_file_name_current = secure_filename(file_name)
     preview_dir = os.path.join(CACHE_DIR, "preview_frames")
 
-    with open(os.path.join(preview_dir, safe_file_name_current), "rb") as image_file:
-        jpg_bytes = image_file.read()
+    try:
+        with open(
+            os.path.join(preview_dir, safe_file_name_current), "rb"
+        ) as image_file:
+            jpg_bytes = image_file.read()
+    except FileNotFoundError:
+        return make_response(
+            jsonify({"success": False, "message": "Image file not found"}), 404
+        )
 
     response = make_response(jpg_bytes)
     response.headers["Content-Type"] = "image/jpeg"

frigate/api/review.py

@@ -27,10 +27,18 @@ def review():
 
     before = request.args.get("before", type=float, default=datetime.now().timestamp())
     after = request.args.get(
-        "after", type=float, default=(datetime.now() - timedelta(hours=18)).timestamp()
+        "after", type=float, default=(datetime.now() - timedelta(hours=24)).timestamp()
     )
 
-    clauses = [((ReviewSegment.start_time > after) & (ReviewSegment.end_time < before))]
+    clauses = [
+        (
+            (ReviewSegment.start_time > after)
+            & (
+                (ReviewSegment.end_time.is_null(True))
+                | (ReviewSegment.end_time < before)
+            )
+        )
+    ]
 
     if cameras != "all":
         camera_list = cameras.split(",")
@@ -45,6 +53,7 @@ def review():
         for label in filtered_labels:
             label_clauses.append(
                 (ReviewSegment.data["objects"].cast("text") % f'*"{label}"*')
+                | (ReviewSegment.data["audio"].cast("text") % f'*"{label}"*')
             )
 
         label_clause = reduce(operator.or_, label_clauses)
@@ -94,6 +103,7 @@ def review_summary():
         for label in filtered_labels:
             label_clauses.append(
                 (ReviewSegment.data["objects"].cast("text") % f'*"{label}"*')
+                | (ReviewSegment.data["audio"].cast("text") % f'*"{label}"*')
             )
 
         label_clause = reduce(operator.or_, label_clauses)

frigate/app.py

@@ -667,6 +667,14 @@ class FrigateApp:
         logger.info("Stopping...")
         self.stop_event.set()
 
+        # set an end_time on entries without an end_time before exiting
+        Event.update(end_time=datetime.datetime.now().timestamp()).where(
+            Event.end_time == None
+        ).execute()
+        ReviewSegment.update(end_time=datetime.datetime.now().timestamp()).where(
+            ReviewSegment.end_time == None
+        ).execute()
+
         # Stop Communicators
         self.inter_process_communicator.stop()
         self.inter_config_updater.stop()

frigate/events/maintainer.py

@@ -1,4 +1,3 @@
-import datetime
 import logging
 import threading
 from multiprocessing import Queue
@@ -112,10 +111,6 @@ class EventProcessor(threading.Thread):
 
                 self.handle_external_detection(event_type, event_data)
 
-        # set an end_time on events without an end_time before exiting
-        Event.update(end_time=datetime.datetime.now().timestamp()).where(
-            Event.end_time == None
-        ).execute()
         self.event_receiver.stop()
         self.event_end_publisher.stop()
         logger.info("Exiting event processor...")

frigate/review/maintainer.py

@@ -66,6 +66,7 @@ class PendingReviewSegment:
         # thumbnail
         self.frame = np.zeros((THUMB_HEIGHT * 3 // 2, THUMB_WIDTH), np.uint8)
         self.frame_active_count = 0
+        self.frame_path = os.path.join(CLIPS_DIR, f"thumb-{self.camera}-{self.id}.jpg")
 
     def update_frame(
         self, camera_config: CameraConfig, frame, objects: list[TrackedObject]
@@ -98,19 +99,19 @@ class PendingReviewSegment:
             color_frame, dsize=(width, THUMB_HEIGHT), interpolation=cv2.INTER_AREA
         )
 
-    def end(self) -> dict:
-        path = os.path.join(CLIPS_DIR, f"thumb-{self.camera}-{self.id}.jpg")
-
         if self.frame is not None:
-            cv2.imwrite(path, self.frame, [int(cv2.IMWRITE_WEBP_QUALITY), 60])
+            cv2.imwrite(
+                self.frame_path, self.frame, [int(cv2.IMWRITE_WEBP_QUALITY), 60]
+            )
 
+    def get_data(self, ended: bool) -> dict:
         return {
             ReviewSegment.id: self.id,
             ReviewSegment.camera: self.camera,
             ReviewSegment.start_time: self.start_time,
-            ReviewSegment.end_time: self.last_update,
+            ReviewSegment.end_time: self.last_update if ended else None,
             ReviewSegment.severity: self.severity.value,
-            ReviewSegment.thumb_path: path,
+            ReviewSegment.thumb_path: self.frame_path,
             ReviewSegment.data: {
                 "detections": list(set(self.detections.keys())),
                 "objects": list(set(self.detections.values())),
@@ -141,9 +142,20 @@ class ReviewSegmentMaintainer(threading.Thread):
 
         self.stop_event = stop_event
 
+    def update_segment(self, segment: PendingReviewSegment) -> None:
+        """Update segment."""
+        seg_data = segment.get_data(ended=False)
+        self.requestor.send_data(UPSERT_REVIEW_SEGMENT, seg_data)
+        self.requestor.send_data(
+            "reviews",
+            json.dumps(
+                {"type": "update", "review": {k.name: v for k, v in seg_data.items()}}
+            ),
+        )
+
     def end_segment(self, segment: PendingReviewSegment) -> None:
         """End segment."""
-        seg_data = segment.end()
+        seg_data = segment.get_data(ended=True)
         self.requestor.send_data(UPSERT_REVIEW_SEGMENT, seg_data)
         self.requestor.send_data(
             "reviews",
@@ -179,6 +191,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                 )
                 segment.update_frame(camera_config, yuv_frame, active_objects)
                 self.frame_manager.close(frame_id)
+                self.update_segment(segment)
 
             for object in active_objects:
                 if not object["sub_label"]:
@@ -263,6 +276,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                 camera_config, yuv_frame, active_objects
             )
             self.frame_manager.close(frame_id)
+            self.update_segment(self.active_review_segments[camera])
         elif len(motion) >= 20:
             self.active_review_segments[camera] = PendingReviewSegment(
                 camera,
@@ -398,6 +412,11 @@ class ReviewSegmentMaintainer(threading.Thread):
                             "end_time"
                         ]
 
+        self.config_subscriber.stop()
+        self.requestor.stop()
+        self.detection_subscriber.stop()
+        logger.info("Exiting review maintainer...")
+
 
 def get_active_objects(
     frame_time: float, camera_config: CameraConfig, all_objects: list[TrackedObject]