Metadata Timeline (#6194)

* Create timeline table

* Fix indexes

* Add other fields

* Adjust schema to be less descriptive

* Handle timeline queue from tracked object data

* Setup timeline queue in events

* Add source id for index

* Add other fields

* Fixes

* Formatting

* Store better data

* Add api with filtering

* Setup basic UI for timeline in events

* Cleanups

* Add recordings snapshot url

* Start working on timeline ui

* Add tooltip with info

* Improve icons

* Fix start time with clip

* Move player logic back to clips

* Make box in timeline relative coordinates

* Make region relative

* Get box overlay working

* Remove overlay when playing again

* Add disclaimer when selecting overlay points

* Add docs for new apis

* Fix mobile

* Fix docs

* Change color of bottom center box

* Fix vscode formatting

frigate/app.py

@@ -23,13 +23,14 @@ from frigate.object_detection import ObjectDetectProcess
 from frigate.events import EventCleanup, EventProcessor
 from frigate.http import create_app
 from frigate.log import log_process, root_configurer
-from frigate.models import Event, Recordings
+from frigate.models import Event, Recordings, Timeline
 from frigate.object_processing import TrackedObjectProcessor
 from frigate.output import output_frames
 from frigate.plus import PlusApi
 from frigate.record import RecordingCleanup, RecordingMaintainer
 from frigate.stats import StatsEmitter, stats_init
 from frigate.storage import StorageMaintainer
+from frigate.timeline import TimelineProcessor
 from frigate.version import VERSION
 from frigate.video import capture_camera, track_camera
 from frigate.watchdog import FrigateWatchdog
@@ -135,6 +136,9 @@ class FrigateApp:
         # Queue for recordings info
         self.recordings_info_queue: Queue = mp.Queue()
 
+        # Queue for timeline events
+        self.timeline_queue: Queue = mp.Queue()
+
     def init_database(self) -> None:
         # Migrate DB location
         old_db_path = os.path.join(CLIPS_DIR, "frigate.db")
@@ -154,7 +158,7 @@ class FrigateApp:
         migrate_db.close()
 
         self.db = SqliteQueueDatabase(self.config.database.path)
-        models = [Event, Recordings]
+        models = [Event, Recordings, Timeline]
         self.db.bind(models)
 
     def init_stats(self) -> None:
@@ -286,12 +290,19 @@ class FrigateApp:
             capture_process.start()
             logger.info(f"Capture process started for {name}: {capture_process.pid}")
 
+    def start_timeline_processor(self) -> None:
+        self.timeline_processor = TimelineProcessor(
+            self.config, self.timeline_queue, self.stop_event
+        )
+        self.timeline_processor.start()
+
     def start_event_processor(self) -> None:
         self.event_processor = EventProcessor(
             self.config,
             self.camera_metrics,
             self.event_queue,
             self.event_processed_queue,
+            self.timeline_queue,
             self.stop_event,
         )
         self.event_processor.start()
@@ -384,6 +395,7 @@ class FrigateApp:
         self.start_storage_maintainer()
         self.init_stats()
         self.init_web_server()
+        self.start_timeline_processor()
         self.start_event_processor()
         self.start_event_cleanup()
         self.start_recording_maintainer()

frigate/events.py

@@ -3,14 +3,14 @@ import logging
 import os
 import queue
 import threading
-import time
 from pathlib import Path
 
 from peewee import fn
 
-from frigate.config import EventsConfig, FrigateConfig, RecordConfig
+from frigate.config import EventsConfig, FrigateConfig
 from frigate.const import CLIPS_DIR
 from frigate.models import Event
+from frigate.timeline import TimelineSourceEnum
 from frigate.types import CameraMetricsTypes
 
 from multiprocessing.queues import Queue
@@ -48,6 +48,7 @@ class EventProcessor(threading.Thread):
         camera_processes: dict[str, CameraMetricsTypes],
         event_queue: Queue,
         event_processed_queue: Queue,
+        timeline_queue: Queue,
         stop_event: MpEvent,
     ):
         threading.Thread.__init__(self)
@@ -56,6 +57,7 @@ class EventProcessor(threading.Thread):
         self.camera_processes = camera_processes
         self.event_queue = event_queue
         self.event_processed_queue = event_processed_queue
+        self.timeline_queue = timeline_queue
         self.events_in_process: Dict[str, Event] = {}
         self.stop_event = stop_event
 
@@ -73,6 +75,16 @@ class EventProcessor(threading.Thread):
 
             logger.debug(f"Event received: {event_type} {camera} {event_data['id']}")
 
+            self.timeline_queue.put(
+                (
+                    camera,
+                    TimelineSourceEnum.tracked_object,
+                    event_type,
+                    self.events_in_process.get(event_data["id"]),
+                    event_data,
+                )
+            )
+
             event_config: EventsConfig = self.config.cameras[camera].record.events
 
             if event_type == "start":

frigate/http.py

@@ -33,7 +33,7 @@ from playhouse.shortcuts import model_to_dict
 
 from frigate.config import FrigateConfig
 from frigate.const import CLIPS_DIR, MAX_SEGMENT_DURATION, RECORD_DIR
-from frigate.models import Event, Recordings
+from frigate.models import Event, Recordings, Timeline
 from frigate.object_processing import TrackedObject
 from frigate.stats import stats_snapshot
 from frigate.util import (
@@ -414,6 +414,42 @@ def event_thumbnail(id, max_cache_age=2592000):
     return response
 
 
+@bp.route("/timeline")
+def timeline():
+    camera = request.args.get("camera", "all")
+    source_id = request.args.get("source_id", type=str)
+    limit = request.args.get("limit", 100)
+
+    clauses = []
+
+    selected_columns = [
+        Timeline.timestamp,
+        Timeline.camera,
+        Timeline.source,
+        Timeline.source_id,
+        Timeline.class_type,
+        Timeline.data,
+    ]
+
+    if camera != "all":
+        clauses.append((Timeline.camera == camera))
+
+    if source_id:
+        clauses.append((Timeline.source_id == source_id))
+
+    if len(clauses) == 0:
+        clauses.append((True))
+
+    timeline = (
+        Timeline.select(*selected_columns)
+        .where(reduce(operator.and_, clauses))
+        .order_by(Timeline.timestamp.asc())
+        .limit(limit)
+    )
+
+    return jsonify([model_to_dict(t) for t in timeline])
+
+
 @bp.route("/<camera_name>/<label>/best.jpg")
 @bp.route("/<camera_name>/<label>/thumbnail.jpg")
 def label_thumbnail(camera_name, label):
@@ -924,6 +960,53 @@ def latest_frame(camera_name):
         return "Camera named {} not found".format(camera_name), 404
 
 
+@bp.route("/<camera_name>/recordings/<frame_time>/snapshot.png")
+def get_snapshot_from_recording(camera_name: str, frame_time: str):
+    if camera_name not in current_app.frigate_config.cameras:
+        return "Camera named {} not found".format(camera_name), 404
+
+    frame_time = float(frame_time)
+    recording_query = (
+        Recordings.select()
+        .where(
+            ((frame_time > Recordings.start_time) & (frame_time < Recordings.end_time))
+        )
+        .where(Recordings.camera == camera_name)
+    )
+
+    try:
+        recording: Recordings = recording_query.get()
+        time_in_segment = frame_time - recording.start_time
+
+        ffmpeg_cmd = [
+            "ffmpeg",
+            "-hide_banner",
+            "-loglevel",
+            "warning",
+            "-ss",
+            f"00:00:{time_in_segment}",
+            "-i",
+            recording.path,
+            "-frames:v",
+            "1",
+            "-c:v",
+            "png",
+            "-f",
+            "image2pipe",
+            "-",
+        ]
+
+        process = sp.run(
+            ffmpeg_cmd,
+            capture_output=True,
+        )
+        response = make_response(process.stdout)
+        response.headers["Content-Type"] = "image/png"
+        return response
+    except DoesNotExist:
+        return "Recording not found for {} at {}".format(camera_name, frame_time), 404
+
+
 @bp.route("/recordings/storage", methods=["GET"])
 def get_recordings_storage_usage():
     recording_stats = stats_snapshot(

frigate/models.py

@@ -32,6 +32,15 @@ class Event(Model):  # type: ignore[misc]
     plus_id = CharField(max_length=30)
 
 
+class Timeline(Model):  # type: ignore[misc]
+    timestamp = DateTimeField()
+    camera = CharField(index=True, max_length=20)
+    source = CharField(index=True, max_length=20)  # ex: tracked object, audio, external
+    source_id = CharField(index=True, max_length=30)
+    class_type = CharField(max_length=50)  # ex: entered_zone, audio_heard
+    data = JSONField()  # ex: tracked object id, region, box, etc.
+
+
 class Recordings(Model):  # type: ignore[misc]
     id = CharField(null=False, primary_key=True, max_length=30)
     camera = CharField(index=True, max_length=20)

frigate/timeline.py

@@ -0,0 +1,140 @@
+"""Record events for object, audio, etc. detections."""
+
+import logging
+import threading
+import queue
+
+from enum import Enum
+
+from frigate.config import FrigateConfig
+from frigate.models import Timeline
+
+from multiprocessing.queues import Queue
+from multiprocessing.synchronize import Event as MpEvent
+
+logger = logging.getLogger(__name__)
+
+
+class TimelineSourceEnum(str, Enum):
+    # api = "api"
+    # audio = "audio"
+    tracked_object = "tracked_object"
+
+
+class TimelineProcessor(threading.Thread):
+    """Handle timeline queue and update DB."""
+
+    def __init__(
+        self,
+        config: FrigateConfig,
+        queue: Queue,
+        stop_event: MpEvent,
+    ) -> None:
+        threading.Thread.__init__(self)
+        self.name = "timeline_processor"
+        self.config = config
+        self.queue = queue
+        self.stop_event = stop_event
+
+    def run(self) -> None:
+        while not self.stop_event.is_set():
+            try:
+                (
+                    camera,
+                    input_type,
+                    event_type,
+                    prev_event_data,
+                    event_data,
+                ) = self.queue.get(timeout=1)
+            except queue.Empty:
+                continue
+
+            if input_type == TimelineSourceEnum.tracked_object:
+                self.handle_object_detection(
+                    camera, event_type, prev_event_data, event_data
+                )
+
+    def handle_object_detection(
+        self,
+        camera: str,
+        event_type: str,
+        prev_event_data: dict[any, any],
+        event_data: dict[any, any],
+    ) -> None:
+        """Handle object detection."""
+        camera_config = self.config.cameras[camera]
+
+        if event_type == "start":
+            Timeline.insert(
+                timestamp=event_data["frame_time"],
+                camera=camera,
+                source="tracked_object",
+                source_id=event_data["id"],
+                class_type="visible",
+                data={
+                    "box": [
+                        event_data["box"][0] / camera_config.detect.width,
+                        event_data["box"][1] / camera_config.detect.height,
+                        event_data["box"][2] / camera_config.detect.width,
+                        event_data["box"][3] / camera_config.detect.height,
+                    ],
+                    "label": event_data["label"],
+                    "region": [
+                        event_data["region"][0] / camera_config.detect.width,
+                        event_data["region"][1] / camera_config.detect.height,
+                        event_data["region"][2] / camera_config.detect.width,
+                        event_data["region"][3] / camera_config.detect.height,
+                    ],
+                },
+            ).execute()
+        elif (
+            event_type == "update"
+            and prev_event_data["current_zones"] != event_data["current_zones"]
+            and len(event_data["current_zones"]) > 0
+        ):
+            Timeline.insert(
+                timestamp=event_data["frame_time"],
+                camera=camera,
+                source="tracked_object",
+                source_id=event_data["id"],
+                class_type="entered_zone",
+                data={
+                    "box": [
+                        event_data["box"][0] / camera_config.detect.width,
+                        event_data["box"][1] / camera_config.detect.height,
+                        event_data["box"][2] / camera_config.detect.width,
+                        event_data["box"][3] / camera_config.detect.height,
+                    ],
+                    "label": event_data["label"],
+                    "region": [
+                        event_data["region"][0] / camera_config.detect.width,
+                        event_data["region"][1] / camera_config.detect.height,
+                        event_data["region"][2] / camera_config.detect.width,
+                        event_data["region"][3] / camera_config.detect.height,
+                    ],
+                    "zones": event_data["current_zones"],
+                },
+            ).execute()
+        elif event_type == "end":
+            Timeline.insert(
+                timestamp=event_data["frame_time"],
+                camera=camera,
+                source="tracked_object",
+                source_id=event_data["id"],
+                class_type="gone",
+                data={
+                    "box": [
+                        event_data["box"][0] / camera_config.detect.width,
+                        event_data["box"][1] / camera_config.detect.height,
+                        event_data["box"][2] / camera_config.detect.width,
+                        event_data["box"][3] / camera_config.detect.height,
+                    ],
+                    "label": event_data["label"],
+                    "region": [
+                        event_data["region"][0] / camera_config.detect.width,
+                        event_data["region"][1] / camera_config.detect.height,
+                        event_data["region"][2] / camera_config.detect.width,
+                        event_data["region"][3] / camera_config.detect.height,
+                    ],
+                },
+            ).execute()