Add ability to restream birdseye (#4761)

* Try using RTSP for restream

* Add ability to get snapshot of birdseye when birdseye restream is enabled

* Write to pipe instead of encoding mpeg1

* Write to cache instead

* Use const for location

* Formatting

* Add hardware encoding for birdseye based on ffmpeg preset

* Provide framerate

* Adjust args

* Fix order

* Delete pipe file if it exists

* Cleanup spacing

* Fix spacing

frigate/config.py

@@ -519,6 +519,7 @@ class RestreamConfig(FrigateBaseModel):
     force_audio: bool = Field(
         default=True, title="Force audio compatibility with the browser."
     )
+    birdseye: bool = Field(default=False, title="Restream the birdseye feed via RTSP.")
     jsmpeg: JsmpegStreamConfig = Field(
         default_factory=JsmpegStreamConfig, title="Jsmpeg Stream Configuration."
     )

frigate/const.py

@@ -1,6 +1,7 @@
 BASE_DIR = "/media/frigate"
 CLIPS_DIR = f"{BASE_DIR}/clips"
 RECORD_DIR = f"{BASE_DIR}/recordings"
+BIRDSEYE_PIPE = "/tmp/cache/birdseye"
 CACHE_DIR = "/tmp/cache"
 YAML_EXT = (".yaml", ".yml")
 PLUS_ENV_VAR = "PLUS_API_KEY"

frigate/ffmpeg_presets.py

@@ -129,6 +129,107 @@ PRESETS_HW_ACCEL_SCALE = {
     ],
 }
 
+PRESETS_HW_ACCEL_ENCODE = {
+    "preset-intel-vaapi": [
+        "-c:v",
+        "h264_vaapi",
+        "-g",
+        "50",
+        "-bf",
+        "0",
+        "-profile:v",
+        "high",
+        "-level:v",
+        "4.1",
+        "-sei:v",
+        "0",
+    ],
+    "preset-intel-qsv-h264": [
+        "-c:v",
+        "h264_qsv",
+        "-g",
+        "50",
+        "-bf",
+        "0",
+        "-profile:v",
+        "high",
+        "-level:v",
+        "4.1",
+        "-async_depth:v",
+        "1",
+    ],
+    "preset-intel-qsv-h265": [
+        "-c:v",
+        "h264_qsv",
+        "-g",
+        "50",
+        "-bf",
+        "0",
+        "-profile:v",
+        "high",
+        "-level:v",
+        "4.1",
+        "-async_depth:v",
+        "1",
+    ],
+    "preset-amd-vaapi": [
+        "-c:v",
+        "h264_vaapi",
+        "-g",
+        "50",
+        "-bf",
+        "0",
+        "-profile:v",
+        "high",
+        "-level:v",
+        "4.1",
+        "-sei:v",
+        "0",
+    ],
+    "preset-nvidia-h264": [
+        "-c:v",
+        "h264_nvenc",
+        "-g",
+        "50",
+        "-profile:v",
+        "high",
+        "-level:v",
+        "auto",
+        "-preset:v",
+        "p2",
+        "-tune:v",
+        "ll",
+    ],
+    "preset-nvidia-h265": [
+        "-c:v",
+        "h264_nvenc",
+        "-g",
+        "50",
+        "-profile:v",
+        "high",
+        "-level:v",
+        "auto",
+        "-preset:v",
+        "p2",
+        "-tune:v",
+        "ll",
+    ],
+    "default": [
+        "-c:v",
+        "libx264",
+        "-g",
+        "50",
+        "-profile:v",
+        "high",
+        "-level:v",
+        "4.1",
+        "-preset:v",
+        "superfast",
+        "-tune:v",
+        "zerolatency",
+    ],
+}
+
 
 def parse_preset_hardware_acceleration_decode(arg: Any) -> list[str]:
     """Return the correct preset if in preset format otherwise return None."""
@@ -158,6 +259,14 @@ def parse_preset_hardware_acceleration_scale(
     return scale
 
 
+def parse_preset_hardware_acceleration_encode(arg: Any) -> list[str]:
+    """Return the correct scaling preset or default preset if none is set."""
+    if not isinstance(arg, str):
+        return PRESETS_HW_ACCEL_ENCODE["default"]
+
+    return PRESETS_HW_ACCEL_ENCODE.get(arg, PRESETS_HW_ACCEL_ENCODE["default"])
+
+
 PRESETS_INPUT = {
     "preset-http-jpeg-generic": _user_agent_args
     + [

frigate/http.py

@@ -825,6 +825,24 @@ def latest_frame(camera_name):
 
         frame = cv2.resize(frame, dsize=(width, height), interpolation=cv2.INTER_AREA)
 
+        ret, jpg = cv2.imencode(
+            ".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), resize_quality]
+        )
+        response = make_response(jpg.tobytes())
+        response.headers["Content-Type"] = "image/jpeg"
+        response.headers["Cache-Control"] = "no-store"
+        return response
+    elif camera_name == "birdseye" and current_app.frigate_config.restream.birdseye:
+        frame = cv2.cvtColor(
+            current_app.detected_frames_processor.get_current_frame(camera_name),
+            cv2.COLOR_YUV2BGR_I420,
+        )
+
+        height = int(request.args.get("h", str(frame.shape[0])))
+        width = int(height * frame.shape[1] / frame.shape[0])
+
+        frame = cv2.resize(frame, dsize=(width, height), interpolation=cv2.INTER_AREA)
+
         ret, jpg = cv2.imencode(
             ".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), resize_quality]
         )

frigate/object_processing.py

@@ -880,6 +880,12 @@ class TrackedObjectProcessor(threading.Thread):
             return {}
 
     def get_current_frame(self, camera, draw_options={}):
+        if camera == "birdseye":
+            return self.frame_manager.get(
+                "birdseye",
+                (self.config.birdseye.height * 3 // 2, self.config.birdseye.width),
+            )
+
         return self.camera_states[camera].get_current_frame(draw_options)
 
     def get_current_frame_time(self, camera) -> int:

frigate/output.py

@@ -3,6 +3,7 @@ import glob
 import logging
 import math
 import multiprocessing as mp
+import os
 import queue
 import signal
 import subprocess as sp
@@ -21,17 +22,56 @@ from ws4py.server.wsgiutils import WebSocketWSGIApplication
 from ws4py.websocket import WebSocket
 
 from frigate.config import BirdseyeModeEnum, FrigateConfig
-from frigate.const import BASE_DIR
+from frigate.const import BASE_DIR, BIRDSEYE_PIPE
 from frigate.util import SharedMemoryFrameManager, copy_yuv_to_position, get_yuv_crop
 
 logger = logging.getLogger(__name__)
 
 
 class FFMpegConverter:
-    def __init__(self, in_width, in_height, out_width, out_height, quality):
-        ffmpeg_cmd = f"ffmpeg -f rawvideo -pix_fmt yuv420p -video_size {in_width}x{in_height} -i pipe: -f mpegts -s {out_width}x{out_height} -codec:v mpeg1video -q {quality} -bf 0 pipe:".split(
-            " "
-        )
+    def __init__(
+        self,
+        in_width: int,
+        in_height: int,
+        out_width: int,
+        out_height: int,
+        quality: int,
+        birdseye_rtsp: bool = False,
+    ):
+        if birdseye_rtsp:
+            if os.path.exists(BIRDSEYE_PIPE):
+                os.remove(BIRDSEYE_PIPE)
+
+            os.mkfifo(BIRDSEYE_PIPE, mode=0o777)
+            stdin = os.open(BIRDSEYE_PIPE, os.O_RDONLY | os.O_NONBLOCK)
+            self.bd_pipe = os.open(BIRDSEYE_PIPE, os.O_WRONLY)
+            os.close(stdin)
+        else:
+            self.bd_pipe = None
+
+        ffmpeg_cmd = [
+            "ffmpeg",
+            "-f",
+            "rawvideo",
+            "-pix_fmt",
+            "yuv420p",
+            "-video_size",
+            f"{in_width}x{in_height}",
+            "-i",
+            "pipe:",
+            "-f",
+            "mpegts",
+            "-s",
+            f"{out_width}x{out_height}",
+            "-codec:v",
+            "mpeg1video",
+            "-q",
+            f"{quality}",
+            "-bf",
+            "0",
+            "pipe:",
+        ]
+
         self.process = sp.Popen(
             ffmpeg_cmd,
             stdout=sp.PIPE,
@@ -40,9 +80,16 @@ class FFMpegConverter:
             start_new_session=True,
         )
 
-    def write(self, b):
+    def write(self, b) -> None:
         self.process.stdin.write(b)
 
+        if self.bd_pipe:
+            try:
+                os.write(self.bd_pipe, b)
+            except BrokenPipeError:
+                # catch error when no one is listening
+                return
+
     def read(self, length):
         try:
             return self.process.stdout.read1(length)
@@ -50,6 +97,9 @@ class FFMpegConverter:
             return False
 
     def exit(self):
+        if self.bd_pipe:
+            os.close(self.bd_pipe)
+
         self.process.terminate()
         try:
             self.process.communicate(timeout=30)
@@ -88,7 +138,7 @@ class BroadcastThread(threading.Thread):
 
 
 class BirdsEyeFrameManager:
-    def __init__(self, config, frame_manager: SharedMemoryFrameManager):
+    def __init__(self, config: FrigateConfig, frame_manager: SharedMemoryFrameManager):
         self.config = config
         self.mode = config.birdseye.mode
         self.frame_manager = frame_manager
@@ -386,6 +436,7 @@ def output_frames(config: FrigateConfig, video_output_queue):
             config.birdseye.width,
             config.birdseye.height,
             config.birdseye.quality,
+            config.restream.birdseye,
         )
         broadcasters["birdseye"] = BroadcastThread(
             "birdseye", converters["birdseye"], websocket_server
@@ -398,6 +449,12 @@ def output_frames(config: FrigateConfig, video_output_queue):
 
     birdseye_manager = BirdsEyeFrameManager(config, frame_manager)
 
+    if config.restream.birdseye:
+        birdseye_buffer = frame_manager.create(
+            "birdseye",
+            birdseye_manager.yuv_shape[0] * birdseye_manager.yuv_shape[1],
+        )
+
     while not stop_event.is_set():
         try:
             (
@@ -421,10 +478,12 @@ def output_frames(config: FrigateConfig, video_output_queue):
             # write to the converter for the camera if clients are listening to the specific camera
             converters[camera].write(frame.tobytes())
 
-        # update birdseye if websockets are connected
-        if config.birdseye.enabled and any(
-            ws.environ["PATH_INFO"].endswith("birdseye")
-            for ws in websocket_server.manager
+        if config.birdseye.enabled and (
+            config.restream.birdseye
+            or any(
+                ws.environ["PATH_INFO"].endswith("birdseye")
+                for ws in websocket_server.manager
+            )
         ):
             if birdseye_manager.update(
                 camera,
@@ -433,7 +492,12 @@ def output_frames(config: FrigateConfig, video_output_queue):
                 frame_time,
                 frame,
             ):
-                converters["birdseye"].write(birdseye_manager.frame.tobytes())
+                frame_bytes = birdseye_manager.frame.tobytes()
+
+                if config.restream.birdseye:
+                    birdseye_buffer[:] = frame_bytes
+
+                converters["birdseye"].write(frame_bytes)
 
         if camera in previous_frames:
             frame_manager.delete(f"{camera}{previous_frames[camera]}")

frigate/restream.py

@@ -6,6 +6,8 @@ import requests
 from frigate.util import escape_special_characters
 
 from frigate.config import FrigateConfig
+from frigate.const import BIRDSEYE_PIPE
+from frigate.ffmpeg_presets import parse_preset_hardware_acceleration_encode
 
 logger = logging.getLogger(__name__)
 
@@ -42,6 +44,11 @@ class RestreamApi:
                             escape_special_characters(input.path)
                         )
 
+        if self.config.restream.birdseye:
+            self.relays[
+                "birdseye"
+            ] = f"exec:ffmpeg -hide_banner -f rawvideo -pix_fmt yuv420p -video_size {self.config.birdseye.width}x{self.config.birdseye.height} -r 10 -i {BIRDSEYE_PIPE} {' '.join(parse_preset_hardware_acceleration_encode(self.config.ffmpeg.hwaccel_args))} -rtsp_transport tcp -f rtsp {{output}}"
+
         for name, path in self.relays.items():
             params = {"src": path, "name": name}
             requests.put("http://127.0.0.1:1984/api/streams", params=params)