Revamped debug UI and add camera / process info, ffprobe copying (#4349)

* Move each camera to a separate card and show per process info

* Install top

* Add support for cpu usage stats

* Use cpu usage stats in debug

* Increase number of runs to ensure good results

* Add ffprobe endpoint

* Get ffprobe for multiple inputs

* Copy ffprobe in output

* Add fps to camera metrics

* Fix lint errors

* Update stats config

* Add ffmpeg pid

* Use grid display so more cameras can take less vertical space

* Fix hanging characters

* Only show the current detector

* Fix bad if statement

* Return full output of ffprobe process

* Return full output of ffprobe process

* Don't specify rtsp_transport

* Make ffprobe button show dialog with output and option to copy

* Adjust ffprobe api to take paths directly

* Add docs for ffprobe api

frigate/http.py

@@ -1,8 +1,8 @@
 import base64
-from collections import OrderedDict
 from datetime import datetime, timedelta
 import copy
 import logging
+import json
 import os
 import subprocess as sp
 import time
@@ -28,9 +28,9 @@ from playhouse.shortcuts import model_to_dict
 
 from frigate.const import CLIPS_DIR
 from frigate.models import Event, Recordings
-from frigate.object_processing import TrackedObject, TrackedObjectProcessor
+from frigate.object_processing import TrackedObject
 from frigate.stats import stats_snapshot
-from frigate.util import clean_camera_user_pass
+from frigate.util import clean_camera_user_pass, ffprobe_stream
 from frigate.version import VERSION
 
 logger = logging.getLogger(__name__)
@@ -957,3 +957,37 @@ def imagestream(detected_frames_processor, camera_name, fps, height, draw_option
             b"--frame\r\n"
             b"Content-Type: image/jpeg\r\n\r\n" + jpg.tobytes() + b"\r\n\r\n"
         )
+
+
+@bp.route("/ffprobe", methods=["GET"])
+def ffprobe():
+    path_param = request.args.get("paths", "")
+
+    if not path_param:
+        return jsonify(
+            {"success": False, "message": f"Path needs to be provided."}, "404"
+        )
+
+    if "," in clean_camera_user_pass(path_param):
+        paths = path_param.split(",")
+    else:
+        paths = [path_param]
+
+    # user has multiple streams
+    output = []
+
+    for path in paths:
+        ffprobe = ffprobe_stream(path)
+        output.append(
+            {
+                "return_code": ffprobe.returncode,
+                "stderr": json.loads(ffprobe.stderr.decode("unicode_escape").strip())
+                if ffprobe.stderr.decode()
+                else {},
+                "stdout": json.loads(ffprobe.stdout.decode("unicode_escape").strip())
+                if ffprobe.stdout.decode()
+                else {},
+            }
+        )
+
+    return jsonify(output)

frigate/stats.py

@@ -14,6 +14,7 @@ from frigate.config import FrigateConfig
 from frigate.const import RECORD_DIR, CLIPS_DIR, CACHE_DIR
 from frigate.types import StatsTrackingTypes, CameraMetricsTypes
 from frigate.version import VERSION
+from frigate.util import get_cpu_stats
 from frigate.object_detection import ObjectDetectProcess
 
 logger = logging.getLogger(__name__)
@@ -90,6 +91,9 @@ def stats_snapshot(stats_tracking: StatsTrackingTypes) -> dict[str, Any]:
     for name, camera_stats in camera_metrics.items():
         total_detection_fps += camera_stats["detection_fps"].value
         pid = camera_stats["process"].pid if camera_stats["process"] else None
+        ffmpeg_pid = (
+            camera_stats["ffmpeg_pid"].value if camera_stats["ffmpeg_pid"] else None
+        )
         cpid = (
             camera_stats["capture_process"].pid
             if camera_stats["capture_process"]
@@ -102,6 +106,7 @@ def stats_snapshot(stats_tracking: StatsTrackingTypes) -> dict[str, Any]:
             "detection_fps": round(camera_stats["detection_fps"].value, 2),
             "pid": pid,
             "capture_pid": cpid,
+            "ffmpeg_pid": ffmpeg_pid,
         }
 
     stats["detectors"] = {}
@@ -114,6 +119,8 @@ def stats_snapshot(stats_tracking: StatsTrackingTypes) -> dict[str, Any]:
         }
     stats["detection_fps"] = round(total_detection_fps, 2)
 
+    stats["cpu_usages"] = get_cpu_stats()
+
     stats["service"] = {
         "uptime": (int(time.time()) - stats_tracking["started"]),
         "version": VERSION,

frigate/util.py

@@ -1,6 +1,8 @@
 import copy
 import datetime
 import logging
+import subprocess as sp
+import json
 import re
 import signal
 import traceback
@@ -679,6 +681,52 @@ def escape_special_characters(path: str) -> str:
         return path
 
 
+def get_cpu_stats() -> dict[str, dict]:
+    """Get cpu usages for each process id"""
+    usages = {}
+    # -n=2 runs to ensure extraneous values are not included
+    top_command = ["top", "-b", "-n", "2"]
+
+    p = sp.run(
+        top_command,
+        encoding="ascii",
+        capture_output=True,
+    )
+
+    if p.returncode != 0:
+        logger.error(p.stderr)
+        return usages
+    else:
+        lines = p.stdout.split("\n")
+
+        for line in lines:
+            stats = list(filter(lambda a: a != "", line.strip().split(" ")))
+            try:
+                usages[stats[0]] = {
+                    "cpu": stats[8],
+                    "mem": stats[9],
+                }
+            except:
+                continue
+
+        return usages
+
+
+def ffprobe_stream(path: str) -> sp.CompletedProcess:
+    """Run ffprobe on stream."""
+    ffprobe_cmd = [
+        "ffprobe",
+        "-print_format",
+        "json",
+        "-show_entries",
+        "stream=codec_long_name,width,height,bit_rate,duration,display_aspect_ratio,avg_frame_rate",
+        "-loglevel",
+        "quiet",
+        path,
+    ]
+    return sp.run(ffprobe_cmd, capture_output=True)
+
+
 class FrameManager(ABC):
     @abstractmethod
     def create(self, name, size) -> AnyStr: