formatting cleanup

frigate/video.py

@@ -1,12 +1,7 @@
-import base64
-import copy
-import ctypes
 import datetime
 import itertools
-import json
 import logging
 import multiprocessing as mp
-import os
 import queue
 import subprocess as sp
 import signal
@@ -16,7 +11,7 @@ from collections import defaultdict
 from setproctitle import setproctitle
 from typing import Dict, List
 
-import cv2
+from cv2 import cv2
 import numpy as np
 
 from frigate.config import CameraConfig
@@ -24,19 +19,25 @@ from frigate.edgetpu import RemoteObjectDetector
 from frigate.log import LogPipe
 from frigate.motion import MotionDetector
 from frigate.objects import ObjectTracker
-from frigate.util import (EventsPerSecond, FrameManager,
-                          SharedMemoryFrameManager, area, calculate_region,
-                          clipped, draw_box_with_label, intersection,
-                          intersection_over_union, listen, yuv_region_2_rgb)
+from frigate.util import (
+    EventsPerSecond,
+    FrameManager,
+    SharedMemoryFrameManager,
+    calculate_region,
+    clipped,
+    listen,
+    yuv_region_2_rgb,
+)
 
 logger = logging.getLogger(__name__)
 
+
 def filtered(obj, objects_to_track, object_filters):
     object_name = obj[0]
 
     if not object_name in objects_to_track:
         return True
-    
+
     if object_name in object_filters:
         obj_settings = object_filters[object_name]
 
@@ -44,7 +45,7 @@ def filtered(obj, objects_to_track, object_filters):
         # detected object, don't add it to detected objects
         if obj_settings.min_area > obj[3]:
             return True
-        
+
         # if the detected object is larger than the
         # max area, don't add it to detected objects
         if obj_settings.max_area < obj[3]:
@@ -53,29 +54,36 @@ def filtered(obj, objects_to_track, object_filters):
         # if the score is lower than the min_score, skip
         if obj_settings.min_score > obj[1]:
             return True
-    
+
         if not obj_settings.mask is None:
             # compute the coordinates of the object and make sure
             # the location isnt outside the bounds of the image (can happen from rounding)
-            y_location = min(int(obj[2][3]), len(obj_settings.mask)-1)
-            x_location = min(int((obj[2][2]-obj[2][0])/2.0)+obj[2][0], len(obj_settings.mask[0])-1)
+            y_location = min(int(obj[2][3]), len(obj_settings.mask) - 1)
+            x_location = min(
+                int((obj[2][2] - obj[2][0]) / 2.0) + obj[2][0],
+                len(obj_settings.mask[0]) - 1,
+            )
 
             # if the object is in a masked location, don't add it to detected objects
             if obj_settings.mask[y_location][x_location] == 0:
                 return True
-        
+
     return False
 
+
 def create_tensor_input(frame, model_shape, region):
     cropped_frame = yuv_region_2_rgb(frame, region)
 
     # Resize to 300x300 if needed
     if cropped_frame.shape != (model_shape[0], model_shape[1], 3):
-        cropped_frame = cv2.resize(cropped_frame, dsize=model_shape, interpolation=cv2.INTER_LINEAR)
-    
+        cropped_frame = cv2.resize(
+            cropped_frame, dsize=model_shape, interpolation=cv2.INTER_LINEAR
+        )
+
     # Expand dimensions since the model expects images to have shape: [1, height, width, 3]
     return np.expand_dims(cropped_frame, axis=0)
 
+
 def stop_ffmpeg(ffmpeg_process, logger):
     logger.info("Terminating the existing ffmpeg process...")
     ffmpeg_process.terminate()
@@ -88,18 +96,43 @@ def stop_ffmpeg(ffmpeg_process, logger):
         ffmpeg_process.communicate()
     ffmpeg_process = None
 
-def start_or_restart_ffmpeg(ffmpeg_cmd, logger, logpipe: LogPipe, frame_size=None, ffmpeg_process=None):
+
+def start_or_restart_ffmpeg(
+    ffmpeg_cmd, logger, logpipe: LogPipe, frame_size=None, ffmpeg_process=None
+):
     if not ffmpeg_process is None:
         stop_ffmpeg(ffmpeg_process, logger)
 
     if frame_size is None:
-        process = sp.Popen(ffmpeg_cmd, stdout = sp.DEVNULL, stderr=logpipe, stdin = sp.DEVNULL, start_new_session=True)
+        process = sp.Popen(
+            ffmpeg_cmd,
+            stdout=sp.DEVNULL,
+            stderr=logpipe,
+            stdin=sp.DEVNULL,
+            start_new_session=True,
+        )
     else:
-        process = sp.Popen(ffmpeg_cmd, stdout = sp.PIPE, stderr=logpipe, stdin = sp.DEVNULL, bufsize=frame_size*10, start_new_session=True)
+        process = sp.Popen(
+            ffmpeg_cmd,
+            stdout=sp.PIPE,
+            stderr=logpipe,
+            stdin=sp.DEVNULL,
+            bufsize=frame_size * 10,
+            start_new_session=True,
+        )
     return process
 
-def capture_frames(ffmpeg_process, camera_name, frame_shape, frame_manager: FrameManager, 
-    frame_queue, fps:mp.Value, skipped_fps: mp.Value, current_frame: mp.Value):
+
+def capture_frames(
+    ffmpeg_process,
+    camera_name,
+    frame_shape,
+    frame_manager: FrameManager,
+    frame_queue,
+    fps: mp.Value,
+    skipped_fps: mp.Value,
+    current_frame: mp.Value,
+):
 
     frame_size = frame_shape[0] * frame_shape[1]
     frame_rate = EventsPerSecond()
@@ -119,7 +152,9 @@ def capture_frames(ffmpeg_process, camera_name, frame_shape, frame_manager: Fram
             logger.info(f"{camera_name}: ffmpeg sent a broken frame. {e}")
 
             if ffmpeg_process.poll() != None:
-                logger.info(f"{camera_name}: ffmpeg process is not running. exiting capture thread...")
+                logger.info(
+                    f"{camera_name}: ffmpeg process is not running. exiting capture thread..."
+                )
                 frame_manager.delete(frame_name)
                 break
             continue
@@ -138,8 +173,11 @@ def capture_frames(ffmpeg_process, camera_name, frame_shape, frame_manager: Fram
         # add to the queue
         frame_queue.put(current_frame.value)
 
+
 class CameraWatchdog(threading.Thread):
-    def __init__(self, camera_name, config, frame_queue, camera_fps, ffmpeg_pid, stop_event):
+    def __init__(
+        self, camera_name, config, frame_queue, camera_fps, ffmpeg_pid, stop_event
+    ):
         threading.Thread.__init__(self)
         self.logger = logging.getLogger(f"watchdog.{camera_name}")
         self.camera_name = camera_name
@@ -159,22 +197,27 @@ class CameraWatchdog(threading.Thread):
         self.start_ffmpeg_detect()
 
         for c in self.config.ffmpeg_cmds:
-            if 'detect' in c['roles']:
+            if "detect" in c["roles"]:
                 continue
-            logpipe = LogPipe(f"ffmpeg.{self.camera_name}.{'_'.join(sorted(c['roles']))}", logging.ERROR)
-            self.ffmpeg_other_processes.append({
-                'cmd': c['cmd'],
-                'logpipe': logpipe,
-                'process': start_or_restart_ffmpeg(c['cmd'], self.logger, logpipe)
-            })
-        
+            logpipe = LogPipe(
+                f"ffmpeg.{self.camera_name}.{'_'.join(sorted(c['roles']))}",
+                logging.ERROR,
+            )
+            self.ffmpeg_other_processes.append(
+                {
+                    "cmd": c["cmd"],
+                    "logpipe": logpipe,
+                    "process": start_or_restart_ffmpeg(c["cmd"], self.logger, logpipe),
+                }
+            )
+
         time.sleep(10)
         while True:
             if self.stop_event.is_set():
                 stop_ffmpeg(self.ffmpeg_detect_process, self.logger)
                 for p in self.ffmpeg_other_processes:
-                    stop_ffmpeg(p['process'], self.logger)
-                    p['logpipe'].close()
+                    stop_ffmpeg(p["process"], self.logger)
+                    p["logpipe"].close()
                 self.logpipe.close()
                 break
 
@@ -184,7 +227,9 @@ class CameraWatchdog(threading.Thread):
                 self.logpipe.dump()
                 self.start_ffmpeg_detect()
             elif now - self.capture_thread.current_frame.value > 20:
-                self.logger.info(f"No frames received from {self.camera_name} in 20 seconds. Exiting ffmpeg...")
+                self.logger.info(
+                    f"No frames received from {self.camera_name} in 20 seconds. Exiting ffmpeg..."
+                )
                 self.ffmpeg_detect_process.terminate()
                 try:
                     self.logger.info("Waiting for ffmpeg to exit gracefully...")
@@ -193,25 +238,37 @@ class CameraWatchdog(threading.Thread):
                     self.logger.info("FFmpeg didnt exit. Force killing...")
                     self.ffmpeg_detect_process.kill()
                     self.ffmpeg_detect_process.communicate()
-            
+
             for p in self.ffmpeg_other_processes:
-                poll = p['process'].poll()
+                poll = p["process"].poll()
                 if poll == None:
                     continue
-                p['logpipe'].dump()
-                p['process'] = start_or_restart_ffmpeg(p['cmd'], self.logger, p['logpipe'], ffmpeg_process=p['process'])
-            
+                p["logpipe"].dump()
+                p["process"] = start_or_restart_ffmpeg(
+                    p["cmd"], self.logger, p["logpipe"], ffmpeg_process=p["process"]
+                )
+
             # wait a bit before checking again
             time.sleep(10)
-    
+
     def start_ffmpeg_detect(self):
-        ffmpeg_cmd = [c['cmd'] for c in self.config.ffmpeg_cmds if 'detect' in c['roles']][0]
-        self.ffmpeg_detect_process = start_or_restart_ffmpeg(ffmpeg_cmd, self.logger, self.logpipe, self.frame_size)
+        ffmpeg_cmd = [
+            c["cmd"] for c in self.config.ffmpeg_cmds if "detect" in c["roles"]
+        ][0]
+        self.ffmpeg_detect_process = start_or_restart_ffmpeg(
+            ffmpeg_cmd, self.logger, self.logpipe, self.frame_size
+        )
         self.ffmpeg_pid.value = self.ffmpeg_detect_process.pid
-        self.capture_thread = CameraCapture(self.camera_name, self.ffmpeg_detect_process, self.frame_shape, self.frame_queue, 
-            self.camera_fps)
+        self.capture_thread = CameraCapture(
+            self.camera_name,
+            self.ffmpeg_detect_process,
+            self.frame_shape,
+            self.frame_queue,
+            self.camera_fps,
+        )
         self.capture_thread.start()
 
+
 class CameraCapture(threading.Thread):
     def __init__(self, camera_name, ffmpeg_process, frame_shape, frame_queue, fps):
         threading.Thread.__init__(self)
@@ -223,32 +280,59 @@ class CameraCapture(threading.Thread):
         self.skipped_fps = EventsPerSecond()
         self.frame_manager = SharedMemoryFrameManager()
         self.ffmpeg_process = ffmpeg_process
-        self.current_frame = mp.Value('d', 0.0)
+        self.current_frame = mp.Value("d", 0.0)
         self.last_frame = 0
 
     def run(self):
         self.skipped_fps.start()
-        capture_frames(self.ffmpeg_process, self.camera_name, self.frame_shape, self.frame_manager, self.frame_queue,
-            self.fps, self.skipped_fps, self.current_frame)
+        capture_frames(
+            self.ffmpeg_process,
+            self.camera_name,
+            self.frame_shape,
+            self.frame_manager,
+            self.frame_queue,
+            self.fps,
+            self.skipped_fps,
+            self.current_frame,
+        )
+
 
 def capture_camera(name, config: CameraConfig, process_info):
     stop_event = mp.Event()
+
     def receiveSignal(signalNumber, frame):
         stop_event.set()
-    
+
     signal.signal(signal.SIGTERM, receiveSignal)
     signal.signal(signal.SIGINT, receiveSignal)
 
-    frame_queue = process_info['frame_queue']
-    camera_watchdog = CameraWatchdog(name, config, frame_queue, process_info['camera_fps'], process_info['ffmpeg_pid'], stop_event)
+    frame_queue = process_info["frame_queue"]
+    camera_watchdog = CameraWatchdog(
+        name,
+        config,
+        frame_queue,
+        process_info["camera_fps"],
+        process_info["ffmpeg_pid"],
+        stop_event,
+    )
     camera_watchdog.start()
     camera_watchdog.join()
 
-def track_camera(name, config: CameraConfig, model_shape, detection_queue, result_connection, detected_objects_queue, process_info):
+
+def track_camera(
+    name,
+    config: CameraConfig,
+    model_shape,
+    detection_queue,
+    result_connection,
+    detected_objects_queue,
+    process_info,
+):
     stop_event = mp.Event()
+
     def receiveSignal(signalNumber, frame):
         stop_event.set()
-    
+
     signal.signal(signal.SIGTERM, receiveSignal)
     signal.signal(signal.SIGINT, receiveSignal)
 
@@ -256,71 +340,113 @@ def track_camera(name, config: CameraConfig, model_shape, detection_queue, resul
     setproctitle(f"frigate.process:{name}")
     listen()
 
-    frame_queue = process_info['frame_queue']
-    detection_enabled = process_info['detection_enabled']
+    frame_queue = process_info["frame_queue"]
+    detection_enabled = process_info["detection_enabled"]
 
     frame_shape = config.frame_shape
     objects_to_track = config.objects.track
     object_filters = config.objects.filters
 
     motion_detector = MotionDetector(frame_shape, config.motion)
-    object_detector = RemoteObjectDetector(name, '/labelmap.txt', detection_queue, result_connection, model_shape)
+    object_detector = RemoteObjectDetector(
+        name, "/labelmap.txt", detection_queue, result_connection, model_shape
+    )
 
     object_tracker = ObjectTracker(config.detect)
 
     frame_manager = SharedMemoryFrameManager()
 
-    process_frames(name, frame_queue, frame_shape, model_shape, frame_manager, motion_detector, object_detector,
-        object_tracker, detected_objects_queue, process_info, objects_to_track, object_filters, detection_enabled, stop_event)
+    process_frames(
+        name,
+        frame_queue,
+        frame_shape,
+        model_shape,
+        frame_manager,
+        motion_detector,
+        object_detector,
+        object_tracker,
+        detected_objects_queue,
+        process_info,
+        objects_to_track,
+        object_filters,
+        detection_enabled,
+        stop_event,
+    )
 
     logger.info(f"{name}: exiting subprocess")
 
+
 def reduce_boxes(boxes):
     if len(boxes) == 0:
         return []
-    reduced_boxes = cv2.groupRectangles([list(b) for b in itertools.chain(boxes, boxes)], 1, 0.2)[0]
+    reduced_boxes = cv2.groupRectangles(
+        [list(b) for b in itertools.chain(boxes, boxes)], 1, 0.2
+    )[0]
     return [tuple(b) for b in reduced_boxes]
 
+
 # modified from https://stackoverflow.com/a/40795835
 def intersects_any(box_a, boxes):
     for box in boxes:
-        if box_a[2] < box[0] or box_a[0] > box[2] or box_a[1] > box[3] or box_a[3] < box[1]:
+        if (
+            box_a[2] < box[0]
+            or box_a[0] > box[2]
+            or box_a[1] > box[3]
+            or box_a[3] < box[1]
+        ):
             continue
         return True
 
-def detect(object_detector, frame, model_shape, region, objects_to_track, object_filters):
+
+def detect(
+    object_detector, frame, model_shape, region, objects_to_track, object_filters
+):
     tensor_input = create_tensor_input(frame, model_shape, region)
 
     detections = []
     region_detections = object_detector.detect(tensor_input)
     for d in region_detections:
         box = d[2]
-        size = region[2]-region[0]
+        size = region[2] - region[0]
         x_min = int((box[1] * size) + region[0])
         y_min = int((box[0] * size) + region[1])
         x_max = int((box[3] * size) + region[0])
         y_max = int((box[2] * size) + region[1])
-        det = (d[0],
+        det = (
+            d[0],
             d[1],
             (x_min, y_min, x_max, y_max),
-            (x_max-x_min)*(y_max-y_min),
-            region)
+            (x_max - x_min) * (y_max - y_min),
+            region,
+        )
         # apply object filters
         if filtered(det, objects_to_track, object_filters):
             continue
         detections.append(det)
     return detections
 
-def process_frames(camera_name: str, frame_queue: mp.Queue, frame_shape, model_shape,
-    frame_manager: FrameManager, motion_detector: MotionDetector, 
-    object_detector: RemoteObjectDetector, object_tracker: ObjectTracker,
-    detected_objects_queue: mp.Queue, process_info: Dict,
-    objects_to_track: List[str], object_filters, detection_enabled: mp.Value, stop_event,
-    exit_on_empty: bool = False):
-    
-    fps = process_info['process_fps']
-    detection_fps = process_info['detection_fps']
-    current_frame_time = process_info['detection_frame']
+
+def process_frames(
+    camera_name: str,
+    frame_queue: mp.Queue,
+    frame_shape,
+    model_shape,
+    frame_manager: FrameManager,
+    motion_detector: MotionDetector,
+    object_detector: RemoteObjectDetector,
+    object_tracker: ObjectTracker,
+    detected_objects_queue: mp.Queue,
+    process_info: Dict,
+    objects_to_track: List[str],
+    object_filters,
+    detection_enabled: mp.Value,
+    stop_event,
+    exit_on_empty: bool = False,
+):
+
+    fps = process_info["process_fps"]
+    detection_fps = process_info["detection_fps"]
+    current_frame_time = process_info["detection_frame"]
 
     fps_tracker = EventsPerSecond()
     fps_tracker.start()
@@ -340,7 +466,9 @@ def process_frames(camera_name: str, frame_queue: mp.Queue, frame_shape, model_s
 
         current_frame_time.value = frame_time
 
-        frame = frame_manager.get(f"{camera_name}{frame_time}", (frame_shape[0]*3//2, frame_shape[1]))
+        frame = frame_manager.get(
+            f"{camera_name}{frame_time}", (frame_shape[0] * 3 // 2, frame_shape[1])
+        )
 
         if frame is None:
             logger.info(f"{camera_name}: frame {frame_time} is not in memory store.")
@@ -349,7 +477,9 @@ def process_frames(camera_name: str, frame_queue: mp.Queue, frame_shape, model_s
         if not detection_enabled.value:
             fps.value = fps_tracker.eps()
             object_tracker.match_and_update(frame_time, [])
-            detected_objects_queue.put((camera_name, frame_time, object_tracker.tracked_objects, [], []))
+            detected_objects_queue.put(
+                (camera_name, frame_time, object_tracker.tracked_objects, [], [])
+            )
             detection_fps.value = object_detector.fps.eps()
             frame_manager.close(f"{camera_name}{frame_time}")
             continue
@@ -358,27 +488,44 @@ def process_frames(camera_name: str, frame_queue: mp.Queue, frame_shape, model_s
         motion_boxes = motion_detector.detect(frame)
 
         # only get the tracked object boxes that intersect with motion
-        tracked_object_boxes = [obj['box'] for obj in object_tracker.tracked_objects.values() if intersects_any(obj['box'], motion_boxes)]
+        tracked_object_boxes = [
+            obj["box"]
+            for obj in object_tracker.tracked_objects.values()
+            if intersects_any(obj["box"], motion_boxes)
+        ]
 
         # combine motion boxes with known locations of existing objects
         combined_boxes = reduce_boxes(motion_boxes + tracked_object_boxes)
 
         # compute regions
-        regions = [calculate_region(frame_shape, a[0], a[1], a[2], a[3], 1.2)
-            for a in combined_boxes]
+        regions = [
+            calculate_region(frame_shape, a[0], a[1], a[2], a[3], 1.2)
+            for a in combined_boxes
+        ]
 
         # combine overlapping regions
         combined_regions = reduce_boxes(regions)
 
         # re-compute regions
-        regions = [calculate_region(frame_shape, a[0], a[1], a[2], a[3], 1.0)
-            for a in combined_regions]
+        regions = [
+            calculate_region(frame_shape, a[0], a[1], a[2], a[3], 1.0)
+            for a in combined_regions
+        ]
 
         # resize regions and detect
         detections = []
         for region in regions:
-            detections.extend(detect(object_detector, frame, model_shape, region, objects_to_track, object_filters))
-        
+            detections.extend(
+                detect(
+                    object_detector,
+                    frame,
+                    model_shape,
+                    region,
+                    objects_to_track,
+                    object_filters,
+                )
+            )
+
         #########
         # merge objects, check for clipped objects and look again up to 4 times
         #########
@@ -396,8 +543,10 @@ def process_frames(camera_name: str, frame_queue: mp.Queue, frame_shape, model_s
             for group in detected_object_groups.values():
 
                 # apply non-maxima suppression to suppress weak, overlapping bounding boxes
-                boxes = [(o[2][0], o[2][1], o[2][2]-o[2][0], o[2][3]-o[2][1])
-                    for o in group]
+                boxes = [
+                    (o[2][0], o[2][1], o[2][2] - o[2][0], o[2][3] - o[2][1])
+                    for o in group
+                ]
                 confidences = [o[1] for o in group]
                 idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
 
@@ -406,17 +555,26 @@ def process_frames(camera_name: str, frame_queue: mp.Queue, frame_shape, model_s
                     if clipped(obj, frame_shape):
                         box = obj[2]
                         # calculate a new region that will hopefully get the entire object
-                        region = calculate_region(frame_shape, 
-                            box[0], box[1],
-                            box[2], box[3])
+                        region = calculate_region(
+                            frame_shape, box[0], box[1], box[2], box[3]
+                        )
 
                         regions.append(region)
-                        
-                        selected_objects.extend(detect(object_detector, frame, model_shape, region, objects_to_track, object_filters))
+
+                        selected_objects.extend(
+                            detect(
+                                object_detector,
+                                frame,
+                                model_shape,
+                                region,
+                                objects_to_track,
+                                object_filters,
+                            )
+                        )
 
                         refining = True
                     else:
-                        selected_objects.append(obj)            
+                        selected_objects.append(obj)
             # set the detections list to only include top, complete objects
             # and new detections
             detections = selected_objects
@@ -426,18 +584,28 @@ def process_frames(camera_name: str, frame_queue: mp.Queue, frame_shape, model_s
 
         # Limit to the detections overlapping with motion areas
         # to avoid picking up stationary background objects
-        detections_with_motion = [d for d in detections if intersects_any(d[2], motion_boxes)]
+        detections_with_motion = [
+            d for d in detections if intersects_any(d[2], motion_boxes)
+        ]
 
         # now that we have refined our detections, we need to track objects
         object_tracker.match_and_update(frame_time, detections_with_motion)
 
         # add to the queue if not full
-        if(detected_objects_queue.full()):
+        if detected_objects_queue.full():
             frame_manager.delete(f"{camera_name}{frame_time}")
             continue
         else:
             fps_tracker.update()
             fps.value = fps_tracker.eps()
-            detected_objects_queue.put((camera_name, frame_time, object_tracker.tracked_objects, motion_boxes, regions))
+            detected_objects_queue.put(
+                (
+                    camera_name,
+                    frame_time,
+                    object_tracker.tracked_objects,
+                    motion_boxes,
+                    regions,
+                )
+            )
             detection_fps.value = object_detector.fps.eps()
             frame_manager.close(f"{camera_name}{frame_time}")