ensure detection_start doesnt change values between conditions

config/config.example.yml

@@ -110,13 +110,6 @@ cameras:
     ################
     take_frame: 1
 
-    ################
-    # The expected framerate for the camera. Frigate will try and ensure it maintains this framerate
-    # by dropping frames as necessary. Setting this lower than the actual framerate will allow frigate
-    # to process every frame at the expense of realtime processing.
-    ################
-    fps: 5
-
     ################
     # Configuration for the snapshots in the debug view and mqtt
     ################

detect_objects.py

@@ -63,7 +63,7 @@ DEBUG = (CONFIG.get('debug', '0') == '1')
 
 def start_plasma_store():
     plasma_cmd = ['plasma_store', '-m', '400000000', '-s', '/tmp/plasma']
-    plasma_process = sp.Popen(plasma_cmd, stdout=sp.DEVNULL)
+    plasma_process = sp.Popen(plasma_cmd, stdout=sp.DEVNULL, stderr=sp.DEVNULL)
     time.sleep(1)
     rc = plasma_process.poll()
     if rc is not None:
@@ -92,8 +92,9 @@ class CameraWatchdog(threading.Thread):
                 self.plasma_process = start_plasma_store()
 
             # check the detection process
-            if (self.tflite_process.detection_start.value > 0.0 and 
-                datetime.datetime.now().timestamp() - self.tflite_process.detection_start.value > 10):
+            detection_start = self.tflite_process.detection_start.value
+            if (detection_start > 0.0 and 
+                datetime.datetime.now().timestamp() - detection_start > 10):
                 print("Detection appears to be stuck. Restarting detection process")
                 self.tflite_process.start_or_restart()
             elif not self.tflite_process.detect_process.is_alive():
@@ -105,14 +106,13 @@ class CameraWatchdog(threading.Thread):
                 process = camera_process['process']
                 if not process.is_alive():
                     print(f"Track process for {name} is not alive. Starting again...")
-                    camera_process['fps'].value = float(self.config[name]['fps'])
-                    camera_process['skipped_fps'].value = 0.0
+                    camera_process['process_fps'].value = 0.0
                     camera_process['detection_fps'].value = 0.0
                     camera_process['read_start'].value = 0.0
                     process = mp.Process(target=track_camera, args=(name, self.config[name], GLOBAL_OBJECT_CONFIG, camera_process['frame_queue'],
                         camera_process['frame_shape'], self.tflite_process.detection_queue, self.tracked_objects_queue, 
-                        camera_process['fps'], camera_process['skipped_fps'], camera_process['detection_fps'],
-                        camera_process['read_start']))
+                        camera_process['process_fps'], camera_process['detection_fps'],
+                        camera_process['read_start'], camera_process['detection_frame']))
                     process.daemon = True
                     camera_process['process'] = process
                     process.start()
@@ -123,7 +123,7 @@ class CameraWatchdog(threading.Thread):
                     frame_size = frame_shape[0] * frame_shape[1] * frame_shape[2]
                     ffmpeg_process = start_or_restart_ffmpeg(camera_process['ffmpeg_cmd'], frame_size)
                     camera_capture = CameraCapture(name, ffmpeg_process, frame_shape, camera_process['frame_queue'], 
-                        camera_process['take_frame'], camera_process['camera_fps'])
+                        camera_process['take_frame'], camera_process['camera_fps'], camera_process['detection_frame'])
                     camera_capture.start()
                     camera_process['ffmpeg_process'] = ffmpeg_process
                     camera_process['capture_thread'] = camera_capture
@@ -193,19 +193,21 @@ def main():
         frame_size = frame_shape[0] * frame_shape[1] * frame_shape[2]
         take_frame = config.get('take_frame', 1)
 
+        detection_frame = mp.Value('d', 0.0)
+
         ffmpeg_process = start_or_restart_ffmpeg(ffmpeg_cmd, frame_size)
         frame_queue = mp.SimpleQueue()
         camera_fps = EventsPerSecond()
         camera_fps.start()
-        camera_capture = CameraCapture(name, ffmpeg_process, frame_shape, frame_queue, take_frame, camera_fps)
+        camera_capture = CameraCapture(name, ffmpeg_process, frame_shape, frame_queue, take_frame, camera_fps, detection_frame)
         camera_capture.start()
 
         camera_processes[name] = {
             'camera_fps': camera_fps,
             'take_frame': take_frame,
-            'fps': mp.Value('d', float(config['fps'])),
-            'skipped_fps': mp.Value('d', 0.0),
+            'process_fps': mp.Value('d', 0.0),
             'detection_fps': mp.Value('d', 0.0),
+            'detection_frame': detection_frame,
             'read_start': mp.Value('d', 0.0),
             'ffmpeg_process': ffmpeg_process,
             'ffmpeg_cmd': ffmpeg_cmd,
@@ -215,9 +217,9 @@ def main():
         }
 
         camera_process = mp.Process(target=track_camera, args=(name, config, GLOBAL_OBJECT_CONFIG, frame_queue, frame_shape,
-            tflite_process.detection_queue, tracked_objects_queue, camera_processes[name]['fps'], 
-            camera_processes[name]['skipped_fps'], camera_processes[name]['detection_fps'], 
-            camera_processes[name]['read_start']))
+            tflite_process.detection_queue, tracked_objects_queue, camera_processes[name]['process_fps'], 
+            camera_processes[name]['detection_fps'], 
+            camera_processes[name]['read_start'], camera_processes[name]['detection_frame']))
         camera_process.daemon = True
         camera_processes[name]['process'] = camera_process
 
@@ -266,13 +268,20 @@ def main():
 
         for name, camera_stats in camera_processes.items():
             total_detection_fps += camera_stats['detection_fps'].value
+            capture_thread = camera_stats['capture_thread']
             stats[name] = {
-                'fps': round(camera_stats['fps'].value, 2),
-                'skipped_fps': round(camera_stats['skipped_fps'].value, 2),
+                'camera_fps': round(capture_thread.fps.eps(), 2),
+                'process_fps': round(camera_stats['process_fps'].value, 2),
+                'skipped_fps': round(capture_thread.skipped_fps.eps(), 2),
                 'detection_fps': round(camera_stats['detection_fps'].value, 2),
                 'read_start': camera_stats['read_start'].value,
                 'pid': camera_stats['process'].pid,
-                'ffmpeg_pid': camera_stats['ffmpeg_process'].pid
+                'ffmpeg_pid': camera_stats['ffmpeg_process'].pid,
+                'frame_info': {
+                    'read': capture_thread.current_frame,
+                    'detect': camera_stats['detection_frame'].value,
+                    'process': object_processor.camera_data[name]['current_frame_time']
+                }
             }
         
         stats['coral'] = {
@@ -320,7 +329,9 @@ def main():
             if frame is None:
                 frame = np.zeros((height,int(height*16/9),3), np.uint8)
 
-            frame = cv2.resize(frame, dsize=(int(height*16/9), height), interpolation=cv2.INTER_LINEAR)
+            width = int(height*frame.shape[1]/frame.shape[0])
+
+            frame = cv2.resize(frame, dsize=(width, height), interpolation=cv2.INTER_LINEAR)
             frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
 
             ret, jpg = cv2.imencode('.jpg', frame)

frigate/object_processing.py

@@ -34,6 +34,7 @@ class TrackedObjectProcessor(threading.Thread):
             'object_status': defaultdict(lambda: defaultdict(lambda: 'OFF')),
             'tracked_objects': {},
             'current_frame': np.zeros((720,1280,3), np.uint8),
+            'current_frame_time': 0.0,
             'object_id': None
         })
         self.plasma_client = PlasmaManager()
@@ -55,6 +56,7 @@ class TrackedObjectProcessor(threading.Thread):
             best_objects = self.camera_data[camera]['best_objects']
             current_object_status = self.camera_data[camera]['object_status']
             self.camera_data[camera]['tracked_objects'] = tracked_objects
+            self.camera_data[camera]['current_frame_time'] = frame_time
 
             ###
             # Draw tracked objects on the frame
@@ -83,14 +85,13 @@ class TrackedObjectProcessor(threading.Thread):
                     cv2.putText(current_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
 
                 ###
-                # Set the current frame as ready
+                # Set the current frame
                 ###
                 self.camera_data[camera]['current_frame'] = current_frame
 
-                # store the object id, so you can delete it at the next loop
-                previous_object_id = f"{camera}{frame_time}"
-                if not previous_object_id is None:
-                    self.plasma_client.delete(f"{camera}{frame_time}")
+                # delete the previous frame from the plasma store and update the object id
+                if not self.camera_data[camera]['object_id'] is None:
+                    self.plasma_client.delete(self.camera_data[camera]['object_id'])
                 self.camera_data[camera]['object_id'] = f"{camera}{frame_time}"
             
             ###

frigate/video.py

@@ -115,7 +115,7 @@ def start_or_restart_ffmpeg(ffmpeg_cmd, frame_size, ffmpeg_process=None):
     return process
 
 class CameraCapture(threading.Thread):
-    def __init__(self, name, ffmpeg_process, frame_shape, frame_queue, take_frame, fps):
+    def __init__(self, name, ffmpeg_process, frame_shape, frame_queue, take_frame, fps, detection_frame):
         threading.Thread.__init__(self)
         self.name = name
         self.frame_shape = frame_shape
@@ -123,42 +123,56 @@ class CameraCapture(threading.Thread):
         self.frame_queue = frame_queue
         self.take_frame = take_frame
         self.fps = fps
+        self.skipped_fps = EventsPerSecond()
         self.plasma_client = PlasmaManager()
         self.ffmpeg_process = ffmpeg_process
+        self.current_frame = 0
+        self.last_frame = 0
+        self.detection_frame = detection_frame
 
     def run(self):
         frame_num = 0
+        self.skipped_fps.start()
         while True:
             if self.ffmpeg_process.poll() != None:
                 print(f"{self.name}: ffmpeg process is not running. exiting capture thread...")
                 break
 
             frame_bytes = self.ffmpeg_process.stdout.read(self.frame_size)
-            frame_time = datetime.datetime.now().timestamp()
+            self.current_frame = datetime.datetime.now().timestamp()
 
             if len(frame_bytes) == 0:
                 print(f"{self.name}: ffmpeg didnt return a frame. something is wrong.")
                 continue
 
+            self.fps.update()
+
             frame_num += 1
             if (frame_num % self.take_frame) != 0:
+                self.skipped_fps.update()
+                continue
+
+            # if the detection process is more than 1 second behind, skip this frame
+            if self.detection_frame.value > 0.0 and (self.last_frame - self.detection_frame.value) > 1:
+                self.skipped_fps.update()
                 continue
 
             # put the frame in the plasma store
-            self.plasma_client.put(f"{self.name}{frame_time}",
+            self.plasma_client.put(f"{self.name}{self.current_frame}",
                     np
                         .frombuffer(frame_bytes, np.uint8)
                         .reshape(self.frame_shape)
                 )
             # add to the queue
-            self.frame_queue.put(frame_time)
+            self.frame_queue.put(self.current_frame)
+            self.last_frame = self.current_frame
 
-            self.fps.update()
-
-def track_camera(name, config, global_objects_config, frame_queue, frame_shape, detection_queue, detected_objects_queue, fps, skipped_fps, detection_fps, read_start):
+def track_camera(name, config, global_objects_config, frame_queue, frame_shape, detection_queue, detected_objects_queue, fps, detection_fps, read_start, detection_frame):
     print(f"Starting process for {name}: {os.getpid()}")
     listen()
 
+    detection_frame.value = 0.0
+
     # Merge the tracked object config with the global config
     camera_objects_config = config.get('objects', {})    
     # combine tracked objects lists
@@ -171,8 +185,6 @@ def track_camera(name, config, global_objects_config, frame_queue, frame_shape,
     for obj in objects_with_config:
         object_filters[obj] = {**global_object_filters.get(obj, {}), **camera_object_filters.get(obj, {})}
 
-    expected_fps = config['fps']
-
     frame = np.zeros(frame_shape, np.uint8)
 
     # load in the mask for object detection
@@ -191,12 +203,9 @@ def track_camera(name, config, global_objects_config, frame_queue, frame_shape,
     object_tracker = ObjectTracker(10)
 
     plasma_client = PlasmaManager()
-    frame_num = 0
     avg_wait = 0.0
     fps_tracker = EventsPerSecond()
-    skipped_fps_tracker = EventsPerSecond()
     fps_tracker.start()
-    skipped_fps_tracker.start()
     object_detector.fps.start()
     while True:
         read_start.value = datetime.datetime.now().timestamp()
@@ -204,31 +213,21 @@ def track_camera(name, config, global_objects_config, frame_queue, frame_shape,
         duration = datetime.datetime.now().timestamp()-read_start.value
         read_start.value = 0.0
         avg_wait = (avg_wait*99+duration)/100
-
-        fps_tracker.update()
-        fps.value = fps_tracker.eps()
-        detection_fps.value = object_detector.fps.eps()
+        detection_frame.value = frame_time
         
         # Get frame from plasma store
         frame = plasma_client.get(f"{name}{frame_time}")
 
         if frame is plasma.ObjectNotAvailable:
-            skipped_fps_tracker.update()
-            skipped_fps.value = skipped_fps_tracker.eps()
             continue
+
+        fps_tracker.update()
+        fps.value = fps_tracker.eps()
+        detection_fps.value = object_detector.fps.eps()
         
         # look for motion
         motion_boxes = motion_detector.detect(frame)
 
-        # skip object detection if we are below the min_fps and wait time is less than half the average
-        if frame_num > 100 and fps.value < expected_fps-1 and duration < 0.5*avg_wait:
-            skipped_fps_tracker.update()
-            skipped_fps.value = skipped_fps_tracker.eps()
-            plasma_client.delete(f"{name}{frame_time}")
-            continue
-        
-        skipped_fps.value = skipped_fps_tracker.eps()
-
         tracked_objects = object_tracker.tracked_objects.values()
 
         # merge areas of motion that intersect with a known tracked object into a single area to look at