Update README.md

Add MQTT connection error handling

.github/FUNDING.yml

@@ -0,0 +1 @@
+ko_fi: blakeblackshear

README.md

@@ -1,3 +1,5 @@
+<a href='https://ko-fi.com/P5P7XGO9' target='_blank'><img height='36' style='border:0px;height:36px;' src='https://az743702.vo.msecnd.net/cdn/kofi4.png?v=2' border='0' alt='Buy Me a Coffee at ko-fi.com' /></a>
+
 # Frigate - Realtime Object Detection for RTSP Cameras
 **Note:** This version requires the use of a [Google Coral USB Accelerator](https://coral.withgoogle.com/products/accelerator/)
 
@@ -62,12 +64,12 @@ camera:
     platform: generic
     still_image_url: http://<ip>:5000/<camera_name>/best_person.jpg
 
-sensor:
+binary_sensor:
   - name: Camera Person
     platform: mqtt
     state_topic: "frigate/<camera_name>/objects"
     value_template: '{{ value_json.person }}'
-    device_class: moving
+    device_class: motion
     availability_topic: "frigate/available"
 ```
 

config/config.yml

@@ -3,6 +3,8 @@ web_port: 5000
 mqtt:
   host: mqtt.server.com
   topic_prefix: frigate
+#  user: username # Optional -- Uncomment for use
+#  password: password # Optional -- Uncomment for use
 
 cameras:
   back:
@@ -13,30 +15,20 @@ cameras:
       # values that begin with a "$" will be replaced with environment variable
       password: $RTSP_PASSWORD
       path: /cam/realmonitor?channel=1&subtype=2
+    mask: back-mask.bmp
     regions:
       - size: 350
         x_offset: 0
         y_offset: 300
+        min_person_area: 5000
+        threshold: 0.5
       - size: 400
         x_offset: 350
         y_offset: 250
+        min_person_area: 2000
+        threshold: 0.5
       - size: 400
         x_offset: 750
         y_offset: 250
-    mask: back-mask.bmp
-    known_sizes:
-      - y: 300
-        min: 700
-        max: 1800
-      - y: 400
-        min: 3000
-        max: 7200
-      - y: 500
-        min: 8500
-        max: 20400
-      - y: 600
-        min: 10000
-        max: 50000
-      - y: 700
-        min: 10000
-        max: 125000
+        min_person_area: 2000
+        threshold: 0.5

detect_objects.py

@@ -25,6 +25,15 @@ def main():
     # connect to mqtt and setup last will
     def on_connect(client, userdata, flags, rc):
         print("On connect called")
+        if rc != 0:
+            if rc == 3:
+                print ("MQTT Server unavailable")
+            elif rc == 4:
+                print ("MQTT Bad username or password")
+            elif rc == 5:
+                print ("MQTT Not authorized")
+            else:
+                print ("Unable to connect to MQTT: Connection refused. Error code: " + str(rc))
         # publish a message to signal that the service is running
         client.publish(MQTT_TOPIC_PREFIX+'/available', 'online', retain=True)
     client = mqtt.Client()
@@ -36,12 +45,12 @@ def main():
     client.loop_start()
     
     # Queue for prepped frames, max size set to (number of cameras * 5)
-    max_queue_size = len(CONFIG['cameras'].items())*10
+    max_queue_size = len(CONFIG['cameras'].items())*5
     prepped_frame_queue = queue.Queue(max_queue_size)
 
     cameras = {}
     for name, config in CONFIG['cameras'].items():
-        cameras[name] = Camera(name, config, prepped_frame_queue, client, MQTT_TOPIC_PREFIX, DEBUG)
+        cameras[name] = Camera(name, config, prepped_frame_queue, client, MQTT_TOPIC_PREFIX)
 
     prepped_queue_processor = PreppedQueueProcessor(
         cameras,
@@ -87,4 +96,4 @@ def main():
     camera.join()
 
 if __name__ == '__main__':
-    main()
+    main()

frigate/object_detection.py

@@ -38,7 +38,7 @@ class PreppedQueueProcessor(threading.Thread):
             frame = self.prepped_frame_queue.get()
 
             # Actual detection.
-            objects = self.engine.DetectWithInputTensor(frame['frame'], threshold=0.5, top_k=3)
+            objects = self.engine.DetectWithInputTensor(frame['frame'], threshold=frame['region_threshold'], top_k=3)
             # parse and pass detected objects back to the camera
             parsed_objects = []
             for obj in objects:
@@ -59,7 +59,7 @@ class PreppedQueueProcessor(threading.Thread):
 class FramePrepper(threading.Thread):
     def __init__(self, camera_name, shared_frame, frame_time, frame_ready, 
         frame_lock,
-        region_size, region_x_offset, region_y_offset,
+        region_size, region_x_offset, region_y_offset, region_threshold,
         prepped_frame_queue):
 
         threading.Thread.__init__(self)
@@ -71,6 +71,7 @@ class FramePrepper(threading.Thread):
         self.region_size = region_size
         self.region_x_offset = region_x_offset
         self.region_y_offset = region_y_offset
+        self.region_threshold = region_threshold
         self.prepped_frame_queue = prepped_frame_queue
 
     def run(self):
@@ -103,6 +104,7 @@ class FramePrepper(threading.Thread):
                     'frame_time': frame_time,
                     'frame': frame_expanded.flatten().copy(),
                     'region_size': self.region_size,
+                    'region_threshold': self.region_threshold,
                     'region_x_offset': self.region_x_offset,
                     'region_y_offset': self.region_y_offset
                 })

frigate/video.py

@@ -13,18 +13,18 @@ from . objects import ObjectCleaner, BestPersonFrame
 from . mqtt import MqttObjectPublisher
 
 # fetch the frames as fast a possible and store current frame in a shared memory array
-def fetch_frames(shared_arr, shared_frame_time, frame_lock, frame_ready, frame_shape, rtsp_url, take_frame=1):
+def fetch_frames(shared_arr, shared_frame_time, frame_lock, frame_ready, frame_shape, rtsp_url):
     # convert shared memory array into numpy and shape into image array
     arr = tonumpyarray(shared_arr).reshape(frame_shape)
 
     # start the video capture
     video = cv2.VideoCapture()
     video.open(rtsp_url)
+    print("Opening the RTSP Url...")
     # keep the buffer small so we minimize old data
     video.set(cv2.CAP_PROP_BUFFERSIZE,1)
 
     bad_frame_counter = 0
-    frame_num = 0
     while True:
         # check if the video stream is still open, and reopen if needed
         if not video.isOpened():
@@ -37,9 +37,6 @@ def fetch_frames(shared_arr, shared_frame_time, frame_lock, frame_ready, frame_s
         # snapshot the time the frame was grabbed
         frame_time = datetime.datetime.now()
         if ret:
-            frame_num += 1
-            if (frame_num % take_frame) != 0:
-                continue
             # go ahead and decode the current frame
             ret, frame = video.retrieve()
             if ret:
@@ -113,70 +110,33 @@ def get_rtsp_url(rtsp_config):
         rtsp_config['password'], rtsp_config['host'], rtsp_config['port'],
         rtsp_config['path'])
 
-def compute_sizes(frame_shape, known_sizes, mask):
-    # create a 3 dimensional numpy array to store estimated sizes
-    estimated_sizes = np.zeros((frame_shape[0], frame_shape[1], 2), np.uint32)
+class CameraWatchdog(threading.Thread):
+    def __init__(self, camera):
+        threading.Thread.__init__(self)
+        self.camera = camera
 
-    sorted_positions = sorted(known_sizes, key=lambda s: s['y'])
+    def run(self):
 
-    last_position = {'y': 0, 'min': 0, 'max': 0}
-    next_position = sorted_positions.pop(0)
-    # if the next position has the same y coordinate, skip
-    while next_position['y'] == last_position['y']:
-        next_position = sorted_positions.pop(0)
-    y_change = next_position['y']-last_position['y']
-    min_size_change = next_position['min']-last_position['min']
-    max_size_change = next_position['max']-last_position['max']
-    min_step_size = min_size_change/y_change
-    max_step_size = max_size_change/y_change
+        while True:
+            # wait a bit before checking
+            time.sleep(60)
 
-    min_current_size = 0
-    max_current_size = 0
-
-    for y_position in range(frame_shape[0]):
-        # fill the row with the estimated size
-        estimated_sizes[y_position,:] = [min_current_size, max_current_size]
-
-        # if you have reached the next size
-        if y_position == next_position['y']:
-            last_position = next_position
-            # if there are still positions left
-            if len(sorted_positions) > 0:
-                next_position = sorted_positions.pop(0)
-                # if the next position has the same y coordinate, skip
-                while next_position['y'] == last_position['y']:
-                    next_position = sorted_positions.pop(0)
-                y_change = next_position['y']-last_position['y']
-                min_size_change = next_position['min']-last_position['min']
-                max_size_change = next_position['max']-last_position['max']
-                min_step_size = min_size_change/y_change
-                max_step_size = max_size_change/y_change
-            else:
-                min_step_size = 0
-                max_step_size = 0
-        
-        min_current_size += min_step_size
-        max_current_size += max_step_size
-
-    # apply mask by filling 0s for all locations a person could not be standing
-    if mask is not None:
-        pass
-
-    return estimated_sizes
+            if (datetime.datetime.now().timestamp() - self.camera.shared_frame_time.value) > 2:
+                print("last frame is more than 2 seconds old, restarting camera capture...")
+                self.camera.start_or_restart_capture()
+                time.sleep(5)
 
 class Camera:
-    def __init__(self, name, config, prepped_frame_queue, mqtt_client, mqtt_prefix, debug=False):
+    def __init__(self, name, config, prepped_frame_queue, mqtt_client, mqtt_prefix):
         self.name = name
         self.config = config
         self.detected_objects = []
         self.recent_frames = {}
         self.rtsp_url = get_rtsp_url(self.config['rtsp'])
-        self.take_frame = self.config.get('take_frame', 1)
         self.regions = self.config['regions']
         self.frame_shape = get_frame_shape(self.rtsp_url)
         self.mqtt_client = mqtt_client
         self.mqtt_topic_prefix = '{}/{}'.format(mqtt_prefix, self.name)
-        self.debug = debug
 
         # compute the flattened array length from the shape of the frame
         flat_array_length = self.frame_shape[0] * self.frame_shape[1] * self.frame_shape[2]
@@ -194,22 +154,24 @@ class Camera:
         # shape current frame so it can be treated as a numpy image
         self.shared_frame_np = tonumpyarray(self.shared_frame_array).reshape(self.frame_shape)
 
-        # create the process to capture frames from the RTSP stream and store in a shared array
-        self.capture_process = mp.Process(target=fetch_frames, args=(self.shared_frame_array, 
-            self.shared_frame_time, self.frame_lock, self.frame_ready, self.frame_shape,
-            self.rtsp_url, self.take_frame))
-        self.capture_process.daemon = True
+        self.capture_process = None
 
         # for each region, create a separate thread to resize the region and prep for detection
         self.detection_prep_threads = []
         for region in self.config['regions']:
+            # set a default threshold of 0.5 if not defined
+            if not 'threshold' in region:
+                region['threshold'] = 0.5
+            if not isinstance(region['threshold'], float):
+                print('Threshold is not a float. Setting to 0.5 default.')
+                region['threshold'] = 0.5
             self.detection_prep_threads.append(FramePrepper(
                 self.name,
                 self.shared_frame_np,
                 self.shared_frame_time,
                 self.frame_ready,
                 self.frame_lock,
-                region['size'], region['x_offset'], region['y_offset'],
+                region['size'], region['x_offset'], region['y_offset'], region['threshold'],
                 prepped_frame_queue
             ))
         
@@ -230,6 +192,9 @@ class Camera:
         mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self.objects_parsed, self.detected_objects)
         mqtt_publisher.start()
 
+        # create a watchdog thread for capture process
+        self.watchdog = CameraWatchdog(self)
+
         # load in the mask for person detection
         if 'mask' in self.config:
             self.mask = cv2.imread("/config/{}".format(self.config['mask']), cv2.IMREAD_GRAYSCALE)
@@ -237,18 +202,28 @@ class Camera:
             self.mask = np.zeros((self.frame_shape[0], self.frame_shape[1], 1), np.uint8)
             self.mask[:] = 255
 
-        # pre-compute estimated person size for every pixel in the image
-        if 'known_sizes' in self.config:
-            self.calculated_person_sizes = compute_sizes((self.frame_shape[0], self.frame_shape[1]), 
-                self.config['known_sizes'], None)
-        else:
-            self.calculated_person_sizes = None
+
+    def start_or_restart_capture(self):
+        if not self.capture_process is None:
+            print("Terminating the existing capture process...")
+            self.capture_process.terminate()
+            del self.capture_process
+            self.capture_process = None
+            
+        # create the process to capture frames from the RTSP stream and store in a shared array
+        print("Creating a new capture process...")
+        self.capture_process = mp.Process(target=fetch_frames, args=(self.shared_frame_array, 
+            self.shared_frame_time, self.frame_lock, self.frame_ready, self.frame_shape, self.rtsp_url))
+        self.capture_process.daemon = True
+        print("Starting a new capture process...")
+        self.capture_process.start()
     
     def start(self):
-        self.capture_process.start()
+        self.start_or_restart_capture()
         # start the object detection prep threads
         for detection_prep_thread in self.detection_prep_threads:
             detection_prep_thread.start()
+        self.watchdog.start()
     
     def join(self):
         self.capture_process.join()
@@ -261,27 +236,32 @@ class Camera:
             return
 
         for obj in objects:
-            if self.debug:
-                # print out the detected objects, scores and locations
-                print(self.name, obj['name'], obj['score'], obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'])
-            
-            location = (int(obj['ymax']), int((obj['xmax']-obj['xmin'])/2))
-
-            # if the person is in a masked location, continue
-            if self.mask[location[0]][location[1]] == [0]:
-                continue
-
-            if self.calculated_person_sizes is not None and obj['name'] == 'person':
-                person_size_range = self.calculated_person_sizes[location[0]][location[1]]
-
-                # if the person isnt on the ground, continue
-                if(person_size_range[0] == 0 and person_size_range[1] == 0):
+            if obj['name'] == 'person':
+                person_area = (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin'])
+                # find the matching region
+                region = None
+                for r in self.regions:
+                    if (
+                            obj['xmin'] >= r['x_offset'] and
+                            obj['ymin'] >= r['y_offset'] and
+                            obj['xmax'] <= r['x_offset']+r['size'] and
+                            obj['ymax'] <= r['y_offset']+r['size']
+                        ): 
+                        region = r
+                        break
+                
+                # if the min person area is larger than the
+                # detected person, don't add it to detected objects
+                if region and region['min_person_area'] > person_area:
                     continue
+            
+                # compute the coordinates of the person and make sure
+                # the location isnt outide the bounds of the image (can happen from rounding)
+                y_location = min(int(obj['ymax']), len(self.mask)-1)
+                x_location = min(int((obj['xmax']-obj['xmin'])/2.0), len(self.mask[0])-1)
 
-                person_size = (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin'])
-
-                # if the person is not within 20% of the estimated size for that location, continue
-                if person_size < person_size_range[0] or person_size > person_size_range[1]:
+                # if the person is in a masked location, continue
+                if self.mask[y_location][x_location] == [0]:
                     continue
 
             self.detected_objects.append(obj)