check to see if we have a frame before trying to send

.github/FUNDING.yml

@@ -1 +1 @@
-ko_fi: blakeblackshear
+github: blakeblackshear

.gitignore

@@ -1,2 +1,4 @@
 *.pyc 
 debug
+.vscode
+config/config.yml

Dockerfile

@@ -53,14 +53,6 @@ RUN apt-get -qq update && apt-get -qq install --no-install-recommends -y \
  libva-drm2 libva2 i965-va-driver vainfo \
  && rm -rf /var/lib/apt/lists/* 
 
-# Install core packages 
-RUN wget -q -O /tmp/get-pip.py --no-check-certificate https://bootstrap.pypa.io/get-pip.py && python3 /tmp/get-pip.py
-RUN  pip install -U pip \
- numpy \
- Flask \
- paho-mqtt \
- PyYAML
-
 # Download & build OpenCV
 # TODO: use multistage build to reduce image size: 
 #   https://medium.com/@denismakogon/pain-and-gain-running-opencv-application-with-golang-and-docker-on-alpine-3-7-435aa11c7aec
@@ -101,6 +93,15 @@ RUN ln -s /coco_labels.txt /label_map.pbtext
 RUN (apt-get autoremove -y; \
      apt-get autoclean -y)
 
+# Install core packages 
+RUN wget -q -O /tmp/get-pip.py --no-check-certificate https://bootstrap.pypa.io/get-pip.py && python3 /tmp/get-pip.py
+RUN  pip install -U pip \
+ numpy \
+ Flask \
+ paho-mqtt \
+ PyYAML \
+ matplotlib
+
 WORKDIR /opt/frigate/
 ADD frigate frigate/
 COPY detect_objects.py .

README.md

@@ -1,9 +1,7 @@
-<a href='https://ko-fi.com/P5P7XGO9' target='_blank'><img height='36' style='border:0px;height:36px;' src='https://az743702.vo.msecnd.net/cdn/kofi4.png?v=2' border='0' alt='Buy Me a Coffee at ko-fi.com' /></a>
-
-# Frigate - Realtime Object Detection for RTSP Cameras
+# Frigate - Realtime Object Detection for IP Cameras
 **Note:** This version requires the use of a [Google Coral USB Accelerator](https://coral.withgoogle.com/products/accelerator/)
 
-Uses OpenCV and Tensorflow to perform realtime object detection locally for RTSP cameras. Designed for integration with HomeAssistant or others via MQTT.
+Uses OpenCV and Tensorflow to perform realtime object detection locally for IP cameras. Designed for integration with HomeAssistant or others via MQTT.
 
 - Leverages multiprocessing and threads heavily with an emphasis on realtime over processing every frame
 - Allows you to define specific regions (squares) in the image to look for objects
@@ -32,8 +30,9 @@ docker run --rm \
 --privileged \
 -v /dev/bus/usb:/dev/bus/usb \
 -v <path_to_config_dir>:/config:ro \
+-v /etc/localtime:/etc/localtime:ro \
 -p 5000:5000 \
--e RTSP_PASSWORD='password' \
+-e FRIGATE_RTSP_PASSWORD='password' \
 frigate:latest
 ```
 
@@ -46,35 +45,58 @@ Example docker-compose:
     image: frigate:latest
     volumes:
       - /dev/bus/usb:/dev/bus/usb
+      - /etc/localtime:/etc/localtime:ro
       - <path_to_config>:/config
     ports:
       - "5000:5000"
     environment:
-      RTSP_PASSWORD: "password"
+      FRIGATE_RTSP_PASSWORD: "password"
 ```
 
-A `config.yml` file must exist in the `config` directory. See example [here](config/config.yml).
+A `config.yml` file must exist in the `config` directory. See example [here](config/config.example.yml) and device specific info can be found [here](docs/DEVICES.md).
 
-Access the mjpeg stream at `http://localhost:5000/<camera_name>` and the best person snapshot at `http://localhost:5000/<camera_name>/best_person.jpg`
+Access the mjpeg stream at `http://localhost:5000/<camera_name>` and the best snapshot for any object type with at `http://localhost:5000/<camera_name>/<object_name>/best.jpg`
 
 ## Integration with HomeAssistant
 ```
 camera:
   - name: Camera Last Person
-    platform: generic
-    still_image_url: http://<ip>:5000/<camera_name>/best_person.jpg
+    platform: mqtt
+    topic: frigate/<camera_name>/person/snapshot
+  - name: Camera Last Car
+    platform: mqtt
+    topic: frigate/<camera_name>/car/snapshot
 
 binary_sensor:
   - name: Camera Person
     platform: mqtt
-    state_topic: "frigate/<camera_name>/objects"
-    value_template: '{{ value_json.person }}'
+    state_topic: "frigate/<camera_name>/person"
     device_class: motion
     availability_topic: "frigate/available"
+
+automation:
+  - alias: Alert me if a person is detected while armed away
+    trigger: 
+      platform: state
+      entity_id: binary_sensor.camera_person
+      from: 'off'
+      to: 'on'
+    condition:
+      - condition: state
+        entity_id: alarm_control_panel.home_alarm
+        state: armed_away
+    action:
+      - service: notify.user_telegram
+        data:
+          message: "A person was detected."
+          data:
+            photo:
+              - url: http://<ip>:5000/<camera_name>/person/best.jpg
+                caption: A person was detected.
 ```
 
 ## Tips
-- Lower the framerate of the RTSP feed on the camera to reduce the CPU usage for capturing the feed
+- Lower the framerate of the video feed on the camera to reduce the CPU usage for capturing the feed
 
 ## Future improvements
 - [x] Remove motion detection for now

config/config.example.yml

@@ -0,0 +1,128 @@
+web_port: 5000
+
+mqtt:
+  host: mqtt.server.com
+  topic_prefix: frigate
+#  client_id: frigate # Optional -- set to override default client id of 'frigate' if running multiple instances
+#  user: username # Optional -- Uncomment for use
+#  password: password # Optional -- Uncomment for use
+
+#################
+# Default ffmpeg args. Optional and can be overwritten per camera.
+# Should work with most RTSP cameras that send h264 video
+# Built from the properties below with:
+# "ffmpeg" + global_args + input_args + "-i" + input + output_args
+#################
+# ffmpeg:
+#   global_args:
+#     - -hide_banner
+#     - -loglevel
+#     - panic
+#   hwaccel_args: []
+#   input_args:
+#     - -avoid_negative_ts
+#     - make_zero
+#     - -fflags
+#     - nobuffer
+#     - -flags
+#     - low_delay
+#     - -strict
+#     - experimental
+#     - -fflags
+#     - +genpts+discardcorrupt
+#     - -vsync
+#     - drop
+#     - -rtsp_transport
+#     - tcp
+#     - -stimeout
+#     - '5000000'
+#     - -use_wallclock_as_timestamps
+#     - '1'
+#   output_args:
+#     - -vf
+#     - mpdecimate
+#     - -f
+#     - rawvideo
+#     - -pix_fmt
+#     - rgb24
+
+####################
+# Global object configuration. Applies to all cameras and regions
+# unless overridden at the camera/region levels.
+# Keys must be valid labels. By default, the model uses coco (https://dl.google.com/coral/canned_models/coco_labels.txt).
+# All labels from the model are reported over MQTT. These values are used to filter out false positives.
+####################
+objects:
+  person:
+    min_area: 5000
+    max_area: 100000
+    threshold: 0.5
+
+cameras:
+  back:
+    ffmpeg:
+      ################
+      # Source passed to ffmpeg after the -i parameter. Supports anything compatible with OpenCV and FFmpeg.
+      # Environment variables that begin with 'FRIGATE_' may be referenced in {}
+      ################
+      input: rtsp://viewer:{FRIGATE_RTSP_PASSWORD}@10.0.10.10:554/cam/realmonitor?channel=1&subtype=2
+      #################
+      # These values will override default values for just this camera
+      #################
+      # global_args: []
+      # hwaccel_args: []
+      # input_args: []
+      # output_args: []
+
+    ################
+    ## Optional mask. Must be the same dimensions as your video feed.
+    ## The mask works by looking at the bottom center of the bounding box for the detected
+    ## person in the image. If that pixel in the mask is a black pixel, it ignores it as a
+    ## false positive. In my mask, the grass and driveway visible from my backdoor camera 
+    ## are white. The garage doors, sky, and trees (anywhere it would be impossible for a 
+    ## person to stand) are black.
+    ################
+    # mask: back-mask.bmp
+
+    ################
+    # Allows you to limit the framerate within frigate for cameras that do not support
+    # custom framerates. A value of 1 tells frigate to look at every frame, 2 every 2nd frame, 
+    # 3 every 3rd frame, etc.
+    ################
+    take_frame: 1
+
+    objects:
+      person:
+        min_area: 5000
+        max_area: 100000
+        threshold: 0.5
+    
+    ################
+    # size: size of the region in pixels
+    # x_offset/y_offset: position of the upper left corner of your region (top left of image is 0,0)
+    # min_person_area (optional): minimum width*height of the bounding box for the detected person
+    # max_person_area (optional): maximum width*height of the bounding box for the detected person
+    # threshold (optional): The minimum decimal percentage (50% hit = 0.5) for the confidence from tensorflow
+    # Tips: All regions are resized to 300x300 before detection because the model is trained on that size.
+    #       Resizing regions takes CPU power. Ideally, all regions should be as close to 300x300 as possible.
+    #       Defining a region that goes outside the bounds of the image will result in errors.
+    ################
+    regions:
+      - size: 350
+        x_offset: 0
+        y_offset: 300
+        objects:
+          car:
+            threshold: 0.2
+      - size: 400
+        x_offset: 350
+        y_offset: 250
+        objects:
+          person:
+            min_area: 2000
+      - size: 400
+        x_offset: 750
+        y_offset: 250
+        objects:
+          person:
+            min_area: 2000

config/config.yml

@@ -1,65 +0,0 @@
-web_port: 5000
-
-mqtt:
-  host: mqtt.server.com
-  topic_prefix: frigate
-#  user: username # Optional -- Uncomment for use
-#  password: password # Optional -- Uncomment for use
-
-cameras:
-  back:
-    rtsp:
-      user: viewer
-      host: 10.0.10.10
-      port: 554
-      # values that begin with a "$" will be replaced with environment variable
-      password: $RTSP_PASSWORD
-      path: /cam/realmonitor?channel=1&subtype=2
-
-    ################
-    ## Optional mask. Must be the same dimensions as your video feed.
-    ## The mask works by looking at the bottom center of the bounding box for the detected
-    ## person in the image. If that pixel in the mask is a black pixel, it ignores it as a
-    ## false positive. In my mask, the grass and driveway visible from my backdoor camera 
-    ## are white. The garage doors, sky, and trees (anywhere it would be impossible for a 
-    ## person to stand) are black.
-    ################
-    # mask: back-mask.bmp
-
-    ################
-    # Allows you to limit the framerate within frigate for cameras that do not support
-    # custom framerates. A value of 1 tells frigate to look at every frame, 2 every 2nd frame, 
-    # 3 every 3rd frame, etc.
-    ################
-    take_frame: 1
-
-    ################
-    # Optional hardware acceleration parameters for ffmpeg. If your hardware supports it, it can
-    # greatly reduce the CPU power used to decode the video stream. You will need to determine which
-    # parameters work for your specific hardware. These may work for those with Intel hardware that
-    # supports QuickSync.
-    ################
-    # ffmpeg_hwaccel_args:
-    #   - -hwaccel
-    #   - vaapi
-    #   - -hwaccel_device
-    #   - /dev/dri/renderD128
-    #   - -hwaccel_output_format
-    #   - yuv420p
-    
-    regions:
-      - size: 350
-        x_offset: 0
-        y_offset: 300
-        min_person_area: 5000
-        threshold: 0.5
-      - size: 400
-        x_offset: 350
-        y_offset: 250
-        min_person_area: 2000
-        threshold: 0.5
-      - size: 400
-        x_offset: 750
-        y_offset: 250
-        min_person_area: 2000
-        threshold: 0.5

detect_objects.py

@@ -17,6 +17,32 @@ MQTT_PORT = CONFIG.get('mqtt', {}).get('port', 1883)
 MQTT_TOPIC_PREFIX = CONFIG.get('mqtt', {}).get('topic_prefix', 'frigate')
 MQTT_USER = CONFIG.get('mqtt', {}).get('user')
 MQTT_PASS = CONFIG.get('mqtt', {}).get('password')
+MQTT_CLIENT_ID = CONFIG.get('mqtt', {}).get('client_id', 'frigate')
+
+# Set the default FFmpeg config
+FFMPEG_CONFIG = CONFIG.get('ffmpeg', {})
+FFMPEG_DEFAULT_CONFIG = {
+    'global_args': FFMPEG_CONFIG.get('global_args', 
+        ['-hide_banner','-loglevel','panic']),
+    'hwaccel_args': FFMPEG_CONFIG.get('hwaccel_args', 
+        []),
+    'input_args': FFMPEG_CONFIG.get('input_args',
+        ['-avoid_negative_ts', 'make_zero',
+         '-fflags', 'nobuffer',
+         '-flags', 'low_delay',
+         '-strict', 'experimental',
+         '-fflags', '+genpts+discardcorrupt',
+         '-vsync', 'drop',
+         '-rtsp_transport', 'tcp',
+         '-stimeout', '5000000',
+         '-use_wallclock_as_timestamps', '1']),
+    'output_args': FFMPEG_CONFIG.get('output_args',
+        ['-vf', 'mpdecimate',
+         '-f', 'rawvideo',
+         '-pix_fmt', 'rgb24'])
+}
+
+GLOBAL_OBJECT_CONFIG = CONFIG.get('objects', {})
 
 WEB_PORT = CONFIG.get('web_port', 5000)
 DEBUG = (CONFIG.get('debug', '0') == '1')
@@ -36,7 +62,7 @@ def main():
                 print ("Unable to connect to MQTT: Connection refused. Error code: " + str(rc))
         # publish a message to signal that the service is running
         client.publish(MQTT_TOPIC_PREFIX+'/available', 'online', retain=True)
-    client = mqtt.Client(client_id="frigate")
+    client = mqtt.Client(client_id=MQTT_CLIENT_ID)
     client.on_connect = on_connect
     client.will_set(MQTT_TOPIC_PREFIX+'/available', payload='offline', qos=1, retain=True)
     if not MQTT_USER is None:
@@ -50,7 +76,7 @@ def main():
 
     cameras = {}
     for name, config in CONFIG['cameras'].items():
-        cameras[name] = Camera(name, config, prepped_frame_queue, client, MQTT_TOPIC_PREFIX)
+        cameras[name] = Camera(name, FFMPEG_DEFAULT_CONFIG, GLOBAL_OBJECT_CONFIG, config, prepped_frame_queue, client, MQTT_TOPIC_PREFIX)
 
     prepped_queue_processor = PreppedQueueProcessor(
         cameras,
@@ -65,31 +91,40 @@ def main():
     # create a flask app that encodes frames a mjpeg on demand
     app = Flask(__name__)
 
-    @app.route('/<camera_name>/best_person.jpg')
-    def best_person(camera_name):
-        best_person_frame = cameras[camera_name].get_best_person()
-        if best_person_frame is None:
-            best_person_frame = np.zeros((720,1280,3), np.uint8)
-        ret, jpg = cv2.imencode('.jpg', best_person_frame)
-        response = make_response(jpg.tobytes())
-        response.headers['Content-Type'] = 'image/jpg'
-        return response
+    @app.route('/')
+    def ishealthy():
+        # return a healh
+        return "Frigate is running. Alive and healthy!"
+
+    @app.route('/<camera_name>/<label>/best.jpg')
+    def best(camera_name, label):
+        if camera_name in cameras:
+            best_frame = cameras[camera_name].get_best(label)
+            if best_frame is None:
+                best_frame = np.zeros((720,1280,3), np.uint8)
+            ret, jpg = cv2.imencode('.jpg', best_frame)
+            response = make_response(jpg.tobytes())
+            response.headers['Content-Type'] = 'image/jpg'
+            return response
+        else:
+            return f'Camera named {camera_name} not found', 404
 
     @app.route('/<camera_name>')
     def mjpeg_feed(camera_name):
-        # return a multipart response
-        return Response(imagestream(camera_name),
-                        mimetype='multipart/x-mixed-replace; boundary=frame')
+        if camera_name in cameras:
+            # return a multipart response
+            return Response(imagestream(camera_name),
+                            mimetype='multipart/x-mixed-replace; boundary=frame')
+        else:
+            return f'Camera named {camera_name} not found', 404
 
     def imagestream(camera_name):
         while True:
-            # max out at 5 FPS
-            time.sleep(0.2)
+            # max out at 1 FPS
+            time.sleep(1)
             frame = cameras[camera_name].get_current_frame_with_objects()
-            # encode the image into a jpg
-            ret, jpg = cv2.imencode('.jpg', frame)
             yield (b'--frame\r\n'
-                b'Content-Type: image/jpeg\r\n\r\n' + jpg.tobytes() + b'\r\n\r\n')
+                b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n\r\n')
 
     app.run(host='0.0.0.0', port=WEB_PORT, debug=False)
 

docs/DEVICES.md

@@ -0,0 +1,74 @@
+# Configuration Examples
+
+### Default (most RTSP cameras)
+This is the default ffmpeg command and should work with most RTSP cameras that send h264 video
+```yaml
+ffmpeg:
+  global_args:
+    - -hide_banner
+    - -loglevel
+    - panic
+  hwaccel_args: []
+  input_args:
+    - -avoid_negative_ts
+    - make_zero 
+    - -fflags
+    - nobuffer
+    - -flags
+    - low_delay
+    - -strict
+    - experimental
+    - -fflags
+    - +genpts+discardcorrupt
+    - -vsync
+    - drop
+    - -rtsp_transport
+    - tcp
+    - -stimeout
+    - '5000000' 
+    - -use_wallclock_as_timestamps
+    - '1'
+  output_args:
+    - -vf
+    - mpdecimate
+    - -f
+    - rawvideo
+    - -pix_fmt
+    - rgb24
+```
+
+### RTMP Cameras
+The input parameters need to be adjusted for RTMP cameras
+```yaml
+ffmpeg:
+input_args:
+    - -avoid_negative_ts
+    - make_zero
+    - -fflags
+    - nobuffer
+    - -flags
+    - low_delay
+    - -strict
+    - experimental
+    - -fflags
+    - +genpts+discardcorrupt
+    - -vsync
+    - drop
+    - -use_wallclock_as_timestamps
+    - '1'
+```
+
+
+### Hardware Acceleration
+
+Intel Quicksync
+```yaml
+ffmpeg:
+  hwaccel_args:
+    - -hwaccel
+    - vaapi
+    - -hwaccel_device
+    - /dev/dri/renderD128
+    - -hwaccel_output_format
+    - yuv420p
+```

frigate/mqtt.py

@@ -1,33 +1,47 @@
 import json
+import cv2
 import threading
+from collections import Counter, defaultdict
 
 class MqttObjectPublisher(threading.Thread):
-    def __init__(self, client, topic_prefix, objects_parsed, detected_objects):
+    def __init__(self, client, topic_prefix, objects_parsed, detected_objects, best_frames):
         threading.Thread.__init__(self)
         self.client = client
         self.topic_prefix = topic_prefix
         self.objects_parsed = objects_parsed
         self._detected_objects = detected_objects
+        self.best_frames = best_frames
 
     def run(self):
-        last_sent_payload = ""
+        current_object_status = defaultdict(lambda: 'OFF')
         while True:
-
-            # initialize the payload
-            payload = {}
-
             # wait until objects have been parsed
             with self.objects_parsed:
                 self.objects_parsed.wait()
 
-            # add all the person scores in detected objects
+            # make a copy of detected objects
             detected_objects = self._detected_objects.copy()
-            person_score = sum([obj['score'] for obj in detected_objects if obj['name'] == 'person'])
-            # if the person score is more than 100, set person to ON
-            payload['person'] = 'ON' if int(person_score*100) > 100 else 'OFF'
 
-            # send message for objects if different
-            new_payload = json.dumps(payload, sort_keys=True)
-            if new_payload != last_sent_payload:
-                last_sent_payload = new_payload
-                self.client.publish(self.topic_prefix+'/objects', new_payload, retain=False)
+            # total up all scores by object type
+            obj_counter = Counter()
+            for obj in detected_objects:
+                obj_counter[obj['name']] += obj['score']
+            
+            # report on detected objects
+            for obj_name, total_score in obj_counter.items():
+                new_status = 'ON' if int(total_score*100) > 100 else 'OFF'
+                if new_status != current_object_status[obj_name]:
+                    current_object_status[obj_name] = new_status
+                    self.client.publish(self.topic_prefix+'/'+obj_name, new_status, retain=False)
+                    # send the snapshot over mqtt if we have it as well
+                    if obj_name in self.best_frames.best_frames:
+                        ret, jpg = cv2.imencode('.jpg', self.best_frames.best_frames[obj_name])
+                        if ret:
+                            jpg_bytes = jpg.tobytes()
+                            self.client.publish(self.topic_prefix+'/'+obj_name+'/snapshot', jpg_bytes, retain=True)
+
+            # expire any objects that are ON and no longer detected
+            expired_objects = [obj_name for obj_name, status in current_object_status.items() if status == 'ON' and not obj_name in obj_counter]
+            for obj_name in expired_objects:
+                current_object_status[obj_name] = 'OFF'
+                self.client.publish(self.topic_prefix+'/'+obj_name, 'OFF', retain=False)

frigate/object_detection.py

@@ -4,22 +4,7 @@ import cv2
 import threading
 import numpy as np
 from edgetpu.detection.engine import DetectionEngine
-from . util import tonumpyarray
-
-# Path to frozen detection graph. This is the actual model that is used for the object detection.
-PATH_TO_CKPT = '/frozen_inference_graph.pb'
-# List of the strings that is used to add correct label for each box.
-PATH_TO_LABELS = '/label_map.pbtext'
-
-# Function to read labels from text files.
-def ReadLabelFile(file_path):
-    with open(file_path, 'r') as f:
-        lines = f.readlines()
-    ret = {}
-    for line in lines:
-        pair = line.strip().split(maxsplit=1)
-        ret[int(pair[0])] = pair[1].strip()
-    return ret
+from . util import tonumpyarray, LABELS, PATH_TO_CKPT
 
 class PreppedQueueProcessor(threading.Thread):
     def __init__(self, cameras, prepped_frame_queue):
@@ -30,7 +15,7 @@ class PreppedQueueProcessor(threading.Thread):
         
         # Load the edgetpu engine and labels
         self.engine = DetectionEngine(PATH_TO_CKPT)
-        self.labels = ReadLabelFile(PATH_TO_LABELS)
+        self.labels = LABELS
 
     def run(self):
         # process queue...
@@ -38,21 +23,18 @@ class PreppedQueueProcessor(threading.Thread):
             frame = self.prepped_frame_queue.get()
 
             # Actual detection.
-            objects = self.engine.DetectWithInputTensor(frame['frame'], threshold=frame['region_threshold'], top_k=3)
+            objects = self.engine.DetectWithInputTensor(frame['frame'], threshold=0.5, top_k=5)
             # print(self.engine.get_inference_time())
 
             # parse and pass detected objects back to the camera
             parsed_objects = []
             for obj in objects:
-                box = obj.bounding_box.flatten().tolist()
                 parsed_objects.append({
+                            'region_id': frame['region_id'],
                             'frame_time': frame['frame_time'],
                             'name': str(self.labels[obj.label_id]),
                             'score': float(obj.score),
-                            'xmin': int((box[0] * frame['region_size']) + frame['region_x_offset']),
-                            'ymin': int((box[1] * frame['region_size']) + frame['region_y_offset']),
-                            'xmax': int((box[2] * frame['region_size']) + frame['region_x_offset']),
-                            'ymax': int((box[3] * frame['region_size']) + frame['region_y_offset'])
+                            'box': obj.bounding_box.flatten().tolist()
                         })
             self.cameras[frame['camera_name']].add_objects(parsed_objects)
 
@@ -61,7 +43,7 @@ class PreppedQueueProcessor(threading.Thread):
 class FramePrepper(threading.Thread):
     def __init__(self, camera_name, shared_frame, frame_time, frame_ready, 
         frame_lock,
-        region_size, region_x_offset, region_y_offset, region_threshold,
+        region_size, region_x_offset, region_y_offset, region_id,
         prepped_frame_queue):
 
         threading.Thread.__init__(self)
@@ -73,7 +55,7 @@ class FramePrepper(threading.Thread):
         self.region_size = region_size
         self.region_x_offset = region_x_offset
         self.region_y_offset = region_y_offset
-        self.region_threshold = region_threshold
+        self.region_id = region_id
         self.prepped_frame_queue = prepped_frame_queue
 
     def run(self):
@@ -104,7 +86,7 @@ class FramePrepper(threading.Thread):
                     'frame_time': frame_time,
                     'frame': frame_expanded.flatten().copy(),
                     'region_size': self.region_size,
-                    'region_threshold': self.region_threshold,
+                    'region_id': self.region_id,
                     'region_x_offset': self.region_x_offset,
                     'region_y_offset': self.region_y_offset
                 })

frigate/objects.py

@@ -2,6 +2,7 @@ import time
 import datetime
 import threading
 import cv2
+import numpy as np
 from . util import draw_box_with_label
 
 class ObjectCleaner(threading.Thread):
@@ -35,16 +36,15 @@ class ObjectCleaner(threading.Thread):
                     self._objects_parsed.notify_all()
 
 
-# Maintains the frame and person with the highest score from the most recent
-# motion event
-class BestPersonFrame(threading.Thread):
+# Maintains the frame and object with the highest score
+class BestFrames(threading.Thread):
     def __init__(self, objects_parsed, recent_frames, detected_objects):
         threading.Thread.__init__(self)
         self.objects_parsed = objects_parsed
         self.recent_frames = recent_frames
         self.detected_objects = detected_objects
-        self.best_person = None
-        self.best_frame = None
+        self.best_objects = {}
+        self.best_frames = {}
 
     def run(self):
         while True:
@@ -55,34 +55,29 @@ class BestPersonFrame(threading.Thread):
 
             # make a copy of detected objects
             detected_objects = self.detected_objects.copy()
-            detected_people = [obj for obj in detected_objects if obj['name'] == 'person']
 
-            # get the highest scoring person
-            new_best_person = max(detected_people, key=lambda x:x['score'], default=self.best_person)
-
-            # if there isnt a person, continue
-            if new_best_person is None:
-                continue
-
-            # if there is no current best_person
-            if self.best_person is None:
-                self.best_person = new_best_person
-            # if there is already a best_person
-            else:
-                now = datetime.datetime.now().timestamp()
-                # if the new best person is a higher score than the current best person 
-                # or the current person is more than 1 minute old, use the new best person
-                if new_best_person['score'] > self.best_person['score'] or (now - self.best_person['frame_time']) > 60:
-                    self.best_person = new_best_person
+            for obj in detected_objects:
+                if obj['name'] in self.best_objects:
+                    now = datetime.datetime.now().timestamp()
+                    # if the object is a higher score than the current best score 
+                    # or the current object is more than 1 minute old, use the new object
+                    if obj['score'] > self.best_objects[obj['name']]['score'] or (now - self.best_objects[obj['name']]['frame_time']) > 60:
+                        self.best_objects[obj['name']] = obj
+                else:
+                    self.best_objects[obj['name']] = obj
             
             # make a copy of the recent frames
             recent_frames = self.recent_frames.copy()
-            
-            if not self.best_person is None and self.best_person['frame_time'] in recent_frames:
-                best_frame = recent_frames[self.best_person['frame_time']]
 
-                label = "{}: {}% {}".format(self.best_person['name'],int(self.best_person['score']*100),int(self.best_person['area']))
-                draw_box_with_label(best_frame, self.best_person['xmin'], self.best_person['ymin'], 
-                    self.best_person['xmax'], self.best_person['ymax'], label)
-                
-                self.best_frame = cv2.cvtColor(best_frame, cv2.COLOR_RGB2BGR)
+            for name, obj in self.best_objects.items():
+                if obj['frame_time'] in recent_frames:
+                    best_frame = recent_frames[obj['frame_time']] #, np.zeros((720,1280,3), np.uint8))
+
+                    draw_box_with_label(best_frame, obj['xmin'], obj['ymin'], 
+                        obj['xmax'], obj['ymax'], obj['name'], obj['score'], obj['area'])
+                    
+                    # print a timestamp
+                    time_to_show = datetime.datetime.fromtimestamp(obj['frame_time']).strftime("%m/%d/%Y %H:%M:%S")
+                    cv2.putText(best_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
+                    
+                    self.best_frames[name] = cv2.cvtColor(best_frame, cv2.COLOR_RGB2BGR)

frigate/util.py

@@ -1,19 +1,31 @@
 import numpy as np
 import cv2
+import matplotlib.pyplot as plt
+
+# Function to read labels from text files.
+def ReadLabelFile(file_path):
+    with open(file_path, 'r') as f:
+        lines = f.readlines()
+    ret = {}
+    for line in lines:
+        pair = line.strip().split(maxsplit=1)
+        ret[int(pair[0])] = pair[1].strip()
+    return ret
 
 # convert shared memory array into numpy array
 def tonumpyarray(mp_arr):
     return np.frombuffer(mp_arr.get_obj(), dtype=np.uint8)
 
-def draw_box_with_label(frame, x_min, y_min, x_max, y_max, label):
-    color = (255,0,0)
+def draw_box_with_label(frame, x_min, y_min, x_max, y_max, label, score, area):
+    color = COLOR_MAP[label]
+    display_text = "{}: {}% {}".format(label,int(score*100),int(area))
     cv2.rectangle(frame, (x_min, y_min), 
         (x_max, y_max), 
         color, 2)
     font_scale = 0.5
     font = cv2.FONT_HERSHEY_SIMPLEX
     # get the width and height of the text box
-    size = cv2.getTextSize(label, font, fontScale=font_scale, thickness=2)
+    size = cv2.getTextSize(display_text, font, fontScale=font_scale, thickness=2)
     text_width = size[0][0]
     text_height = size[0][1]
     line_height = text_height + size[1]
@@ -23,4 +35,16 @@ def draw_box_with_label(frame, x_min, y_min, x_max, y_max, label):
     # make the coords of the box with a small padding of two pixels
     textbox_coords = ((text_offset_x, text_offset_y), (text_offset_x + text_width + 2, text_offset_y + line_height))
     cv2.rectangle(frame, textbox_coords[0], textbox_coords[1], color, cv2.FILLED)
-    cv2.putText(frame, label, (text_offset_x, text_offset_y + line_height - 3), font, fontScale=font_scale, color=(0, 0, 0), thickness=2)
+    cv2.putText(frame, display_text, (text_offset_x, text_offset_y + line_height - 3), font, fontScale=font_scale, color=(0, 0, 0), thickness=2)
+
+# Path to frozen detection graph. This is the actual model that is used for the object detection.
+PATH_TO_CKPT = '/frozen_inference_graph.pb'
+# List of the strings that is used to add correct label for each box.
+PATH_TO_LABELS = '/label_map.pbtext'
+
+LABELS = ReadLabelFile(PATH_TO_LABELS)
+cmap = plt.cm.get_cmap('tab10', len(LABELS.keys()))
+
+COLOR_MAP = {}
+for key, val in LABELS.items():
+    COLOR_MAP[val] = tuple(int(round(255 * c)) for c in cmap(key)[:3])

frigate/video.py

@@ -7,9 +7,10 @@ import ctypes
 import multiprocessing as mp
 import subprocess as sp
 import numpy as np
+from collections import defaultdict
 from . util import tonumpyarray, draw_box_with_label
 from . object_detection import FramePrepper
-from . objects import ObjectCleaner, BestPersonFrame
+from . objects import ObjectCleaner, BestFrames
 from . mqtt import MqttObjectPublisher
 
 # Stores 2 seconds worth of frames when motion is detected so they can be used for other threads
@@ -46,21 +47,18 @@ class FrameTracker(threading.Thread):
                 if (now - k) > 2:
                     del self.recent_frames[k]
 
-def get_frame_shape(rtsp_url):
+def get_frame_shape(source):
     # capture a single frame and check the frame shape so the correct array
     # size can be allocated in memory
-    video = cv2.VideoCapture(rtsp_url)
+    video = cv2.VideoCapture(source)
     ret, frame = video.read()
     frame_shape = frame.shape
     video.release()
     return frame_shape
 
-def get_rtsp_url(rtsp_config):
-    if (rtsp_config['password'].startswith('$')):
-        rtsp_config['password'] = os.getenv(rtsp_config['password'][1:])
-    return 'rtsp://{}:{}@{}:{}{}'.format(rtsp_config['user'], 
-        rtsp_config['password'], rtsp_config['host'], rtsp_config['port'],
-        rtsp_config['path'])
+def get_ffmpeg_input(ffmpeg_input):
+    frigate_vars = {k: v for k, v in os.environ.items() if k.startswith('FRIGATE_')}
+    return ffmpeg_input.format(**frigate_vars)
 
 class CameraWatchdog(threading.Thread):
     def __init__(self, camera):
@@ -73,8 +71,8 @@ class CameraWatchdog(threading.Thread):
             # wait a bit before checking
             time.sleep(10)
 
-            if (datetime.datetime.now().timestamp() - self.camera.frame_time.value) > 2:
-                print("last frame is more than 2 seconds old, restarting camera capture...")
+            if (datetime.datetime.now().timestamp() - self.camera.frame_time.value) > 300:
+                print("last frame is more than 5 minutes old, restarting camera capture...")
                 self.camera.start_or_restart_capture()
                 time.sleep(5)
 
@@ -114,16 +112,24 @@ class CameraCapture(threading.Thread):
                 self.camera.frame_ready.notify_all()
 
 class Camera:
-    def __init__(self, name, config, prepped_frame_queue, mqtt_client, mqtt_prefix):
+    def __init__(self, name, ffmpeg_config, global_objects_config, config, prepped_frame_queue, mqtt_client, mqtt_prefix):
         self.name = name
         self.config = config
         self.detected_objects = []
         self.recent_frames = {}
-        self.rtsp_url = get_rtsp_url(self.config['rtsp'])
+
+        self.ffmpeg = config.get('ffmpeg', {})
+        self.ffmpeg_input = get_ffmpeg_input(self.ffmpeg['input'])
+        self.ffmpeg_global_args = self.ffmpeg.get('global_args', ffmpeg_config['global_args'])
+        self.ffmpeg_hwaccel_args = self.ffmpeg.get('hwaccel_args', ffmpeg_config['hwaccel_args'])
+        self.ffmpeg_input_args = self.ffmpeg.get('input_args', ffmpeg_config['input_args'])
+        self.ffmpeg_output_args = self.ffmpeg.get('output_args', ffmpeg_config['output_args'])
+
+        camera_objects_config = config.get('objects', {})
+
         self.take_frame = self.config.get('take_frame', 1)
-        self.ffmpeg_hwaccel_args = self.config.get('ffmpeg_hwaccel_args', [])
         self.regions = self.config['regions']
-        self.frame_shape = get_frame_shape(self.rtsp_url)
+        self.frame_shape = get_frame_shape(self.ffmpeg_input)
         self.frame_size = self.frame_shape[0] * self.frame_shape[1] * self.frame_shape[2]
         self.mqtt_client = mqtt_client
         self.mqtt_topic_prefix = '{}/{}'.format(mqtt_prefix, self.name)
@@ -138,26 +144,35 @@ class Camera:
         self.frame_ready = mp.Condition()
         # Condition for notifying that objects were parsed
         self.objects_parsed = mp.Condition()
+        
+        # initialize the frame cache
+        self.cached_frame_with_objects = {
+            'frame_bytes': [],
+            'frame_time': 0
+        }
 
         self.ffmpeg_process = None
         self.capture_thread = None
 
         # for each region, create a separate thread to resize the region and prep for detection
         self.detection_prep_threads = []
-        for region in self.config['regions']:
-            # set a default threshold of 0.5 if not defined
-            if not 'threshold' in region:
-                region['threshold'] = 0.5
-            if not isinstance(region['threshold'], float):
-                print('Threshold is not a float. Setting to 0.5 default.')
-                region['threshold'] = 0.5
+        for index, region in enumerate(self.config['regions']):
+            region_objects = region.get('objects', {})
+            # build objects config for region
+            objects_with_config = set().union(global_objects_config.keys(), camera_objects_config.keys(), region_objects.keys())
+            merged_objects_config = defaultdict(lambda: {})
+            for obj in objects_with_config:
+                merged_objects_config[obj] = {**global_objects_config.get(obj,{}), **camera_objects_config.get(obj, {}), **region_objects.get(obj, {})}
+            
+            region['objects'] = merged_objects_config
+
             self.detection_prep_threads.append(FramePrepper(
                 self.name,
                 self.current_frame,
                 self.frame_time,
                 self.frame_ready,
                 self.frame_lock,
-                region['size'], region['x_offset'], region['y_offset'], region['threshold'],
+                region['size'], region['x_offset'], region['y_offset'], index,
                 prepped_frame_queue
             ))
         
@@ -166,22 +181,22 @@ class Camera:
             self.frame_ready, self.frame_lock, self.recent_frames)
         self.frame_tracker.start()
 
-        # start a thread to store the highest scoring recent person frame
-        self.best_person_frame = BestPersonFrame(self.objects_parsed, self.recent_frames, self.detected_objects)
-        self.best_person_frame.start()
+        # start a thread to store the highest scoring recent frames for monitored object types
+        self.best_frames = BestFrames(self.objects_parsed, self.recent_frames, self.detected_objects)
+        self.best_frames.start()
 
         # start a thread to expire objects from the detected objects list
         self.object_cleaner = ObjectCleaner(self.objects_parsed, self.detected_objects)
         self.object_cleaner.start()
 
-        # start a thread to publish object scores (currently only person)
-        mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self.objects_parsed, self.detected_objects)
+        # start a thread to publish object scores
+        mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self.objects_parsed, self.detected_objects, self.best_frames)
         mqtt_publisher.start()
 
         # create a watchdog thread for capture process
         self.watchdog = CameraWatchdog(self)
 
-        # load in the mask for person detection
+        # load in the mask for object detection
         if 'mask' in self.config:
             self.mask = cv2.imread("/config/{}".format(self.config['mask']), cv2.IMREAD_GRAYSCALE)
         else:
@@ -194,15 +209,22 @@ class Camera:
 
     def start_or_restart_capture(self):
         if not self.ffmpeg_process is None:
-            print("Killing the existing ffmpeg process...")
-            self.ffmpeg_process.kill()
-            self.ffmpeg_process.wait()
+            print("Terminating the existing ffmpeg process...")
+            self.ffmpeg_process.terminate()
+            try:
+                print("Waiting for ffmpeg to exit gracefully...")
+                self.ffmpeg_process.wait(timeout=30)
+            except sp.TimeoutExpired:
+                print("FFmpeg didnt exit. Force killing...")
+                self.ffmpeg_process.kill()
+                self.ffmpeg_process.wait()
+
             print("Waiting for the capture thread to exit...")
             self.capture_thread.join()
             self.ffmpeg_process = None
             self.capture_thread = None
             
-        # create the process to capture frames from the RTSP stream and store in a shared array
+        # create the process to capture frames from the input stream and store in a shared array
         print("Creating a new ffmpeg process...")
         self.start_ffmpeg()
         
@@ -212,28 +234,13 @@ class Camera:
         self.capture_thread.start()
     
     def start_ffmpeg(self):
-        ffmpeg_global_args = [
-            '-hide_banner', '-loglevel', 'panic'
-        ]
-        ffmpeg_input_args = [
-            '-avoid_negative_ts', 'make_zero', 
-            '-fflags', 'nobuffer',
-            '-flags', 'low_delay',
-            '-strict', 'experimental',
-            '-fflags', '+genpts', 
-            '-rtsp_transport', 'tcp', 
-            '-stimeout', '5000000', 
-            '-use_wallclock_as_timestamps', '1'
-        ]
-
         ffmpeg_cmd = (['ffmpeg'] +
-            ffmpeg_global_args +
+            self.ffmpeg_global_args +
             self.ffmpeg_hwaccel_args +
-            ffmpeg_input_args +
-            ['-i', self.rtsp_url,
-            '-f', 'rawvideo',
-            '-pix_fmt', 'rgb24',
-            'pipe:'])
+            self.ffmpeg_input_args +
+            ['-i', self.ffmpeg_input] +
+            self.ffmpeg_output_args +
+            ['pipe:'])
 
         print(" ".join(ffmpeg_cmd))
         
@@ -257,33 +264,45 @@ class Camera:
             return
 
         for obj in objects:
-            # Store object area to use in bounding box labels
+            # find the matching region
+            region = self.regions[obj['region_id']]
+
+            # Compute some extra properties
+            obj.update({
+                'xmin': int((obj['box'][0] * region['size']) + region['x_offset']),
+                'ymin': int((obj['box'][1] * region['size']) + region['y_offset']),
+                'xmax': int((obj['box'][2] * region['size']) + region['x_offset']),
+                'ymax': int((obj['box'][3] * region['size']) + region['y_offset'])
+            })
+            
+            # Compute the area
             obj['area'] = (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin'])
 
-            if obj['name'] == 'person':
-                # find the matching region
-                region = None
-                for r in self.regions:
-                    if (
-                            obj['xmin'] >= r['x_offset'] and
-                            obj['ymin'] >= r['y_offset'] and
-                            obj['xmax'] <= r['x_offset']+r['size'] and
-                            obj['ymax'] <= r['y_offset']+r['size']
-                        ): 
-                        region = r
-                        break
+            object_name = obj['name']
+
+            if object_name in region['objects']:
+                obj_settings = region['objects'][object_name]
+
+                # if the min area is larger than the
+                # detected object, don't add it to detected objects
+                if obj_settings.get('min_area',-1) > obj['area']:
+                    continue
                 
-                # if the min person area is larger than the
-                # detected person, don't add it to detected objects
-                if region and 'min_person_area' in region and region['min_person_area'] > obj['area']:
+                # if the detected object is larger than the
+                # max area, don't add it to detected objects
+                if obj_settings.get('max_area', region['size']**2) < obj['area']:
+                    continue
+
+                # if the score is lower than the threshold, skip
+                if obj_settings.get('threshold', 0) > obj['score']:
                     continue
             
-                # compute the coordinates of the person and make sure
+                # compute the coordinates of the object and make sure
                 # the location isnt outside the bounds of the image (can happen from rounding)
                 y_location = min(int(obj['ymax']), len(self.mask)-1)
                 x_location = min(int((obj['xmax']-obj['xmin'])/2.0)+obj['xmin'], len(self.mask[0])-1)
 
-                # if the person is in a masked location, continue
+                # if the object is in a masked location, don't add it to detected objects
                 if self.mask[y_location][x_location] == [0]:
                     continue
 
@@ -291,9 +310,9 @@ class Camera:
 
         with self.objects_parsed:
             self.objects_parsed.notify_all()
-
-    def get_best_person(self):
-        return self.best_person_frame.best_frame
+    
+    def get_best(self, label):
+        return self.best_frames.best_frames.get(label)
     
     def get_current_frame_with_objects(self):
         # make a copy of the current detected objects
@@ -301,22 +320,39 @@ class Camera:
         # lock and make a copy of the current frame
         with self.frame_lock:
             frame = self.current_frame.copy()
+            frame_time = self.frame_time.value
+        
+        if frame_time == self.cached_frame_with_objects['frame_time']:
+            return self.cached_frame_with_objects['frame_bytes']
 
         # draw the bounding boxes on the screen
         for obj in detected_objects:
-            label = "{}: {}% {}".format(obj['name'],int(obj['score']*100),int(obj['area']))
-            draw_box_with_label(frame, obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'], label)
+            draw_box_with_label(frame, obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'], obj['name'], obj['score'], obj['area'])
 
         for region in self.regions:
             color = (255,255,255)
             cv2.rectangle(frame, (region['x_offset'], region['y_offset']), 
                 (region['x_offset']+region['size'], region['y_offset']+region['size']), 
                 color, 2)
+        
+        # print a timestamp
+        time_to_show = datetime.datetime.fromtimestamp(frame_time).strftime("%m/%d/%Y %H:%M:%S")
+        cv2.putText(frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
 
         # convert to BGR
         frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
 
-        return frame
+        # encode the image into a jpg
+        ret, jpg = cv2.imencode('.jpg', frame)
+
+        frame_bytes = jpg.tobytes()
+
+        self.cached_frame_with_objects = {
+            'frame_bytes': frame_bytes,
+            'frame_time': frame_time
+        }
+
+        return frame_bytes