update config example

* Add object size to the bounding box

Remove script from Dockerfile

Fix framerate command

Move default value for framerate

update dockerfile

dockerfile changes

Add person_area label to surrounding box


Update dockerfile


ffmpeg config bug


Add `person_area` label to `best_person` frame


Resolve debug view showing area label for non-persons


Add object size to the bounding box


Add object size to the bounding box

* Move object area outside of conditional to work with all object types

.dockerignore

@@ -1 +1,6 @@
-README.md
+README.md
+diagram.png
+.gitignore
+debug
+config/
+*.pyc

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: blakeblackshear

.gitignore

@@ -1,2 +1,4 @@
 *.pyc 
 debug
+.vscode
+config/config.yml

Dockerfile

@@ -1,90 +1,53 @@
-FROM ubuntu:16.04
+FROM debian:stretch-slim
+LABEL maintainer "blakeb@blakeshome.com"
 
-# Install system packages
-RUN apt-get -qq update && apt-get -qq install --no-install-recommends -y python3 \ 
- python3-dev \
- python-pil \
- python-lxml \
- python-tk \
- build-essential \
- cmake \ 
- git \ 
- libgtk2.0-dev \ 
- pkg-config \ 
- libavcodec-dev \ 
- libavformat-dev \ 
- libswscale-dev \ 
- libtbb2 \
- libtbb-dev \ 
- libjpeg-dev \
- libpng-dev \
- libtiff-dev \
- libjasper-dev \
- libdc1394-22-dev \
- x11-apps \
- wget \
- vim \
- ffmpeg \
- unzip \
- && rm -rf /var/lib/apt/lists/* 
+ENV DEBIAN_FRONTEND=noninteractive
+# Install packages for apt repo
+RUN apt -qq update && apt -qq install --no-install-recommends -y \
+    apt-transport-https ca-certificates \
+    gnupg wget \
+    ffmpeg \
+    python3 \
+    python3-pip \
+    python3-dev \
+    python3-numpy \
+    # python-prctl
+    build-essential libcap-dev \
+    # pillow-simd
+    # zlib1g-dev libjpeg-dev \
+    # VAAPI drivers for Intel hardware accel
+    i965-va-driver vainfo \
+    && echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" > /etc/apt/sources.list.d/coral-edgetpu.list \
+    && wget -q -O - https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
+    && apt -qq update \
+    && echo "libedgetpu1-max libedgetpu/accepted-eula boolean true" | debconf-set-selections \
+    && apt -qq install --no-install-recommends -y \
+    libedgetpu1-max \
+    python3-edgetpu \
+    && rm -rf /var/lib/apt/lists/* \
+    && (apt-get autoremove -y; apt-get autoclean -y)
 
-# Install core packages 
-RUN wget -q -O /tmp/get-pip.py --no-check-certificate https://bootstrap.pypa.io/get-pip.py && python3 /tmp/get-pip.py
-RUN  pip install -U pip \
- numpy \
- matplotlib \
- notebook \
- jupyter \
- pandas \
- moviepy \
- tensorflow \
- keras \
- autovizwidget \
- Flask \
- imutils \
- paho-mqtt
+# needs to be installed before others
+RUN pip3 install -U wheel setuptools
 
-# Install tensorflow models object detection
-RUN GIT_SSL_NO_VERIFY=true git clone -q https://github.com/tensorflow/models /usr/local/lib/python3.5/dist-packages/tensorflow/models
-RUN wget -q -P /usr/local/src/ --no-check-certificate https://github.com/google/protobuf/releases/download/v3.5.1/protobuf-python-3.5.1.tar.gz
+RUN pip3 install -U \
+    opencv-python-headless \
+    python-prctl \
+    Flask \
+    paho-mqtt \
+    PyYAML \
+    matplotlib \
+    scipy
 
-# Download & build protobuf-python
-RUN cd /usr/local/src/ \
- && tar xf protobuf-python-3.5.1.tar.gz \
- && rm protobuf-python-3.5.1.tar.gz \
- && cd /usr/local/src/protobuf-3.5.1/ \
- && ./configure \
- && make \
- && make install \
- && ldconfig \
- && rm -rf /usr/local/src/protobuf-3.5.1/
-
-# Add dataframe display widget
-RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
-
-# Download & build OpenCV
-RUN wget -q -P /usr/local/src/ --no-check-certificate https://github.com/opencv/opencv/archive/4.0.1.zip
-RUN cd /usr/local/src/ \
- && unzip 4.0.1.zip \
- && rm 4.0.1.zip \
- && cd /usr/local/src/opencv-4.0.1/ \
- && mkdir build \
- && cd /usr/local/src/opencv-4.0.1/build \ 
- && cmake -D CMAKE_INSTALL_TYPE=Release -D CMAKE_INSTALL_PREFIX=/usr/local/ .. \
- && make -j4 \
- && make install \
- && rm -rf /usr/local/src/opencv-4.0.1
-
-# Minimize image size 
-RUN (apt-get autoremove -y; \
-     apt-get autoclean -y)
-
-# Set TF object detection available
-ENV PYTHONPATH "$PYTHONPATH:/usr/local/lib/python3.5/dist-packages/tensorflow/models/research:/usr/local/lib/python3.5/dist-packages/tensorflow/models/research/slim"
-RUN cd /usr/local/lib/python3.5/dist-packages/tensorflow/models/research && protoc object_detection/protos/*.proto --python_out=.
+# symlink the model and labels
+RUN wget -q https://github.com/google-coral/edgetpu/raw/master/test_data/mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite -O mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite --trust-server-names
+RUN wget -q https://dl.google.com/coral/canned_models/coco_labels.txt -O coco_labels.txt --trust-server-names
+RUN ln -s mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite /frozen_inference_graph.pb
+RUN ln -s /coco_labels.txt /label_map.pbtext
 
 WORKDIR /opt/frigate/
 ADD frigate frigate/
 COPY detect_objects.py .
+COPY benchmark.py .
 
-CMD ["python3", "-u", "detect_objects.py"]
+CMD ["python3", "-u", "detect_objects.py"]

README.md

@@ -1,18 +1,18 @@
-# Frigate - Realtime Object Detection for RTSP Cameras
-Uses OpenCV and Tensorflow to perform realtime object detection locally for RTSP cameras. Designed for integration with HomeAssistant or others via MQTT.
+# Frigate - Realtime Object Detection for IP Cameras
+**Note:** This version requires the use of a [Google Coral USB Accelerator](https://coral.withgoogle.com/products/accelerator/)
+
+Uses OpenCV and Tensorflow to perform realtime object detection locally for IP cameras. Designed for integration with HomeAssistant or others via MQTT.
 
 - Leverages multiprocessing and threads heavily with an emphasis on realtime over processing every frame
-- Allows you to define specific regions (squares) in the image to look for motion/objects
-- Motion detection runs in a separate process per region and signals to object detection to avoid wasting CPU cycles looking for objects when there is no motion
-- Object detection with Tensorflow runs in a separate process per region
-- Detected objects are placed on a shared mp.Queue and aggregated into a list of recently detected objects in a separate thread
-- A person score is calculated as the sum of all scores/5
-- Motion and object info is published over MQTT for integration into HomeAssistant or others
+- Allows you to define specific regions (squares) in the image to look for objects
+- No motion detection (for now)
+- Object detection with Tensorflow runs in a separate thread
+- Object info is published over MQTT for integration into HomeAssistant as a binary sensor
 - An endpoint is available to view an MJPEG stream for debugging
 
 ![Diagram](diagram.png)
 
-## Example video
+## Example video (from older version)
 You see multiple bounding boxes because it draws bounding boxes from all frames in the past 1 second where a person was detected. Not all of the bounding boxes were from the current frame.
 [![](http://img.youtube.com/vi/nqHbCtyo4dY/0.jpg)](http://www.youtube.com/watch?v=nqHbCtyo4dY "Frigate")
 
@@ -22,24 +22,17 @@ Build the container with
 docker build -t frigate .
 ```
 
-Download a model from the [zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md).
-
-Download the cooresponding label map from [here](https://github.com/tensorflow/models/tree/master/research/object_detection/data).
+The `mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite` model is included and used by default. You can use your own model and labels by mounting files in the container at `/frozen_inference_graph.pb` and `/label_map.pbtext`. Models must be compatible with the Coral according to [this](https://coral.withgoogle.com/models/).
 
 Run the container with
 ```
 docker run --rm \
--v <path_to_frozen_detection_graph.pb>:/frozen_inference_graph.pb:ro \
--v <path_to_labelmap.pbtext>:/label_map.pbtext:ro \
+--privileged \
+-v /dev/bus/usb:/dev/bus/usb \
 -v <path_to_config_dir>:/config:ro \
+-v /etc/localtime:/etc/localtime:ro \
 -p 5000:5000 \
--e RTSP_URL='<rtsp_url>' \
--e REGIONS='<box_size_1>,<x_offset_1>,<y_offset_1>,<min_person_size_1>,<min_motion_size_1>,<mask_file_1>:<box_size_2>,<x_offset_2>,<y_offset_2>,<min_person_size_2>,<min_motion_size_2>,<mask_file_2>' \
--e MQTT_HOST='your.mqtthost.com' \
--e MQTT_USER='username' \
--e MQTT_PASS='password' \
--e MQTT_TOPIC_PREFIX='cameras/1' \
--e DEBUG='0' \
+-e FRIGATE_RTSP_PASSWORD='password' \
 frigate:latest
 ```
 
@@ -48,100 +41,82 @@ Example docker-compose:
   frigate:
     container_name: frigate
     restart: unless-stopped
+    privileged: true
     image: frigate:latest
     volumes:
-      - <path_to_frozen_detection_graph.pb>:/frozen_inference_graph.pb:ro
-      - <path_to_labelmap.pbtext>:/label_map.pbtext:ro
+      - /dev/bus/usb:/dev/bus/usb
+      - /etc/localtime:/etc/localtime:ro
       - <path_to_config>:/config
     ports:
-      - "127.0.0.1:5000:5000"
+      - "5000:5000"
     environment:
-      RTSP_URL: "<rtsp_url>"
-      REGIONS: "<box_size_1>,<x_offset_1>,<y_offset_1>,<min_person_size_1>,<min_motion_size_1>,<mask_file_1>:<box_size_2>,<x_offset_2>,<y_offset_2>,<min_person_size_2>,<min_motion_size_2>,<mask_file_2>"
-      MQTT_HOST: "your.mqtthost.com"
-      MQTT_USER: "username" #optional
-      MQTT_PASS: "password" #optional
-      MQTT_TOPIC_PREFIX: "cameras/1"
-      DEBUG: "0"
+      FRIGATE_RTSP_PASSWORD: "password"
 ```
 
-Here is an example `REGIONS` env variable:
-`350,0,300,5000,200,mask-0-300.bmp:400,350,250,2000,200,mask-350-250.bmp:400,750,250,2000,200,mask-750-250.bmp`
+A `config.yml` file must exist in the `config` directory. See example [here](config/config.example.yml) and device specific info can be found [here](docs/DEVICES.md).
 
-First region broken down (all are required):
-- `350` - size of the square (350px by 350px)
-- `0` - x coordinate of upper left corner (top left of image is 0,0)
-- `300` - y coordinate of upper left corner (top left of image is 0,0)
-- `5000` - minimum person bounding box size (width*height for bounding box of identified person)
-- `200` - minimum number of changed pixels to trigger motion
-- `mask-0-300.bmp` - a bmp file with the masked regions as pure black, must be the same size as the region
-
-Mask files go in the `/config` directory.
-
-Access the mjpeg stream at http://localhost:5000
+Access the mjpeg stream at `http://localhost:5000/<camera_name>` and the best snapshot for any object type with at `http://localhost:5000/<camera_name>/<object_name>/best.jpg`
 
 ## Integration with HomeAssistant
 ```
 camera:
   - name: Camera Last Person
-    platform: generic
-    still_image_url: http://<ip>:5000/best_person.jpg
+    platform: mqtt
+    topic: frigate/<camera_name>/person/snapshot
+  - name: Camera Last Car
+    platform: mqtt
+    topic: frigate/<camera_name>/car/snapshot
 
 binary_sensor:
-  - name: Camera Motion
+  - name: Camera Person
     platform: mqtt
-    state_topic: "cameras/1/motion"
+    state_topic: "frigate/<camera_name>/person"
     device_class: motion
-    availability_topic: "cameras/1/available"
+    availability_topic: "frigate/available"
 
-sensor:
-  - name: Camera Person Score
-    platform: mqtt
-    state_topic: "cameras/1/objects"
-    value_template: '{{ value_json.person }}'
-    unit_of_measurement: '%'
-    availability_topic: "cameras/1/available"
+automation:
+  - alias: Alert me if a person is detected while armed away
+    trigger: 
+      platform: state
+      entity_id: binary_sensor.camera_person
+      from: 'off'
+      to: 'on'
+    condition:
+      - condition: state
+        entity_id: alarm_control_panel.home_alarm
+        state: armed_away
+    action:
+      - service: notify.user_telegram
+        data:
+          message: "A person was detected."
+          data:
+            photo:
+              - url: http://<ip>:5000/<camera_name>/person/best.jpg
+                caption: A person was detected.
 ```
 
 ## Tips
-- Lower the framerate of the RTSP feed on the camera to reduce the CPU usage for capturing the feed
-- Use SSDLite models to reduce CPU usage
+- Lower the framerate of the video feed on the camera to reduce the CPU usage for capturing the feed
 
 ## Future improvements
-- [ ] Build tensorflow from source for CPU optimizations
+- [x] Remove motion detection for now
+- [x] Try running object detection in a thread rather than a process
+- [x] Implement min person size again
+- [x] Switch to a config file
+- [x] Handle multiple cameras in the same container
+- [ ] Attempt to figure out coral symlinking
+- [ ] Add object list to config with min scores for mqtt
+- [ ] Move mjpeg encoding to a separate process
+- [ ] Simplify motion detection (check entire image against mask, resize instead of gaussian blur)
+- [ ] See if motion detection is even worth running
+- [ ] Scan for people across entire image rather than specfic regions
+- [ ] Dynamically resize detection area and follow people
 - [ ] Add ability to turn detection on and off via MQTT
-- [ ] MQTT motion occasionally gets stuck ON
 - [ ] Output movie clips of people for notifications, etc.
 - [ ] Integrate with homeassistant push camera
 - [ ] Merge bounding boxes that span multiple regions
-- [ ] Switch to a config file
-- [ ] Allow motion regions to be different than object detection regions
 - [ ] Implement mode to save labeled objects for training
 - [ ] Try and reduce CPU usage by simplifying the tensorflow model to just include the objects we care about
 - [ ] Look into GPU accelerated decoding of RTSP stream
 - [ ] Send video over a socket and use JSMPEG
-- [ ] Look into neural compute stick
-
-## Building Tensorflow from source for CPU optimizations
-https://www.tensorflow.org/install/source#docker_linux_builds
-used `tensorflow/tensorflow:1.12.0-devel-py3`
-
-## Optimizing the graph (cant say I saw much difference in CPU usage)
-https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/graph_transforms/README.md#optimizing-for-deployment
-```
-docker run -it -v ${PWD}:/lab -v ${PWD}/../back_camera_model/models/ssd_mobilenet_v2_coco_2018_03_29/frozen_inference_graph.pb:/frozen_inference_graph.pb:ro tensorflow/tensorflow:1.12.0-devel-py3 bash
-
-bazel build tensorflow/tools/graph_transforms:transform_graph
-
-bazel-bin/tensorflow/tools/graph_transforms/transform_graph \
---in_graph=/frozen_inference_graph.pb \
---out_graph=/lab/optimized_inception_graph.pb \
---inputs='image_tensor' \
---outputs='num_detections,detection_scores,detection_boxes,detection_classes' \
---transforms='
-  strip_unused_nodes(type=float, shape="1,300,300,3")
-  remove_nodes(op=Identity, op=CheckNumerics)
-  fold_constants(ignore_errors=true)
-  fold_batch_norms
-  fold_old_batch_norms'
-```
+- [x] Look into neural compute stick

benchmark.py

@@ -0,0 +1,20 @@
+import statistics
+import numpy as np
+from edgetpu.detection.engine import DetectionEngine
+
+# Path to frozen detection graph. This is the actual model that is used for the object detection.
+PATH_TO_CKPT = '/frozen_inference_graph.pb'
+
+# Load the edgetpu engine and labels
+engine = DetectionEngine(PATH_TO_CKPT)
+
+frame = np.zeros((300,300,3), np.uint8)
+flattened_frame = np.expand_dims(frame, axis=0).flatten()
+
+detection_times = []
+
+for x in range(0, 1000):
+    objects = engine.detect_with_input_tensor(flattened_frame, threshold=0.1, top_k=3)
+    detection_times.append(engine.get_inference_time())
+
+print("Average inference time: " + str(statistics.mean(detection_times)))

config/config.example.yml

@@ -0,0 +1,130 @@
+web_port: 5000
+
+mqtt:
+  host: mqtt.server.com
+  topic_prefix: frigate
+#  client_id: frigate # Optional -- set to override default client id of 'frigate' if running multiple instances
+#  user: username # Optional -- Uncomment for use
+#  password: password # Optional -- Uncomment for use
+
+#################
+# Default ffmpeg args. Optional and can be overwritten per camera.
+# Should work with most RTSP cameras that send h264 video
+# Built from the properties below with:
+# "ffmpeg" + global_args + input_args + "-i" + input + output_args
+#################
+# ffmpeg:
+#   global_args:
+#     - -hide_banner
+#     - -loglevel
+#     - panic
+#   hwaccel_args: []
+#   input_args:
+#     - -avoid_negative_ts
+#     - make_zero
+#     - -fflags
+#     - nobuffer
+#     - -flags
+#     - low_delay
+#     - -strict
+#     - experimental
+#     - -fflags
+#     - +genpts+discardcorrupt
+#     - -vsync
+#     - drop
+#     - -rtsp_transport
+#     - tcp
+#     - -stimeout
+#     - '5000000'
+#     - -use_wallclock_as_timestamps
+#     - '1'
+#   output_args:
+#     - -vf
+#     - mpdecimate
+#     - -f
+#     - rawvideo
+#     - -pix_fmt
+#     - rgb24
+
+####################
+# Global object configuration. Applies to all cameras
+# unless overridden at the camera levels.
+# Keys must be valid labels. By default, the model uses coco (https://dl.google.com/coral/canned_models/coco_labels.txt).
+# All labels from the model are reported over MQTT. These values are used to filter out false positives.
+# min_area (optional): minimum width*height of the bounding box for the detected person
+# max_area (optional): maximum width*height of the bounding box for the detected person
+# threshold (optional): The minimum decimal percentage (50% hit = 0.5) for the confidence from tensorflow
+####################
+objects:
+  track:
+    - person
+    - car
+    - truck
+  filters:
+    person:
+      min_area: 5000
+      max_area: 100000
+      threshold: 0.5
+
+cameras:
+  back:
+    ffmpeg:
+      ################
+      # Source passed to ffmpeg after the -i parameter. Supports anything compatible with OpenCV and FFmpeg.
+      # Environment variables that begin with 'FRIGATE_' may be referenced in {}
+      ################
+      input: rtsp://viewer:{FRIGATE_RTSP_PASSWORD}@10.0.10.10:554/cam/realmonitor?channel=1&subtype=2
+      #################
+      # These values will override default values for just this camera
+      #################
+      # global_args: []
+      # hwaccel_args: []
+      # input_args: []
+      # output_args: []
+
+    ################
+    ## Optional mask. Must be the same dimensions as your video feed.
+    ## The mask works by looking at the bottom center of the bounding box for the detected
+    ## person in the image. If that pixel in the mask is a black pixel, it ignores it as a
+    ## false positive. In my mask, the grass and driveway visible from my backdoor camera 
+    ## are white. The garage doors, sky, and trees (anywhere it would be impossible for a 
+    ## person to stand) are black.
+    ################
+    # mask: back-mask.bmp
+
+    ################
+    # Allows you to limit the framerate within frigate for cameras that do not support
+    # custom framerates. A value of 1 tells frigate to look at every frame, 2 every 2nd frame, 
+    # 3 every 3rd frame, etc.
+    ################
+    take_frame: 1
+
+    ################
+    # Overrides for global object config
+    ################
+    objects:
+      track:
+        - person
+      filters:
+        person:
+          min_area: 5000
+          max_area: 100000
+          threshold: 0.5
+    
+    ################
+    # size: size of the region in pixels
+    # x_offset/y_offset: position of the upper left corner of your region (top left of image is 0,0)
+    # Tips: All regions are resized to 300x300 before detection because the model is trained on that size.
+    #       Resizing regions takes CPU power. Ideally, all regions should be as close to 300x300 as possible.
+    #       Defining a region that goes outside the bounds of the image will result in errors.
+    ################
+    regions:
+      - size: 350
+        x_offset: 0
+        y_offset: 300
+      - size: 400
+        x_offset: 350
+        y_offset: 250
+      - size: 400
+        x_offset: 750
+        y_offset: 250

detect_objects.py

@@ -1,247 +1,155 @@
-import os
 import cv2
-import imutils
 import time
-import datetime
-import ctypes
-import logging
-import multiprocessing as mp
-import threading
-import json
-from contextlib import closing
+import queue
+import yaml
 import numpy as np
-from object_detection.utils import visualization_utils as vis_util
-from flask import Flask, Response, make_response, send_file
+from flask import Flask, Response, make_response, jsonify
 import paho.mqtt.client as mqtt
 
-from frigate.util import tonumpyarray
-from frigate.mqtt import MqttMotionPublisher, MqttObjectPublisher
-from frigate.objects import ObjectParser, ObjectCleaner, BestPersonFrame
-from frigate.motion import detect_motion
-from frigate.video import fetch_frames, FrameTracker
-from frigate.object_detection import detect_objects
+from frigate.video import Camera
+from frigate.object_detection import PreppedQueueProcessor
+from frigate.util import EventsPerSecond
 
-RTSP_URL = os.getenv('RTSP_URL')
+with open('/config/config.yml') as f:
+    CONFIG = yaml.safe_load(f)
 
-MQTT_HOST = os.getenv('MQTT_HOST')
-MQTT_USER = os.getenv('MQTT_USER')
-MQTT_PASS = os.getenv('MQTT_PASS')
-MQTT_TOPIC_PREFIX = os.getenv('MQTT_TOPIC_PREFIX')
+MQTT_HOST = CONFIG['mqtt']['host']
+MQTT_PORT = CONFIG.get('mqtt', {}).get('port', 1883)
+MQTT_TOPIC_PREFIX = CONFIG.get('mqtt', {}).get('topic_prefix', 'frigate')
+MQTT_USER = CONFIG.get('mqtt', {}).get('user')
+MQTT_PASS = CONFIG.get('mqtt', {}).get('password')
+MQTT_CLIENT_ID = CONFIG.get('mqtt', {}).get('client_id', 'frigate')
 
-# REGIONS = "350,0,300,50:400,350,250,50:400,750,250,50"
-# REGIONS = "400,350,250,50"
-REGIONS = os.getenv('REGIONS')
+# Set the default FFmpeg config
+FFMPEG_CONFIG = CONFIG.get('ffmpeg', {})
+FFMPEG_DEFAULT_CONFIG = {
+    'global_args': FFMPEG_CONFIG.get('global_args', 
+        ['-hide_banner','-loglevel','panic']),
+    'hwaccel_args': FFMPEG_CONFIG.get('hwaccel_args', 
+        []),
+    'input_args': FFMPEG_CONFIG.get('input_args',
+        ['-avoid_negative_ts', 'make_zero',
+         '-fflags', 'nobuffer',
+         '-flags', 'low_delay',
+         '-strict', 'experimental',
+         '-fflags', '+genpts+discardcorrupt',
+         '-vsync', 'drop',
+         '-rtsp_transport', 'tcp',
+         '-stimeout', '5000000',
+         '-use_wallclock_as_timestamps', '1']),
+    'output_args': FFMPEG_CONFIG.get('output_args',
+        ['-vf', 'mpdecimate',
+         '-f', 'rawvideo',
+         '-pix_fmt', 'rgb24'])
+}
 
-DEBUG = (os.getenv('DEBUG') == '1')
+GLOBAL_OBJECT_CONFIG = CONFIG.get('objects', {})
+
+WEB_PORT = CONFIG.get('web_port', 5000)
+DEBUG = (CONFIG.get('debug', '0') == '1')
 
 def main():
-    DETECTED_OBJECTS = []
-    recent_motion_frames = {}
-    # Parse selected regions
-    regions = []
-    for region_string in REGIONS.split(':'):
-        region_parts = region_string.split(',')
-        region_mask_image = cv2.imread("/config/{}".format(region_parts[5]), cv2.IMREAD_GRAYSCALE)
-        region_mask = np.where(region_mask_image==[0])
-        regions.append({
-            'size': int(region_parts[0]),
-            'x_offset': int(region_parts[1]),
-            'y_offset': int(region_parts[2]),
-            'min_person_area': int(region_parts[3]),
-            'min_object_size': int(region_parts[4]),
-            'mask': region_mask,
-            # Event for motion detection signaling
-            'motion_detected': mp.Event(),
-            # create shared array for storing 10 detected objects
-            # note: this must be a double even though the value you are storing
-            #       is a float. otherwise it stops updating the value in shared
-            #       memory. probably something to do with the size of the memory block
-            'output_array': mp.Array(ctypes.c_double, 6*10)
-        })
-    # capture a single frame and check the frame shape so the correct array
-    # size can be allocated in memory
-    video = cv2.VideoCapture(RTSP_URL)
-    ret, frame = video.read()
-    if ret:
-        frame_shape = frame.shape
-    else:
-        print("Unable to capture video stream")
-        exit(1)
-    video.release()
-        
-    # compute the flattened array length from the array shape
-    flat_array_length = frame_shape[0] * frame_shape[1] * frame_shape[2]
-    # create shared array for storing the full frame image data
-    shared_arr = mp.Array(ctypes.c_uint16, flat_array_length)
-    # create shared value for storing the frame_time
-    shared_frame_time = mp.Value('d', 0.0)
-    # Lock to control access to the frame
-    frame_lock = mp.Lock()
-    # Condition for notifying that a new frame is ready
-    frame_ready = mp.Condition()
-    # Condition for notifying that motion status changed globally
-    motion_changed = mp.Condition()
-    # Condition for notifying that objects were parsed
-    objects_parsed = mp.Condition()
-    # Queue for detected objects
-    object_queue = mp.Queue()
-
-    # shape current frame so it can be treated as an image
-    frame_arr = tonumpyarray(shared_arr).reshape(frame_shape)
-
-    # start the process to capture frames from the RTSP stream and store in a shared array
-    capture_process = mp.Process(target=fetch_frames, args=(shared_arr, 
-        shared_frame_time, frame_lock, frame_ready, frame_shape, RTSP_URL))
-    capture_process.daemon = True
-
-    # for each region, start a separate process for motion detection and object detection
-    detection_processes = []
-    motion_processes = []
-    for region in regions:
-        detection_process = mp.Process(target=detect_objects, args=(shared_arr, 
-            object_queue,
-            shared_frame_time,
-            frame_lock, frame_ready,
-            region['motion_detected'],
-            frame_shape, 
-            region['size'], region['x_offset'], region['y_offset'],
-            region['min_person_area'],
-            DEBUG))
-        detection_process.daemon = True
-        detection_processes.append(detection_process)
-
-        motion_process = mp.Process(target=detect_motion, args=(shared_arr,
-            shared_frame_time,
-            frame_lock, frame_ready,
-            region['motion_detected'],
-            motion_changed,
-            frame_shape, 
-            region['size'], region['x_offset'], region['y_offset'],
-            region['min_object_size'], region['mask'],
-            DEBUG))
-        motion_process.daemon = True
-        motion_processes.append(motion_process)
-
-    # start a thread to store recent motion frames for processing
-    frame_tracker = FrameTracker(frame_arr, shared_frame_time, frame_ready, frame_lock, 
-        recent_motion_frames, motion_changed, [region['motion_detected'] for region in regions])
-    frame_tracker.start()
-
-    # start a thread to store the highest scoring recent person frame
-    best_person_frame = BestPersonFrame(objects_parsed, recent_motion_frames, DETECTED_OBJECTS, 
-        motion_changed, [region['motion_detected'] for region in regions])
-    best_person_frame.start()
-
-    # start a thread to parse objects from the queue
-    object_parser = ObjectParser(object_queue, objects_parsed, DETECTED_OBJECTS)
-    object_parser.start()
-    # start a thread to expire objects from the detected objects list
-    object_cleaner = ObjectCleaner(objects_parsed, DETECTED_OBJECTS)
-    object_cleaner.start()
-
     # connect to mqtt and setup last will
-    def on_connect(client, userdata, flags, rc): 
+    def on_connect(client, userdata, flags, rc):
         print("On connect called")
+        if rc != 0:
+            if rc == 3:
+                print ("MQTT Server unavailable")
+            elif rc == 4:
+                print ("MQTT Bad username or password")
+            elif rc == 5:
+                print ("MQTT Not authorized")
+            else:
+                print ("Unable to connect to MQTT: Connection refused. Error code: " + str(rc))
         # publish a message to signal that the service is running
         client.publish(MQTT_TOPIC_PREFIX+'/available', 'online', retain=True)
-    client = mqtt.Client()
+    client = mqtt.Client(client_id=MQTT_CLIENT_ID)
     client.on_connect = on_connect
     client.will_set(MQTT_TOPIC_PREFIX+'/available', payload='offline', qos=1, retain=True)
     if not MQTT_USER is None:
         client.username_pw_set(MQTT_USER, password=MQTT_PASS)
-
-    client.connect(MQTT_HOST, 1883, 60)
+    client.connect(MQTT_HOST, MQTT_PORT, 60)
     client.loop_start()
-
-    # start a thread to publish object scores (currently only person)
-    mqtt_publisher = MqttObjectPublisher(client, MQTT_TOPIC_PREFIX, objects_parsed, DETECTED_OBJECTS)
-    mqtt_publisher.start()
-
-    # start thread to publish motion status
-    mqtt_motion_publisher = MqttMotionPublisher(client, MQTT_TOPIC_PREFIX, motion_changed,
-        [region['motion_detected'] for region in regions])
-    mqtt_motion_publisher.start()
-
-    # start the process of capturing frames
-    capture_process.start()
-    print("capture_process pid ", capture_process.pid)
-
-    # start the object detection processes
-    for detection_process in detection_processes:
-        detection_process.start()
-        print("detection_process pid ", detection_process.pid)
     
-    # start the motion detection processes
-    for motion_process in motion_processes:
-        motion_process.start()
-        print("motion_process pid ", motion_process.pid)
+    # Queue for prepped frames, max size set to number of regions * 3
+    prepped_frame_queue = queue.Queue()
+
+    cameras = {}
+    for name, config in CONFIG['cameras'].items():
+        cameras[name] = Camera(name, FFMPEG_DEFAULT_CONFIG, GLOBAL_OBJECT_CONFIG, config, 
+            prepped_frame_queue, client, MQTT_TOPIC_PREFIX)
+
+    fps_tracker = EventsPerSecond()
+
+    prepped_queue_processor = PreppedQueueProcessor(
+        cameras,
+        prepped_frame_queue,
+        fps_tracker
+    )
+    prepped_queue_processor.start()
+    fps_tracker.start()
+
+    for name, camera in cameras.items():
+        camera.start()
+        print("Capture process for {}: {}".format(name, camera.get_capture_pid()))
 
     # create a flask app that encodes frames a mjpeg on demand
     app = Flask(__name__)
 
-    @app.route('/best_person.jpg')
-    def best_person():
-        frame = np.zeros(frame_shape, np.uint8) if best_person_frame.best_frame is None else best_person_frame.best_frame
-        ret, jpg = cv2.imencode('.jpg', frame)
-        response = make_response(jpg.tobytes())
-        response.headers['Content-Type'] = 'image/jpg'
-        return response
-
     @app.route('/')
-    def index():
-        # return a multipart response
-        return Response(imagestream(),
-                        mimetype='multipart/x-mixed-replace; boundary=frame')
-    def imagestream():
+    def ishealthy():
+        # return a healh
+        return "Frigate is running. Alive and healthy!"
+
+    @app.route('/debug/stats')
+    def stats():
+        stats = {
+            'coral': {
+                'fps': fps_tracker.eps(),
+                'inference_speed': prepped_queue_processor.avg_inference_speed,
+                'queue_length': prepped_frame_queue.qsize()
+            }
+        }
+
+        for name, camera in cameras.items():
+            stats[name] = camera.stats()
+
+        return jsonify(stats)
+
+    @app.route('/<camera_name>/<label>/best.jpg')
+    def best(camera_name, label):
+        if camera_name in cameras:
+            best_frame = cameras[camera_name].get_best(label)
+            if best_frame is None:
+                best_frame = np.zeros((720,1280,3), np.uint8)
+            best_frame = cv2.cvtColor(best_frame, cv2.COLOR_RGB2BGR)
+            ret, jpg = cv2.imencode('.jpg', best_frame)
+            response = make_response(jpg.tobytes())
+            response.headers['Content-Type'] = 'image/jpg'
+            return response
+        else:
+            return "Camera named {} not found".format(camera_name), 404
+
+    @app.route('/<camera_name>')
+    def mjpeg_feed(camera_name):
+        if camera_name in cameras:
+            # return a multipart response
+            return Response(imagestream(camera_name),
+                            mimetype='multipart/x-mixed-replace; boundary=frame')
+        else:
+            return "Camera named {} not found".format(camera_name), 404
+
+    def imagestream(camera_name):
         while True:
-            # max out at 5 FPS
-            time.sleep(0.2)
-            # make a copy of the current detected objects
-            detected_objects = DETECTED_OBJECTS.copy()
-            # lock and make a copy of the current frame
-            with frame_lock:
-                frame = frame_arr.copy()
-            # convert to RGB for drawing
-            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
-            # draw the bounding boxes on the screen
-            for obj in detected_objects:
-                vis_util.draw_bounding_box_on_image_array(frame,
-                    obj['ymin'],
-                    obj['xmin'],
-                    obj['ymax'],
-                    obj['xmax'],
-                    color='red',
-                    thickness=2,
-                    display_str_list=["{}: {}%".format(obj['name'],int(obj['score']*100))],
-                    use_normalized_coordinates=False)
-
-            for region in regions:
-                color = (255,255,255)
-                if region['motion_detected'].is_set():
-                    color = (0,255,0)
-                cv2.rectangle(frame, (region['x_offset'], region['y_offset']), 
-                    (region['x_offset']+region['size'], region['y_offset']+region['size']), 
-                    color, 2)
-
-            # convert back to BGR
-            frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
-            # encode the image into a jpg
-            ret, jpg = cv2.imencode('.jpg', frame)
+            # max out at 1 FPS
+            time.sleep(1)
+            frame = cameras[camera_name].get_current_frame_with_objects()
             yield (b'--frame\r\n'
-                b'Content-Type: image/jpeg\r\n\r\n' + jpg.tobytes() + b'\r\n\r\n')
+                b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n\r\n')
 
-    app.run(host='0.0.0.0', debug=False)
+    app.run(host='0.0.0.0', port=WEB_PORT, debug=False)
 
-    capture_process.join()
-    for detection_process in detection_processes:
-        detection_process.join()
-    for motion_process in motion_processes:
-        motion_process.join()
-    frame_tracker.join()
-    best_person_frame.join()
-    object_parser.join()
-    object_cleaner.join()
-    mqtt_publisher.join()
+    camera.join()
 
 if __name__ == '__main__':
-    main()
+    main()

docs/DEVICES.md

@@ -0,0 +1,74 @@
+# Configuration Examples
+
+### Default (most RTSP cameras)
+This is the default ffmpeg command and should work with most RTSP cameras that send h264 video
+```yaml
+ffmpeg:
+  global_args:
+    - -hide_banner
+    - -loglevel
+    - panic
+  hwaccel_args: []
+  input_args:
+    - -avoid_negative_ts
+    - make_zero 
+    - -fflags
+    - nobuffer
+    - -flags
+    - low_delay
+    - -strict
+    - experimental
+    - -fflags
+    - +genpts+discardcorrupt
+    - -vsync
+    - drop
+    - -rtsp_transport
+    - tcp
+    - -stimeout
+    - '5000000' 
+    - -use_wallclock_as_timestamps
+    - '1'
+  output_args:
+    - -vf
+    - mpdecimate
+    - -f
+    - rawvideo
+    - -pix_fmt
+    - rgb24
+```
+
+### RTMP Cameras
+The input parameters need to be adjusted for RTMP cameras
+```yaml
+ffmpeg:
+input_args:
+    - -avoid_negative_ts
+    - make_zero
+    - -fflags
+    - nobuffer
+    - -flags
+    - low_delay
+    - -strict
+    - experimental
+    - -fflags
+    - +genpts+discardcorrupt
+    - -vsync
+    - drop
+    - -use_wallclock_as_timestamps
+    - '1'
+```
+
+
+### Hardware Acceleration
+
+Intel Quicksync
+```yaml
+ffmpeg:
+  hwaccel_args:
+    - -hwaccel
+    - vaapi
+    - -hwaccel_device
+    - /dev/dri/renderD128
+    - -hwaccel_output_format
+    - yuv420p
+```

frigate/motion.py

@@ -1,109 +0,0 @@
-import datetime
-import numpy as np
-import cv2
-import imutils
-from . util import tonumpyarray
-
-# do the actual motion detection
-def detect_motion(shared_arr, shared_frame_time, frame_lock, frame_ready, motion_detected, motion_changed,
-                  frame_shape, region_size, region_x_offset, region_y_offset, min_motion_area, mask, debug):
-    # shape shared input array into frame for processing
-    arr = tonumpyarray(shared_arr).reshape(frame_shape)
-
-    avg_frame = None
-    avg_delta = None
-    frame_time = 0.0
-    motion_frames = 0
-    while True:
-        now = datetime.datetime.now().timestamp()
-        
-        with frame_ready:
-            # if there isnt a frame ready for processing or it is old, wait for a signal
-            if shared_frame_time.value == frame_time or (now - shared_frame_time.value) > 0.5:
-                frame_ready.wait()
-        
-        # lock and make a copy of the cropped frame
-        with frame_lock: 
-            cropped_frame = arr[region_y_offset:region_y_offset+region_size, region_x_offset:region_x_offset+region_size].copy().astype('uint8')
-            frame_time = shared_frame_time.value
-
-        # convert to grayscale
-        gray = cv2.cvtColor(cropped_frame, cv2.COLOR_BGR2GRAY)
-
-        # apply image mask to remove areas from motion detection
-        gray[mask] = [255]
-
-        # apply gaussian blur
-        gray = cv2.GaussianBlur(gray, (21, 21), 0)
-
-        if avg_frame is None:
-            avg_frame = gray.copy().astype("float")
-            continue
-        
-        # look at the delta from the avg_frame
-        frameDelta = cv2.absdiff(gray, cv2.convertScaleAbs(avg_frame))
-        
-        if avg_delta is None:
-            avg_delta = frameDelta.copy().astype("float")
-
-        # compute the average delta over the past few frames
-        # the alpha value can be modified to configure how sensitive the motion detection is.
-        # higher values mean the current frame impacts the delta a lot, and a single raindrop may
-        # register as motion, too low and a fast moving person wont be detected as motion
-        # this also assumes that a person is in the same location across more than a single frame
-        cv2.accumulateWeighted(frameDelta, avg_delta, 0.2)
-
-        # compute the threshold image for the current frame
-        current_thresh = cv2.threshold(frameDelta, 25, 255, cv2.THRESH_BINARY)[1]
-
-        # black out everything in the avg_delta where there isnt motion in the current frame
-        avg_delta_image = cv2.convertScaleAbs(avg_delta)
-        avg_delta_image[np.where(current_thresh==[0])] = [0]
-
-        # then look for deltas above the threshold, but only in areas where there is a delta
-        # in the current frame. this prevents deltas from previous frames from being included
-        thresh = cv2.threshold(avg_delta_image, 25, 255, cv2.THRESH_BINARY)[1]
- 
-        # dilate the thresholded image to fill in holes, then find contours
-        # on thresholded image
-        thresh = cv2.dilate(thresh, None, iterations=2)
-        cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
-        cnts = imutils.grab_contours(cnts)
-
-        motion_found = False
-
-        # loop over the contours
-        for c in cnts:
-            # if the contour is big enough, count it as motion
-            contour_area = cv2.contourArea(c)
-            if contour_area > min_motion_area:
-                motion_found = True
-                if debug:
-                    cv2.drawContours(cropped_frame, [c], -1, (0, 255, 0), 2)
-                    x, y, w, h = cv2.boundingRect(c)
-                    cv2.putText(cropped_frame, str(contour_area), (x, y),
-		                cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 100, 0), 2)
-                else:
-                    break
-        
-        if motion_found:
-            motion_frames += 1
-            # if there have been enough consecutive motion frames, report motion
-            if motion_frames >= 3:
-                # only average in the current frame if the difference persists for at least 3 frames
-                cv2.accumulateWeighted(gray, avg_frame, 0.01)
-                motion_detected.set()
-                with motion_changed:
-                    motion_changed.notify_all()
-        else:
-            # when no motion, just keep averaging the frames together
-            cv2.accumulateWeighted(gray, avg_frame, 0.01)
-            motion_frames = 0
-            if motion_detected.is_set():
-                motion_detected.clear()
-                with motion_changed:
-                    motion_changed.notify_all()
-
-        if debug and motion_frames == 3:
-            cv2.imwrite("/lab/debug/motion-{}-{}-{}.jpg".format(region_x_offset, region_y_offset, datetime.datetime.now().timestamp()), cropped_frame)
-            cv2.imwrite("/lab/debug/avg_delta-{}-{}-{}.jpg".format(region_x_offset, region_y_offset, datetime.datetime.now().timestamp()), avg_delta_image)

frigate/mqtt.py

@@ -1,56 +1,54 @@
 import json
+import cv2
 import threading
-
-class MqttMotionPublisher(threading.Thread):
-    def __init__(self, client, topic_prefix, motion_changed, motion_flags):
-        threading.Thread.__init__(self)
-        self.client = client
-        self.topic_prefix = topic_prefix
-        self.motion_changed = motion_changed
-        self.motion_flags = motion_flags
-
-    def run(self):
-        last_sent_motion = ""
-        while True:
-            with self.motion_changed:
-                self.motion_changed.wait()
-            
-            # send message for motion
-            motion_status = 'OFF'
-            if any(obj.is_set() for obj in self.motion_flags):
-                motion_status = 'ON'
-
-            if last_sent_motion != motion_status:
-                last_sent_motion = motion_status
-                self.client.publish(self.topic_prefix+'/motion', motion_status, retain=False)
+import prctl
+from collections import Counter, defaultdict
+import itertools
 
 class MqttObjectPublisher(threading.Thread):
-    def __init__(self, client, topic_prefix, objects_parsed, detected_objects):
+    def __init__(self, client, topic_prefix, camera):
         threading.Thread.__init__(self)
         self.client = client
         self.topic_prefix = topic_prefix
-        self.objects_parsed = objects_parsed
-        self._detected_objects = detected_objects
+        self.camera = camera
 
     def run(self):
-        last_sent_payload = ""
+        prctl.set_name(self.__class__.__name__)
+        current_object_status = defaultdict(lambda: 'OFF')
         while True:
+            # wait until objects have been tracked
+            with self.camera.objects_tracked:
+                self.camera.objects_tracked.wait()
 
-            # initialize the payload
-            payload = {}
+            # count objects with more than 2 entries in history by type
+            obj_counter = Counter()
+            for obj in self.camera.object_tracker.tracked_objects.values():
+                if len(obj['history']) > 1:
+                    obj_counter[obj['name']] += 1
+            
+            # report on detected objects
+            for obj_name, count in obj_counter.items():
+                new_status = 'ON' if count > 0 else 'OFF'
+                if new_status != current_object_status[obj_name]:
+                    current_object_status[obj_name] = new_status
+                    self.client.publish(self.topic_prefix+'/'+obj_name, new_status, retain=False)
+                    # send the snapshot over mqtt if we have it as well
+                    if obj_name in self.camera.best_frames.best_frames:
+                        best_frame = cv2.cvtColor(self.camera.best_frames.best_frames[obj_name], cv2.COLOR_RGB2BGR)
+                        ret, jpg = cv2.imencode('.jpg', best_frame)
+                        if ret:
+                            jpg_bytes = jpg.tobytes()
+                            self.client.publish(self.topic_prefix+'/'+obj_name+'/snapshot', jpg_bytes, retain=True)
 
-            # wait until objects have been parsed
-            with self.objects_parsed:
-                self.objects_parsed.wait()
-
-            # add all the person scores in detected objects and 
-            # average over past 1 seconds (5fps)
-            detected_objects = self._detected_objects.copy()
-            avg_person_score = sum([obj['score'] for obj in detected_objects if obj['name'] == 'person'])/5
-            payload['person'] = int(avg_person_score*100)
-
-            # send message for objects if different
-            new_payload = json.dumps(payload, sort_keys=True)
-            if new_payload != last_sent_payload:
-                last_sent_payload = new_payload
-                self.client.publish(self.topic_prefix+'/objects', new_payload, retain=False)
+            # expire any objects that are ON and no longer detected
+            expired_objects = [obj_name for obj_name, status in current_object_status.items() if status == 'ON' and not obj_name in obj_counter]
+            for obj_name in expired_objects:
+                current_object_status[obj_name] = 'OFF'
+                self.client.publish(self.topic_prefix+'/'+obj_name, 'OFF', retain=False)
+                # send updated snapshot snapshot over mqtt if we have it as well
+                if obj_name in self.camera.best_frames.best_frames:
+                    best_frame = cv2.cvtColor(self.camera.best_frames.best_frames[obj_name], cv2.COLOR_RGB2BGR)
+                    ret, jpg = cv2.imencode('.jpg', best_frame)
+                    if ret:
+                        jpg_bytes = jpg.tobytes()
+                        self.client.publish(self.topic_prefix+'/'+obj_name+'/snapshot', jpg_bytes, retain=True)

frigate/object_detection.py

@@ -1,114 +1,139 @@
 import datetime
+import time
 import cv2
+import threading
+import copy
+import prctl
 import numpy as np
-import tensorflow as tf
-from object_detection.utils import label_map_util
-from object_detection.utils import visualization_utils as vis_util
-from . util import tonumpyarray
+from edgetpu.detection.engine import DetectionEngine
 
-# TODO: make dynamic?
-NUM_CLASSES = 90
-# Path to frozen detection graph. This is the actual model that is used for the object detection.
-PATH_TO_CKPT = '/frozen_inference_graph.pb'
-# List of the strings that is used to add correct label for each box.
-PATH_TO_LABELS = '/label_map.pbtext'
+from frigate.util import tonumpyarray, LABELS, PATH_TO_CKPT, calculate_region
 
-# Loading label map
-label_map = label_map_util.load_labelmap(PATH_TO_LABELS)
-categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES,
-                                                            use_display_name=True)
-category_index = label_map_util.create_category_index(categories)
+class PreppedQueueProcessor(threading.Thread):
+    def __init__(self, cameras, prepped_frame_queue, fps):
 
-# do the actual object detection
-def tf_detect_objects(cropped_frame, sess, detection_graph, region_size, region_x_offset, region_y_offset, debug):
-    # Expand dimensions since the model expects images to have shape: [1, None, None, 3]
-    image_np_expanded = np.expand_dims(cropped_frame, axis=0)
-    image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
-
-    # Each box represents a part of the image where a particular object was detected.
-    boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
-
-    # Each score represent how level of confidence for each of the objects.
-    # Score is shown on the result image, together with the class label.
-    scores = detection_graph.get_tensor_by_name('detection_scores:0')
-    classes = detection_graph.get_tensor_by_name('detection_classes:0')
-    num_detections = detection_graph.get_tensor_by_name('num_detections:0')
-
-    # Actual detection.
-    (boxes, scores, classes, num_detections) = sess.run(
-        [boxes, scores, classes, num_detections],
-        feed_dict={image_tensor: image_np_expanded})
-
-    if debug:
-        if len([value for index,value in enumerate(classes[0]) if str(category_index.get(value).get('name')) == 'person' and scores[0,index] > 0.5]) > 0:
-            vis_util.visualize_boxes_and_labels_on_image_array(
-                cropped_frame,
-                np.squeeze(boxes),
-                np.squeeze(classes).astype(np.int32),
-                np.squeeze(scores),
-                category_index,
-                use_normalized_coordinates=True,
-                line_thickness=4)
-            cv2.imwrite("/lab/debug/obj-{}-{}-{}.jpg".format(region_x_offset, region_y_offset, datetime.datetime.now().timestamp()), cropped_frame)
-
-
-    # build an array of detected objects
-    objects = []
-    for index, value in enumerate(classes[0]):
-        score = scores[0, index]
-        if score > 0.5:
-            box = boxes[0, index].tolist()
-            objects.append({
-                        'name': str(category_index.get(value).get('name')),
-                        'score': float(score),
-                        'ymin': int((box[0] * region_size) + region_y_offset),
-                        'xmin': int((box[1] * region_size) + region_x_offset),
-                        'ymax': int((box[2] * region_size) + region_y_offset),
-                        'xmax': int((box[3] * region_size) + region_x_offset)
-                    })
-
-    return objects
-
-def detect_objects(shared_arr, object_queue, shared_frame_time, frame_lock, frame_ready, 
-                   motion_detected, frame_shape, region_size, region_x_offset, region_y_offset,
-                   min_person_area, debug):
-    # shape shared input array into frame for processing
-    arr = tonumpyarray(shared_arr).reshape(frame_shape)
-
-    # Load a (frozen) Tensorflow model into memory before the processing loop
-    detection_graph = tf.Graph()
-    with detection_graph.as_default():
-        od_graph_def = tf.GraphDef()
-        with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid:
-            serialized_graph = fid.read()
-            od_graph_def.ParseFromString(serialized_graph)
-            tf.import_graph_def(od_graph_def, name='')
-        sess = tf.Session(graph=detection_graph)
-
-    frame_time = 0.0
-    while True:
-        now = datetime.datetime.now().timestamp()
-
-        # wait until motion is detected
-        motion_detected.wait()
-
-        with frame_ready:
-            # if there isnt a frame ready for processing or it is old, wait for a new frame
-            if shared_frame_time.value == frame_time or (now - shared_frame_time.value) > 0.5:
-                frame_ready.wait()
+        threading.Thread.__init__(self)
+        self.cameras = cameras
+        self.prepped_frame_queue = prepped_frame_queue
         
-        # make a copy of the cropped frame
-        with frame_lock:
-            cropped_frame = arr[region_y_offset:region_y_offset+region_size, region_x_offset:region_x_offset+region_size].copy()
-            frame_time = shared_frame_time.value
+        # Load the edgetpu engine and labels
+        self.engine = DetectionEngine(PATH_TO_CKPT)
+        self.labels = LABELS
+        self.fps = fps
+        self.avg_inference_speed = 10
 
-        # convert to RGB
-        cropped_frame_rgb = cv2.cvtColor(cropped_frame, cv2.COLOR_BGR2RGB)
-        # do the object detection
-        objects = tf_detect_objects(cropped_frame_rgb, sess, detection_graph, region_size, region_x_offset, region_y_offset, debug)
-        for obj in objects:
-            # ignore persons below the size threshold
-            if obj['name'] == 'person' and (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin']) < min_person_area:
+    def run(self):
+        prctl.set_name(self.__class__.__name__)
+        # process queue...
+        while True:
+            frame = self.prepped_frame_queue.get()
+
+            # Actual detection.
+            frame['detected_objects'] = self.engine.detect_with_input_tensor(frame['frame'], threshold=0.2, top_k=5)
+            self.fps.update()
+            self.avg_inference_speed = (self.avg_inference_speed*9 + self.engine.get_inference_time())/10
+
+            self.cameras[frame['camera_name']].detected_objects_queue.put(frame)
+
+class RegionRequester(threading.Thread):
+    def __init__(self, camera):
+        threading.Thread.__init__(self)
+        self.camera = camera
+
+    def run(self):
+        prctl.set_name(self.__class__.__name__)
+        frame_time = 0.0
+        while True:
+            now = datetime.datetime.now().timestamp()
+
+            with self.camera.frame_ready:
+                # if there isnt a frame ready for processing or it is old, wait for a new frame
+                if self.camera.frame_time.value == frame_time or (now - self.camera.frame_time.value) > 0.5:
+                    self.camera.frame_ready.wait()
+            
+            # make a copy of the frame_time
+            frame_time = self.camera.frame_time.value
+
+            # grab the current tracked objects
+            with self.camera.object_tracker.tracked_objects_lock:
+                tracked_objects = copy.deepcopy(self.camera.object_tracker.tracked_objects).values()
+
+            with self.camera.regions_in_process_lock:
+                self.camera.regions_in_process[frame_time] = len(self.camera.config['regions'])
+                self.camera.regions_in_process[frame_time] += len(tracked_objects)
+
+            for index, region in enumerate(self.camera.config['regions']):
+                self.camera.resize_queue.put({
+                    'camera_name': self.camera.name,
+                    'frame_time': frame_time,
+                    'region_id': index,
+                    'size': region['size'],
+                    'x_offset': region['x_offset'],
+                    'y_offset': region['y_offset']
+                })
+            
+            # request a region for tracked objects
+            for tracked_object in tracked_objects:
+                box = tracked_object['box']
+                # calculate a new region that will hopefully get the entire object
+                (size, x_offset, y_offset) = calculate_region(self.camera.frame_shape, 
+                    box['xmin'], box['ymin'],
+                    box['xmax'], box['ymax'])
+
+                self.camera.resize_queue.put({
+                    'camera_name': self.camera.name,
+                    'frame_time': frame_time,
+                    'region_id': -1,
+                    'size': size,
+                    'x_offset': x_offset,
+                    'y_offset': y_offset
+                })
+
+
+class RegionPrepper(threading.Thread):
+    def __init__(self, camera, frame_cache, resize_request_queue, prepped_frame_queue):
+        threading.Thread.__init__(self)
+        self.camera = camera
+        self.frame_cache = frame_cache
+        self.resize_request_queue = resize_request_queue
+        self.prepped_frame_queue = prepped_frame_queue
+
+    def run(self):
+        prctl.set_name(self.__class__.__name__)
+        while True:
+
+            resize_request = self.resize_request_queue.get()
+
+            # if the queue is over 100 items long, only prep dynamic regions
+            if resize_request['region_id'] != -1 and self.prepped_frame_queue.qsize() > 100:
+                with self.camera.regions_in_process_lock:
+                    self.camera.regions_in_process[resize_request['frame_time']] -= 1
+                    if self.camera.regions_in_process[resize_request['frame_time']] == 0:
+                        del self.camera.regions_in_process[resize_request['frame_time']]
+                self.camera.skipped_region_tracker.update()
                 continue
-            obj['frame_time'] = frame_time
-            object_queue.put(obj)
+
+            frame = self.frame_cache.get(resize_request['frame_time'], None)
+            
+            if frame is None:
+                print("RegionPrepper: frame_time not in frame_cache")
+                with self.camera.regions_in_process_lock:
+                    self.camera.regions_in_process[resize_request['frame_time']] -= 1
+                    if self.camera.regions_in_process[resize_request['frame_time']] == 0:
+                        del self.camera.regions_in_process[resize_request['frame_time']]
+                self.camera.skipped_region_tracker.update()
+                continue
+
+            # make a copy of the region
+            cropped_frame = frame[resize_request['y_offset']:resize_request['y_offset']+resize_request['size'], resize_request['x_offset']:resize_request['x_offset']+resize_request['size']].copy()
+
+            # Resize to 300x300 if needed
+            if cropped_frame.shape != (300, 300, 3):
+                # TODO: use Pillow-SIMD?
+                cropped_frame = cv2.resize(cropped_frame, dsize=(300, 300), interpolation=cv2.INTER_LINEAR)
+            # Expand dimensions since the model expects images to have shape: [1, 300, 300, 3]
+            frame_expanded = np.expand_dims(cropped_frame, axis=0)
+
+            # add the frame to the queue
+            resize_request['frame'] = frame_expanded.flatten().copy()
+            self.prepped_frame_queue.put(resize_request)

frigate/objects.py

@@ -2,122 +2,404 @@ import time
 import datetime
 import threading
 import cv2
-from object_detection.utils import visualization_utils as vis_util
-class ObjectParser(threading.Thread):
-    def __init__(self, object_queue, objects_parsed, detected_objects):
-        threading.Thread.__init__(self)
-        self._object_queue = object_queue
-        self._objects_parsed = objects_parsed
-        self._detected_objects = detected_objects
-
-    def run(self):
-        while True:
-            obj = self._object_queue.get()
-            self._detected_objects.append(obj)
-
-            # notify that objects were parsed
-            with self._objects_parsed:
-                self._objects_parsed.notify_all()
+import prctl
+import itertools
+import copy
+import numpy as np
+import multiprocessing as mp
+from collections import defaultdict
+from scipy.spatial import distance as dist
+from frigate.util import draw_box_with_label, LABELS, compute_intersection_rectangle, compute_intersection_over_union, calculate_region
 
 class ObjectCleaner(threading.Thread):
-    def __init__(self, objects_parsed, detected_objects):
+    def __init__(self, camera):
         threading.Thread.__init__(self)
-        self._objects_parsed = objects_parsed
-        self._detected_objects = detected_objects
+        self.camera = camera
 
     def run(self):
+        prctl.set_name("ObjectCleaner")
         while True:
 
-            # expire the objects that are more than 1 second old
-            now = datetime.datetime.now().timestamp()
-            # look for the first object found within the last second
-            # (newest objects are appended to the end)
-            detected_objects = self._detected_objects.copy()
-            num_to_delete = 0
-            for obj in detected_objects:
-                if now-obj['frame_time']<1:
-                    break
-                num_to_delete += 1
-            if num_to_delete > 0:
-                del self._detected_objects[:num_to_delete]
-
-                # notify that parsed objects were changed
-                with self._objects_parsed:
-                    self._objects_parsed.notify_all()
-            
-            # wait a bit before checking for more expired frames
+            # wait a bit before checking for expired frames
             time.sleep(0.2)
 
-# Maintains the frame and person with the highest score from the most recent
-# motion event
-class BestPersonFrame(threading.Thread):
-    def __init__(self, objects_parsed, recent_frames, detected_objects, motion_changed, motion_regions):
+            for frame_time in list(self.camera.detected_objects.keys()).copy():
+                if not frame_time in self.camera.frame_cache:
+                    del self.camera.detected_objects[frame_time]
+            
+            objects_deregistered = False
+            with self.camera.object_tracker.tracked_objects_lock:
+                now = datetime.datetime.now().timestamp()
+                for id, obj in list(self.camera.object_tracker.tracked_objects.items()):
+                    # if the object is more than 10 seconds old
+                    # and not in the most recent frame, deregister
+                    if (now - obj['frame_time']) > 10 and self.camera.object_tracker.most_recent_frame_time > obj['frame_time']:
+                        self.camera.object_tracker.deregister(id)
+                        objects_deregistered = True
+            
+            if objects_deregistered:
+                with self.camera.objects_tracked:
+                    self.camera.objects_tracked.notify_all()
+
+class DetectedObjectsProcessor(threading.Thread):
+    def __init__(self, camera):
         threading.Thread.__init__(self)
-        self.objects_parsed = objects_parsed
-        self.recent_frames = recent_frames
-        self.detected_objects = detected_objects
-        self.motion_changed = motion_changed
-        self.motion_regions = motion_regions
-        self.best_person = None
-        self.best_frame = None
+        self.camera = camera
 
     def run(self):
-        motion_start = 0.0
-        motion_end = 0.0
-
+        prctl.set_name(self.__class__.__name__)
         while True:
+            frame = self.camera.detected_objects_queue.get()
 
-             # while there is motion
-            while len([r for r in self.motion_regions if r.is_set()]) > 0:
-                # wait until objects have been parsed
-                with self.objects_parsed:
-                    self.objects_parsed.wait()
+            objects = frame['detected_objects']
 
-                # make a copy of detected objects
-                detected_objects = self.detected_objects.copy()
-                detected_people = [obj for obj in detected_objects if obj['name'] == 'person']
-                # make a copy of the recent frames
-                recent_frames = self.recent_frames.copy()
+            for raw_obj in objects:
+                name = str(LABELS[raw_obj.label_id])
 
-                # get the highest scoring person
-                new_best_person = max(detected_people, key=lambda x:x['score'], default=self.best_person)
-
-                # if there isnt a person, continue
-                if new_best_person is None:
+                if not name in self.camera.objects_to_track:
                     continue
 
-                # if there is no current best_person
-                if self.best_person is None:
-                    self.best_person = new_best_person
-                # if there is already a best_person
-                else:
-                    now = datetime.datetime.now().timestamp()
-                    # if the new best person is a higher score than the current best person 
-                    # or the current person is more than 1 minute old, use the new best person
-                    if new_best_person['score'] > self.best_person['score'] or (now - self.best_person['frame_time']) > 60:
-                        self.best_person = new_best_person
+                obj = {
+                    'name': name,
+                    'score': float(raw_obj.score),
+                    'box': {
+                        'xmin': int((raw_obj.bounding_box[0][0] * frame['size']) + frame['x_offset']),
+                        'ymin': int((raw_obj.bounding_box[0][1] * frame['size']) + frame['y_offset']),
+                        'xmax': int((raw_obj.bounding_box[1][0] * frame['size']) + frame['x_offset']),
+                        'ymax': int((raw_obj.bounding_box[1][1] * frame['size']) + frame['y_offset'])
+                    },
+                    'region': {
+                        'xmin': frame['x_offset'],
+                        'ymin': frame['y_offset'],
+                        'xmax': frame['x_offset']+frame['size'],
+                        'ymax': frame['y_offset']+frame['size']
+                    },
+                    'frame_time': frame['frame_time'],
+                    'region_id': frame['region_id']
+                }
+                
+                # if the object is within 5 pixels of the region border, and the region is not on the edge
+                # consider the object to be clipped
+                obj['clipped'] = False
+                if ((obj['region']['xmin'] > 5 and obj['box']['xmin']-obj['region']['xmin'] <= 5) or 
+                    (obj['region']['ymin'] > 5 and obj['box']['ymin']-obj['region']['ymin'] <= 5) or
+                    (self.camera.frame_shape[1]-obj['region']['xmax'] > 5 and obj['region']['xmax']-obj['box']['xmax'] <= 5) or
+                    (self.camera.frame_shape[0]-obj['region']['ymax'] > 5 and obj['region']['ymax']-obj['box']['ymax'] <= 5)):
+                    obj['clipped'] = True
+                
+                # Compute the area
+                obj['area'] = (obj['box']['xmax']-obj['box']['xmin'])*(obj['box']['ymax']-obj['box']['ymin'])
 
-                if not self.best_person is None and self.best_person['frame_time'] in recent_frames:
-                    best_frame = recent_frames[self.best_person['frame_time']]
-                    best_frame = cv2.cvtColor(best_frame, cv2.COLOR_BGR2RGB)
-                    # draw the bounding box on the frame
-                    vis_util.draw_bounding_box_on_image_array(best_frame,
-                        self.best_person['ymin'],
-                        self.best_person['xmin'],
-                        self.best_person['ymax'],
-                        self.best_person['xmax'],
-                        color='red',
-                        thickness=2,
-                        display_str_list=["{}: {}%".format(self.best_person['name'],int(self.best_person['score']*100))],
-                        use_normalized_coordinates=False)
-
-                    # convert back to BGR
-                    self.best_frame = cv2.cvtColor(best_frame, cv2.COLOR_RGB2BGR)
-
-            motion_end = datetime.datetime.now().timestamp()
-
-            # wait for the global motion flag to change
-            with self.motion_changed:
-                self.motion_changed.wait()
+                self.camera.detected_objects[frame['frame_time']].append(obj)
             
-            motion_start = datetime.datetime.now().timestamp()
+            with self.camera.regions_in_process_lock:
+                self.camera.regions_in_process[frame['frame_time']] -= 1
+                # print(f"{frame['frame_time']} remaining regions {self.camera.regions_in_process[frame['frame_time']]}")
+
+                if self.camera.regions_in_process[frame['frame_time']] == 0:
+                    del self.camera.regions_in_process[frame['frame_time']]
+                    # print(f"{frame['frame_time']} no remaining regions")
+                    self.camera.finished_frame_queue.put(frame['frame_time'])
+
+# Thread that checks finished frames for clipped objects and sends back
+# for processing if needed
+class RegionRefiner(threading.Thread):
+    def __init__(self, camera):
+        threading.Thread.__init__(self)
+        self.camera = camera
+
+    def run(self):
+        prctl.set_name(self.__class__.__name__)
+        while True:
+            frame_time = self.camera.finished_frame_queue.get()
+
+            detected_objects = self.camera.detected_objects[frame_time].copy()
+            # print(f"{frame_time} finished")
+
+            # group by name
+            detected_object_groups = defaultdict(lambda: [])
+            for obj in detected_objects:
+                detected_object_groups[obj['name']].append(obj)
+
+            look_again = False
+            selected_objects = []
+            for group in detected_object_groups.values():
+
+                # apply non-maxima suppression to suppress weak, overlapping bounding boxes
+                boxes = [(o['box']['xmin'], o['box']['ymin'], o['box']['xmax']-o['box']['xmin'], o['box']['ymax']-o['box']['ymin'])
+                    for o in group]
+                confidences = [o['score'] for o in group]
+                idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
+
+                for index in idxs:
+                    obj = group[index[0]]
+                    selected_objects.append(obj)
+                    if obj['clipped']:
+                        box = obj['box']
+                        # calculate a new region that will hopefully get the entire object
+                        (size, x_offset, y_offset) = calculate_region(self.camera.frame_shape, 
+                            box['xmin'], box['ymin'],
+                            box['xmax'], box['ymax'])
+                        # print(f"{frame_time} new region: {size} {x_offset} {y_offset}")
+
+                        with self.camera.regions_in_process_lock:
+                            if not frame_time in self.camera.regions_in_process:
+                                self.camera.regions_in_process[frame_time] = 1
+                            else:
+                                self.camera.regions_in_process[frame_time] += 1
+
+                        # add it to the queue
+                        self.camera.resize_queue.put({
+                            'camera_name': self.camera.name,
+                            'frame_time': frame_time,
+                            'region_id': -1,
+                            'size': size,
+                            'x_offset': x_offset,
+                            'y_offset': y_offset
+                        })
+                        self.camera.dynamic_region_fps.update()
+                        look_again = True
+
+            # if we are looking again, then this frame is not ready for processing
+            if look_again:
+                # remove the clipped objects
+                self.camera.detected_objects[frame_time] = [o for o in selected_objects if not o['clipped']]
+                continue
+
+            # filter objects based on camera settings
+            selected_objects = [o for o in selected_objects if not self.filtered(o)]
+
+            self.camera.detected_objects[frame_time] = selected_objects
+            
+            # print(f"{frame_time} is actually finished")
+
+            # keep adding frames to the refined queue as long as they are finished
+            with self.camera.regions_in_process_lock:
+                while self.camera.frame_queue.qsize() > 0 and self.camera.frame_queue.queue[0] not in self.camera.regions_in_process:
+                    self.camera.last_processed_frame = self.camera.frame_queue.get()
+                    self.camera.refined_frame_queue.put(self.camera.last_processed_frame)
+
+    def filtered(self, obj):
+        object_name = obj['name']
+        
+        if object_name in self.camera.object_filters:
+            obj_settings = self.camera.object_filters[object_name]
+
+            # if the min area is larger than the
+            # detected object, don't add it to detected objects
+            if obj_settings.get('min_area',-1) > obj['area']:
+                return True
+            
+            # if the detected object is larger than the
+            # max area, don't add it to detected objects
+            if obj_settings.get('max_area', self.camera.frame_shape[0]*self.camera.frame_shape[1]) < obj['area']:
+                return True
+
+            # if the score is lower than the threshold, skip
+            if obj_settings.get('threshold', 0) > obj['score']:
+                return True
+        
+            # compute the coordinates of the object and make sure
+            # the location isnt outside the bounds of the image (can happen from rounding)
+            y_location = min(int(obj['box']['ymax']), len(self.camera.mask)-1)
+            x_location = min(int((obj['box']['xmax']-obj['box']['xmin'])/2.0)+obj['box']['xmin'], len(self.camera.mask[0])-1)
+
+            # if the object is in a masked location, don't add it to detected objects
+            if self.camera.mask[y_location][x_location] == [0]:
+                return True
+            
+            return False
+             
+    def has_overlap(self, new_obj, obj, overlap=.7):
+        # compute intersection rectangle with existing object and new objects region
+        existing_obj_current_region = compute_intersection_rectangle(obj['box'], new_obj['region'])
+
+        # compute intersection rectangle with new object and existing objects region
+        new_obj_existing_region = compute_intersection_rectangle(new_obj['box'], obj['region'])
+
+        # compute iou for the two intersection rectangles that were just computed
+        iou = compute_intersection_over_union(existing_obj_current_region, new_obj_existing_region)
+
+        # if intersection is greater than overlap
+        if iou > overlap:
+            return True
+        else:
+            return False
+    
+    def find_group(self, new_obj, groups):
+        for index, group in enumerate(groups):
+            for obj in group:
+                if self.has_overlap(new_obj, obj):
+                    return index
+        return None
+
+class ObjectTracker(threading.Thread):
+    def __init__(self, camera, max_disappeared):
+        threading.Thread.__init__(self)
+        self.camera = camera
+        self.tracked_objects = {}
+        self.tracked_objects_lock = mp.Lock()
+        self.most_recent_frame_time = None
+    
+    def run(self):
+        prctl.set_name(self.__class__.__name__)
+        while True:
+            frame_time = self.camera.refined_frame_queue.get()
+            with self.tracked_objects_lock:
+                self.match_and_update(self.camera.detected_objects[frame_time])
+                self.most_recent_frame_time = frame_time
+                self.camera.frame_output_queue.put((frame_time, copy.deepcopy(self.tracked_objects)))
+            if len(self.tracked_objects) > 0:
+                with self.camera.objects_tracked:
+                    self.camera.objects_tracked.notify_all()
+
+    def register(self, index, obj):
+        id = "{}-{}".format(str(obj['frame_time']), index)
+        obj['id'] = id
+        obj['top_score'] = obj['score']
+        self.add_history(obj)
+        self.tracked_objects[id] = obj
+
+    def deregister(self, id):
+        del self.tracked_objects[id]
+    
+    def update(self, id, new_obj):
+        self.tracked_objects[id].update(new_obj)
+        self.add_history(self.tracked_objects[id])
+        if self.tracked_objects[id]['score'] > self.tracked_objects[id]['top_score']:
+            self.tracked_objects[id]['top_score'] = self.tracked_objects[id]['score']
+    
+    def add_history(self, obj):
+        entry = {
+            'score': obj['score'],
+            'box': obj['box'],
+            'region': obj['region'],
+            'centroid': obj['centroid'],
+            'frame_time': obj['frame_time']
+        }
+        if 'history' in obj:
+            obj['history'].append(entry)
+        else:
+            obj['history'] = [entry]
+
+    def match_and_update(self, new_objects):
+        if len(new_objects) == 0:
+            return
+            
+        # group by name
+        new_object_groups = defaultdict(lambda: [])
+        for obj in new_objects:
+            new_object_groups[obj['name']].append(obj)
+        
+        # track objects for each label type
+        for label, group in new_object_groups.items():
+            current_objects = [o for o in self.tracked_objects.values() if o['name'] == label]
+            current_ids = [o['id'] for o in current_objects]
+            current_centroids = np.array([o['centroid'] for o in current_objects])
+
+            # compute centroids of new objects
+            for obj in group:
+                centroid_x = int((obj['box']['xmin']+obj['box']['xmax']) / 2.0)
+                centroid_y = int((obj['box']['ymin']+obj['box']['ymax']) / 2.0)
+                obj['centroid'] = (centroid_x, centroid_y)
+
+            if len(current_objects) == 0:
+                for index, obj in enumerate(group):
+                    self.register(index, obj)
+                return
+            
+            new_centroids = np.array([o['centroid'] for o in group])
+
+            # compute the distance between each pair of tracked
+            # centroids and new centroids, respectively -- our
+            # goal will be to match each new centroid to an existing
+            # object centroid
+            D = dist.cdist(current_centroids, new_centroids)
+
+            # in order to perform this matching we must (1) find the
+            # smallest value in each row and then (2) sort the row
+            # indexes based on their minimum values so that the row
+            # with the smallest value is at the *front* of the index
+            # list
+            rows = D.min(axis=1).argsort()
+
+            # next, we perform a similar process on the columns by
+            # finding the smallest value in each column and then
+            # sorting using the previously computed row index list
+            cols = D.argmin(axis=1)[rows]
+
+            # in order to determine if we need to update, register,
+            # or deregister an object we need to keep track of which
+            # of the rows and column indexes we have already examined
+            usedRows = set()
+            usedCols = set()
+
+            # loop over the combination of the (row, column) index
+            # tuples
+            for (row, col) in zip(rows, cols):
+                # if we have already examined either the row or
+                # column value before, ignore it
+                if row in usedRows or col in usedCols:
+                    continue
+
+                # otherwise, grab the object ID for the current row,
+                # set its new centroid, and reset the disappeared
+                # counter
+                objectID = current_ids[row]
+                self.update(objectID, group[col])
+
+                # indicate that we have examined each of the row and
+                # column indexes, respectively
+                usedRows.add(row)
+                usedCols.add(col)
+
+            # compute the column index we have NOT yet examined
+            unusedCols = set(range(0, D.shape[1])).difference(usedCols)
+
+            # if the number of input centroids is greater
+            # than the number of existing object centroids we need to
+            # register each new input centroid as a trackable object
+            # if D.shape[0] < D.shape[1]:
+            for col in unusedCols:
+                self.register(col, group[col])
+
+# Maintains the frame and object with the highest score
+class BestFrames(threading.Thread):
+    def __init__(self, camera):
+        threading.Thread.__init__(self)
+        self.camera = camera
+        self.best_objects = {}
+        self.best_frames = {}
+
+    def run(self):
+        prctl.set_name(self.__class__.__name__)
+        while True:
+            # wait until objects have been tracked
+            with self.camera.objects_tracked:
+                self.camera.objects_tracked.wait()
+
+            # make a copy of tracked objects
+            tracked_objects = list(self.camera.object_tracker.tracked_objects.values())
+
+            for obj in tracked_objects:
+                if obj['name'] in self.best_objects:
+                    now = datetime.datetime.now().timestamp()
+                    # if the object is a higher score than the current best score 
+                    # or the current object is more than 1 minute old, use the new object
+                    if obj['score'] > self.best_objects[obj['name']]['score'] or (now - self.best_objects[obj['name']]['frame_time']) > 60:
+                        self.best_objects[obj['name']] = copy.deepcopy(obj)
+                else:
+                    self.best_objects[obj['name']] = copy.deepcopy(obj)
+            
+            for name, obj in self.best_objects.items():
+                if obj['frame_time'] in self.camera.frame_cache:
+                    best_frame = self.camera.frame_cache[obj['frame_time']]
+
+                    draw_box_with_label(best_frame, obj['box']['xmin'], obj['box']['ymin'], 
+                        obj['box']['xmax'], obj['box']['ymax'], obj['name'], "{}% {}".format(int(obj['score']*100), obj['area']))
+                    
+                    # print a timestamp
+                    time_to_show = datetime.datetime.fromtimestamp(obj['frame_time']).strftime("%m/%d/%Y %H:%M:%S")
+                    cv2.putText(best_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
+                    
+                    self.best_frames[name] = best_frame

frigate/util.py

@@ -1,5 +1,161 @@
+import datetime
+import collections
 import numpy as np
+import cv2
+import threading
+import matplotlib.pyplot as plt
+
+# Function to read labels from text files.
+def ReadLabelFile(file_path):
+    with open(file_path, 'r') as f:
+        lines = f.readlines()
+    ret = {}
+    for line in lines:
+        pair = line.strip().split(maxsplit=1)
+        ret[int(pair[0])] = pair[1].strip()
+    return ret
+
+def calculate_region(frame_shape, xmin, ymin, xmax, ymax):    
+    # size is larger than longest edge
+    size = int(max(xmax-xmin, ymax-ymin)*2)
+    # if the size is too big to fit in the frame
+    if size > min(frame_shape[0], frame_shape[1]):
+        size = min(frame_shape[0], frame_shape[1])
+
+    # x_offset is midpoint of bounding box minus half the size
+    x_offset = int((xmax-xmin)/2.0+xmin-size/2.0)
+    # if outside the image
+    if x_offset < 0:
+        x_offset = 0
+    elif x_offset > (frame_shape[1]-size):
+        x_offset = (frame_shape[1]-size)
+
+    # y_offset is midpoint of bounding box minus half the size
+    y_offset = int((ymax-ymin)/2.0+ymin-size/2.0)
+    # if outside the image
+    if y_offset < 0:
+        y_offset = 0
+    elif y_offset > (frame_shape[0]-size):
+        y_offset = (frame_shape[0]-size)
+
+    return (size, x_offset, y_offset)
+
+def compute_intersection_rectangle(box_a, box_b):
+    return {
+        'xmin': max(box_a['xmin'], box_b['xmin']),
+        'ymin': max(box_a['ymin'], box_b['ymin']),
+        'xmax': min(box_a['xmax'], box_b['xmax']),
+        'ymax': min(box_a['ymax'], box_b['ymax'])
+    }
+    
+def compute_intersection_over_union(box_a, box_b):
+    # determine the (x, y)-coordinates of the intersection rectangle
+    intersect = compute_intersection_rectangle(box_a, box_b)
+
+    # compute the area of intersection rectangle
+    inter_area = max(0, intersect['xmax'] - intersect['xmin'] + 1) * max(0, intersect['ymax'] - intersect['ymin'] + 1)
+
+    if inter_area == 0:
+        return 0.0
+    
+    # compute the area of both the prediction and ground-truth
+    # rectangles
+    box_a_area = (box_a['xmax'] - box_a['xmin'] + 1) * (box_a['ymax'] - box_a['ymin'] + 1)
+    box_b_area = (box_b['xmax'] - box_b['xmin'] + 1) * (box_b['ymax'] - box_b['ymin'] + 1)
+
+    # compute the intersection over union by taking the intersection
+    # area and dividing it by the sum of prediction + ground-truth
+    # areas - the interesection area
+    iou = inter_area / float(box_a_area + box_b_area - inter_area)
+
+    # return the intersection over union value
+    return iou
 
 # convert shared memory array into numpy array
 def tonumpyarray(mp_arr):
-    return np.frombuffer(mp_arr.get_obj(), dtype=np.uint16)
+    return np.frombuffer(mp_arr.get_obj(), dtype=np.uint8)
+
+def draw_box_with_label(frame, x_min, y_min, x_max, y_max, label, info, thickness=2, color=None, position='ul'):
+    if color is None:
+        color = COLOR_MAP[label]
+    display_text = "{}: {}".format(label, info)
+    cv2.rectangle(frame, (x_min, y_min), 
+        (x_max, y_max), 
+        color, thickness)
+    font_scale = 0.5
+    font = cv2.FONT_HERSHEY_SIMPLEX
+    # get the width and height of the text box
+    size = cv2.getTextSize(display_text, font, fontScale=font_scale, thickness=2)
+    text_width = size[0][0]
+    text_height = size[0][1]
+    line_height = text_height + size[1]
+    # set the text start position
+    if position == 'ul':
+        text_offset_x = x_min
+        text_offset_y = 0 if y_min < line_height else y_min - (line_height+8)
+    elif position == 'ur':
+        text_offset_x = x_max - (text_width+8)
+        text_offset_y = 0 if y_min < line_height else y_min - (line_height+8)
+    elif position == 'bl':
+        text_offset_x = x_min
+        text_offset_y = y_max
+    elif position == 'br':
+        text_offset_x = x_max - (text_width+8)
+        text_offset_y = y_max
+    # make the coords of the box with a small padding of two pixels
+    textbox_coords = ((text_offset_x, text_offset_y), (text_offset_x + text_width + 2, text_offset_y + line_height))
+    cv2.rectangle(frame, textbox_coords[0], textbox_coords[1], color, cv2.FILLED)
+    cv2.putText(frame, display_text, (text_offset_x, text_offset_y + line_height - 3), font, fontScale=font_scale, color=(0, 0, 0), thickness=2)
+
+# Path to frozen detection graph. This is the actual model that is used for the object detection.
+PATH_TO_CKPT = '/frozen_inference_graph.pb'
+# List of the strings that is used to add correct label for each box.
+PATH_TO_LABELS = '/label_map.pbtext'
+
+LABELS = ReadLabelFile(PATH_TO_LABELS)
+cmap = plt.cm.get_cmap('tab10', len(LABELS.keys()))
+
+COLOR_MAP = {}
+for key, val in LABELS.items():
+    COLOR_MAP[val] = tuple(int(round(255 * c)) for c in cmap(key)[:3])
+
+class QueueMerger():
+    def __init__(self, from_queues, to_queue):
+        self.from_queues = from_queues
+        self.to_queue = to_queue
+        self.merge_threads = []
+
+    def start(self):
+        for from_q in self.from_queues:
+            self.merge_threads.append(QueueTransfer(from_q,self.to_queue))
+
+class QueueTransfer(threading.Thread):
+    def __init__(self, from_queue, to_queue):
+        threading.Thread.__init__(self)
+        self.from_queue = from_queue
+        self.to_queue = to_queue
+
+    def run(self):
+        while True:
+            self.to_queue.put(self.from_queue.get())
+
+class EventsPerSecond:
+    def __init__(self, max_events=1000):
+        self._start = None
+        self._max_events = max_events
+        self._timestamps = []
+    
+    def start(self):
+        self._start = datetime.datetime.now().timestamp()
+
+    def update(self):
+        self._timestamps.append(datetime.datetime.now().timestamp())
+        # truncate the list when it goes 100 over the max_size
+        if len(self._timestamps) > self._max_events+100:
+            self._timestamps = self._timestamps[(1-self._max_events):]
+
+    def eps(self, last_n_seconds=10):
+		# compute the (approximate) events in the last n seconds
+        now = datetime.datetime.now().timestamp()
+        seconds = min(now-self._start, last_n_seconds)
+        return len([t for t in self._timestamps if t > (now-last_n_seconds)]) / seconds

frigate/video.py

@@ -1,95 +1,376 @@
+import os
 import time
 import datetime
 import cv2
+import queue
 import threading
-from . util import tonumpyarray
+import ctypes
+import multiprocessing as mp
+import subprocess as sp
+import numpy as np
+import prctl
+import copy
+import itertools
+from collections import defaultdict
+from frigate.util import tonumpyarray, LABELS, draw_box_with_label, calculate_region, EventsPerSecond
+from frigate.object_detection import RegionPrepper, RegionRequester
+from frigate.objects import ObjectCleaner, BestFrames, DetectedObjectsProcessor, RegionRefiner, ObjectTracker
+from frigate.mqtt import MqttObjectPublisher
 
-# fetch the frames as fast a possible, only decoding the frames when the
-# detection_process has consumed the current frame
-def fetch_frames(shared_arr, shared_frame_time, frame_lock, frame_ready, frame_shape, rtsp_url):
-    # convert shared memory array into numpy and shape into image array
-    arr = tonumpyarray(shared_arr).reshape(frame_shape)
-
-    # start the video capture
-    video = cv2.VideoCapture()
-    video.open(rtsp_url)
-    # keep the buffer small so we minimize old data
-    video.set(cv2.CAP_PROP_BUFFERSIZE,1)
-
-    bad_frame_counter = 0
-    while True:
-        # check if the video stream is still open, and reopen if needed
-        if not video.isOpened():
-            success = video.open(rtsp_url)
-            if not success:
-                time.sleep(1)
-                continue
-        # grab the frame, but dont decode it yet
-        ret = video.grab()
-        # snapshot the time the frame was grabbed
-        frame_time = datetime.datetime.now()
-        if ret:
-            # go ahead and decode the current frame
-            ret, frame = video.retrieve()
-            if ret:
-                # Lock access and update frame
-                with frame_lock:
-                    arr[:] = frame
-                    shared_frame_time.value = frame_time.timestamp()
-                # Notify with the condition that a new frame is ready
-                with frame_ready:
-                    frame_ready.notify_all()
-                bad_frame_counter = 0
-            else:
-                print("Unable to decode frame")
-                bad_frame_counter += 1
-        else:
-            print("Unable to grab a frame")
-            bad_frame_counter += 1
-        
-        if bad_frame_counter > 100:
-            video.release()
-    
-    video.release()
-
-# Stores 2 seconds worth of frames when motion is detected so they can be used for other threads
+# Stores 2 seconds worth of frames so they can be used for other threads
 class FrameTracker(threading.Thread):
-    def __init__(self, shared_frame, frame_time, frame_ready, frame_lock, recent_frames, motion_changed, motion_regions):
+    def __init__(self, frame_time, frame_ready, frame_lock, recent_frames):
         threading.Thread.__init__(self)
-        self.shared_frame = shared_frame
         self.frame_time = frame_time
         self.frame_ready = frame_ready
         self.frame_lock = frame_lock
         self.recent_frames = recent_frames
-        self.motion_changed = motion_changed
-        self.motion_regions = motion_regions
+    
+    def run(self):
+        prctl.set_name(self.__class__.__name__)
+        while True:
+            # wait for a frame
+            with self.frame_ready:
+                self.frame_ready.wait()
+
+            # delete any old frames
+            stored_frame_times = list(self.recent_frames.keys())
+            stored_frame_times.sort(reverse=True)
+            if len(stored_frame_times) > 100:
+                frames_to_delete = stored_frame_times[50:]
+                for k in frames_to_delete:
+                    del self.recent_frames[k]
+
+def get_frame_shape(source):
+    # capture a single frame and check the frame shape so the correct array
+    # size can be allocated in memory
+    video = cv2.VideoCapture(source)
+    ret, frame = video.read()
+    frame_shape = frame.shape
+    video.release()
+    return frame_shape
+
+def get_ffmpeg_input(ffmpeg_input):
+    frigate_vars = {k: v for k, v in os.environ.items() if k.startswith('FRIGATE_')}
+    return ffmpeg_input.format(**frigate_vars)
+
+class CameraWatchdog(threading.Thread):
+    def __init__(self, camera):
+        threading.Thread.__init__(self)
+        self.camera = camera
 
     def run(self):
-        frame_time = 0.0
+        prctl.set_name(self.__class__.__name__)
         while True:
-            # while there is motion
-            while len([r for r in self.motion_regions if r.is_set()]) > 0:
-                now = datetime.datetime.now().timestamp()
-                # wait for a frame
-                with self.frame_ready:
-                    # if there isnt a frame ready for processing or it is old, wait for a signal
-                    if self.frame_time.value == frame_time or (now - self.frame_time.value) > 0.5:
-                        self.frame_ready.wait()
-                
-                # lock and make a copy of the frame
-                with self.frame_lock: 
-                    frame = self.shared_frame.copy().astype('uint8')
-                    frame_time = self.frame_time.value
-                
-                # add the frame to recent frames
-                self.recent_frames[frame_time] = frame
+            # wait a bit before checking
+            time.sleep(10)
 
-                # delete any old frames
-                stored_frame_times = list(self.recent_frames.keys())
-                for k in stored_frame_times:
-                    if (now - k) > 2:
-                        del self.recent_frames[k]
-                
-            # wait for the global motion flag to change
-            with self.motion_changed:
-                self.motion_changed.wait()
+            if self.camera.frame_time.value != 0.0 and (datetime.datetime.now().timestamp() - self.camera.frame_time.value) > 300:
+                print(self.camera.name + ": last frame is more than 5 minutes old, restarting camera capture...")
+                self.camera.start_or_restart_capture()
+                time.sleep(5)
+
+# Thread to read the stdout of the ffmpeg process and update the current frame
+class CameraCapture(threading.Thread):
+    def __init__(self, camera):
+        threading.Thread.__init__(self)
+        self.camera = camera
+
+    def run(self):
+        prctl.set_name(self.__class__.__name__)
+        frame_num = 0
+        while True:
+            if self.camera.ffmpeg_process.poll() != None:
+                print(self.camera.name + ": ffmpeg process is not running. exiting capture thread...")
+                break
+
+            raw_image = self.camera.ffmpeg_process.stdout.read(self.camera.frame_size)
+
+            if len(raw_image) == 0:
+                print(self.camera.name + ": ffmpeg didnt return a frame. something is wrong. exiting capture thread...")
+                break
+
+            frame_num += 1
+            if (frame_num % self.camera.take_frame) != 0:
+                continue
+
+            with self.camera.frame_lock:
+                # TODO: use frame_queue instead
+                self.camera.frame_time.value = datetime.datetime.now().timestamp()
+                self.camera.frame_cache[self.camera.frame_time.value] = (
+                    np
+                    .frombuffer(raw_image, np.uint8)
+                    .reshape(self.camera.frame_shape)
+                )
+                self.camera.frame_queue.put(self.camera.frame_time.value)
+            # Notify with the condition that a new frame is ready
+            with self.camera.frame_ready:
+                self.camera.frame_ready.notify_all()
+
+            self.camera.fps.update()
+
+class VideoWriter(threading.Thread):
+    def __init__(self, camera):
+        threading.Thread.__init__(self)
+        self.camera = camera
+
+    def run(self):
+        prctl.set_name(self.__class__.__name__)
+        while True:
+            (frame_time, tracked_objects) = self.camera.frame_output_queue.get()
+            # if len(tracked_objects) == 0:
+            #     continue
+            # f = open(f"/debug/output/{self.camera.name}-{str(format(frame_time, '.8f'))}.jpg", 'wb')
+            # f.write(self.camera.frame_with_objects(frame_time, tracked_objects))
+            # f.close()
+
+class Camera:
+    def __init__(self, name, ffmpeg_config, global_objects_config, config, prepped_frame_queue, mqtt_client, mqtt_prefix):
+        self.name = name
+        self.config = config
+        self.detected_objects = defaultdict(lambda: [])
+        self.frame_cache = {}
+        self.last_processed_frame = None
+        # queue for re-assembling frames in order
+        self.frame_queue = queue.Queue()
+        # track how many regions have been requested for a frame so we know when a frame is complete
+        self.regions_in_process = {}
+        # Lock to control access
+        self.regions_in_process_lock = mp.Lock()
+        self.finished_frame_queue = queue.Queue()
+        self.refined_frame_queue = queue.Queue()
+        self.frame_output_queue = queue.Queue()
+
+        self.ffmpeg = config.get('ffmpeg', {})
+        self.ffmpeg_input = get_ffmpeg_input(self.ffmpeg['input'])
+        self.ffmpeg_global_args = self.ffmpeg.get('global_args', ffmpeg_config['global_args'])
+        self.ffmpeg_hwaccel_args = self.ffmpeg.get('hwaccel_args', ffmpeg_config['hwaccel_args'])
+        self.ffmpeg_input_args = self.ffmpeg.get('input_args', ffmpeg_config['input_args'])
+        self.ffmpeg_output_args = self.ffmpeg.get('output_args', ffmpeg_config['output_args'])
+
+        camera_objects_config = config.get('objects', {})
+
+        self.take_frame = self.config.get('take_frame', 1)
+        self.regions = self.config['regions']
+        self.frame_shape = get_frame_shape(self.ffmpeg_input)
+        self.frame_size = self.frame_shape[0] * self.frame_shape[1] * self.frame_shape[2]
+        self.mqtt_client = mqtt_client
+        self.mqtt_topic_prefix = '{}/{}'.format(mqtt_prefix, self.name)
+
+        # create shared value for storing the frame_time
+        self.frame_time = mp.Value('d', 0.0)
+        # Lock to control access to the frame
+        self.frame_lock = mp.Lock()
+        # Condition for notifying that a new frame is ready
+        self.frame_ready = mp.Condition()
+        # Condition for notifying that objects were tracked
+        self.objects_tracked = mp.Condition()
+
+        # Queue for prepped frames, max size set to (number of regions * 5)
+        self.resize_queue = queue.Queue()
+
+        # Queue for raw detected objects
+        self.detected_objects_queue = queue.Queue()
+        self.detected_objects_processor = DetectedObjectsProcessor(self)
+        self.detected_objects_processor.start()
+
+        # initialize the frame cache
+        self.cached_frame_with_objects = {
+            'frame_bytes': [],
+            'frame_time': 0
+        }
+
+        self.ffmpeg_process = None
+        self.capture_thread = None
+        self.fps = EventsPerSecond()
+        self.skipped_region_tracker = EventsPerSecond()
+
+        # combine tracked objects lists
+        self.objects_to_track = set().union(global_objects_config.get('track', ['person', 'car', 'truck']), camera_objects_config.get('track', []))
+
+        # merge object filters
+        global_object_filters = global_objects_config.get('filters', {})
+        camera_object_filters = camera_objects_config.get('filters', {})
+        objects_with_config = set().union(global_object_filters.keys(), camera_object_filters.keys())
+        self.object_filters = {}
+        for obj in objects_with_config:
+            self.object_filters[obj] = {**global_object_filters.get(obj, {}), **camera_object_filters.get(obj, {})}
+
+        # start a thread to track objects
+        self.object_tracker = ObjectTracker(self, 10)
+        self.object_tracker.start()
+
+        # start a thread to write tracked frames to disk
+        self.video_writer = VideoWriter(self)
+        self.video_writer.start()
+
+        # start a thread to queue resize requests for regions
+        self.region_requester = RegionRequester(self)
+        self.region_requester.start()
+
+        # start a thread to cache recent frames for processing
+        self.frame_tracker = FrameTracker(self.frame_time, 
+            self.frame_ready, self.frame_lock, self.frame_cache)
+        self.frame_tracker.start()
+
+        # start a thread to resize regions
+        self.region_prepper = RegionPrepper(self, self.frame_cache, self.resize_queue, prepped_frame_queue)
+        self.region_prepper.start()
+
+        # start a thread to store the highest scoring recent frames for monitored object types
+        self.best_frames = BestFrames(self)
+        self.best_frames.start()
+
+        # start a thread to expire objects from the detected objects list
+        self.object_cleaner = ObjectCleaner(self)
+        self.object_cleaner.start()
+
+        # start a thread to refine regions when objects are clipped
+        self.dynamic_region_fps = EventsPerSecond()
+        self.region_refiner = RegionRefiner(self)
+        self.region_refiner.start()
+        self.dynamic_region_fps.start()
+
+        # start a thread to publish object scores
+        mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self)
+        mqtt_publisher.start()
+
+        # create a watchdog thread for capture process
+        self.watchdog = CameraWatchdog(self)
+
+        # load in the mask for object detection
+        if 'mask' in self.config:
+            self.mask = cv2.imread("/config/{}".format(self.config['mask']), cv2.IMREAD_GRAYSCALE)
+        else:
+            self.mask = None
+
+        if self.mask is None:
+            self.mask = np.zeros((self.frame_shape[0], self.frame_shape[1], 1), np.uint8)
+            self.mask[:] = 255
+
+
+    def start_or_restart_capture(self):
+        if not self.ffmpeg_process is None:
+            print("Terminating the existing ffmpeg process...")
+            self.ffmpeg_process.terminate()
+            try:
+                print("Waiting for ffmpeg to exit gracefully...")
+                self.ffmpeg_process.wait(timeout=30)
+            except sp.TimeoutExpired:
+                print("FFmpeg didnt exit. Force killing...")
+                self.ffmpeg_process.kill()
+                self.ffmpeg_process.wait()
+
+            print("Waiting for the capture thread to exit...")
+            self.capture_thread.join()
+            self.ffmpeg_process = None
+            self.capture_thread = None
+            
+        # create the process to capture frames from the input stream and store in a shared array
+        print("Creating a new ffmpeg process...")
+        self.start_ffmpeg()
+        
+        print("Creating a new capture thread...")
+        self.capture_thread = CameraCapture(self)
+        print("Starting a new capture thread...")
+        self.capture_thread.start()
+        self.fps.start()
+        self.skipped_region_tracker.start()
+    
+    def start_ffmpeg(self):
+        ffmpeg_cmd = (['ffmpeg'] +
+            self.ffmpeg_global_args +
+            self.ffmpeg_hwaccel_args +
+            self.ffmpeg_input_args +
+            ['-i', self.ffmpeg_input] +
+            self.ffmpeg_output_args +
+            ['pipe:'])
+
+        print(" ".join(ffmpeg_cmd))
+        
+        self.ffmpeg_process = sp.Popen(ffmpeg_cmd, stdout = sp.PIPE, bufsize=self.frame_size)
+    
+    def start(self):
+        self.start_or_restart_capture()
+        self.watchdog.start()
+    
+    def join(self):
+        self.capture_thread.join()
+    
+    def get_capture_pid(self):
+        return self.ffmpeg_process.pid
+    
+    def get_best(self, label):
+        return self.best_frames.best_frames.get(label)
+
+    def stats(self):
+        return {
+            'camera_fps': self.fps.eps(60),
+            'resize_queue': self.resize_queue.qsize(),
+            'frame_queue': self.frame_queue.qsize(),
+            'finished_frame_queue': self.finished_frame_queue.qsize(),
+            'refined_frame_queue': self.refined_frame_queue.qsize(),
+            'regions_in_process': self.regions_in_process,
+            'dynamic_regions_per_sec': self.dynamic_region_fps.eps(),
+            'skipped_regions_per_sec': self.skipped_region_tracker.eps(60)
+        }
+    
+    def frame_with_objects(self, frame_time, tracked_objects=None):
+        if not frame_time in self.frame_cache:
+            frame = np.zeros(self.frame_shape, np.uint8)
+        else:
+            frame = self.frame_cache[frame_time].copy()
+            
+        detected_objects = self.detected_objects[frame_time].copy()
+
+        for region in self.regions:
+            color = (255,255,255)
+            cv2.rectangle(frame, (region['x_offset'], region['y_offset']), 
+                (region['x_offset']+region['size'], region['y_offset']+region['size']), 
+                color, 2)
+
+        # draw the bounding boxes on the screen
+
+        if tracked_objects is None:
+            with self.object_tracker.tracked_objects_lock:
+                tracked_objects = copy.deepcopy(self.object_tracker.tracked_objects)
+
+        for obj in detected_objects:
+            draw_box_with_label(frame, obj['box']['xmin'], obj['box']['ymin'], obj['box']['xmax'], obj['box']['ymax'], obj['name'], "{}% {}".format(int(obj['score']*100), obj['area']), thickness=3)
+        
+        for id, obj in tracked_objects.items():
+            color = (0, 255,0) if obj['frame_time'] == frame_time else (255, 0, 0)
+            draw_box_with_label(frame, obj['box']['xmin'], obj['box']['ymin'], obj['box']['xmax'], obj['box']['ymax'], obj['name'], id, color=color, thickness=1, position='bl')
+
+        # print a timestamp
+        time_to_show = datetime.datetime.fromtimestamp(frame_time).strftime("%m/%d/%Y %H:%M:%S")
+        cv2.putText(frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
+        
+        # print fps
+        cv2.putText(frame, str(self.fps.eps())+'FPS', (10, 60), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
+
+        # convert to BGR
+        frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
+
+        # encode the image into a jpg
+        ret, jpg = cv2.imencode('.jpg', frame)
+
+        return jpg.tobytes()
+
+    def get_current_frame_with_objects(self):
+        frame_time = self.last_processed_frame
+        if frame_time == self.cached_frame_with_objects['frame_time']:
+            return self.cached_frame_with_objects['frame_bytes']
+
+        frame_bytes = self.frame_with_objects(frame_time)
+
+        self.cached_frame_with_objects = {
+            'frame_bytes': frame_bytes,
+            'frame_time': frame_time
+        }
+
+        return frame_bytes
+
+
+    
+