increase queue size and add ability to take every nth frame

.dockerignore

@@ -1,6 +1 @@
-README.md
-diagram.png
-.gitignore
-debug
-config/
-*.pyc
+README.md

.github/FUNDING.yml

@@ -1 +0,0 @@
-github: blakeblackshear

.gitignore

@@ -1,4 +1,2 @@
 *.pyc 
 debug
-.vscode
-config/config.yml

Dockerfile

@@ -1,70 +1,71 @@
-FROM ubuntu:18.04
+FROM ubuntu:16.04
 
-ARG DEVICE
-
-# Install packages for apt repo
-RUN apt-get -qq update && apt-get -qq install --no-install-recommends -y \
-    apt-transport-https \
-    ca-certificates \
-    curl \
-    wget \
-    gnupg-agent \
-    dirmngr \
-    software-properties-common \
-    && rm -rf /var/lib/apt/lists/*
-
-COPY scripts/install_odroid_repo.sh .
-
-RUN if [ "$DEVICE" = "odroid" ]; then \
-      sh /install_odroid_repo.sh; \
-    fi
-
-RUN apt-get -qq update && apt-get -qq install --no-install-recommends -y \
- python3 \ 
- # OpenCV dependencies
- ffmpeg \
+# Install system packages
+RUN apt-get -qq update && apt-get -qq install --no-install-recommends -y python3 \ 
+ python3-dev \
+ python-pil \
+ python-lxml \
+ python-tk \
  build-essential \
- cmake \
- unzip \
- pkg-config \
+ cmake \ 
+ git \ 
+ libgtk2.0-dev \ 
+ pkg-config \ 
+ libavcodec-dev \ 
+ libavformat-dev \ 
+ libswscale-dev \ 
+ libtbb2 \
+ libtbb-dev \ 
  libjpeg-dev \
  libpng-dev \
  libtiff-dev \
- libavcodec-dev \
- libavformat-dev \
- libswscale-dev \
- libv4l-dev \
- libxvidcore-dev \
- libx264-dev \
- libgtk-3-dev \
- libatlas-base-dev \
- gfortran \
- python3-dev \
- # Coral USB Python API Dependencies
- libusb-1.0-0 \
- python3-pip \
- python3-pil \
+ libjasper-dev \
+ libdc1394-22-dev \
+ x11-apps \
+ wget \
+ vim \
+ ffmpeg \
+ unzip \
+ libusb-1.0-0-dev \
+ python3-setuptools \
  python3-numpy \
- libc++1 \
- libc++abi1 \
- libunwind8 \
- libgcc1 \
- # VAAPI drivers for Intel hardware accel
- libva-drm2 libva2 i965-va-driver vainfo \
+ zlib1g-dev \
+ libgoogle-glog-dev \
+ swig \
+ libunwind-dev \
+ libc++-dev \
+ libc++abi-dev \
+ build-essential \
  && rm -rf /var/lib/apt/lists/* 
 
 # Install core packages 
 RUN wget -q -O /tmp/get-pip.py --no-check-certificate https://bootstrap.pypa.io/get-pip.py && python3 /tmp/get-pip.py
 RUN  pip install -U pip \
  numpy \
+ pillow \
+ matplotlib \
+ notebook \
  Flask \
+ imutils \
  paho-mqtt \
  PyYAML
 
+# Install tensorflow models object detection
+RUN GIT_SSL_NO_VERIFY=true git clone -q https://github.com/tensorflow/models /usr/local/lib/python3.5/dist-packages/tensorflow/models
+RUN wget -q -P /usr/local/src/ --no-check-certificate https://github.com/google/protobuf/releases/download/v3.5.1/protobuf-python-3.5.1.tar.gz
+
+# Download & build protobuf-python
+RUN cd /usr/local/src/ \
+ && tar xf protobuf-python-3.5.1.tar.gz \
+ && rm protobuf-python-3.5.1.tar.gz \
+ && cd /usr/local/src/protobuf-3.5.1/ \
+ && ./configure \
+ && make \
+ && make install \
+ && ldconfig \
+ && rm -rf /usr/local/src/protobuf-3.5.1/
+
 # Download & build OpenCV
-# TODO: use multistage build to reduce image size: 
-#   https://medium.com/@denismakogon/pain-and-gain-running-opencv-application-with-golang-and-docker-on-alpine-3-7-435aa11c7aec
-#   https://www.merixstudio.com/blog/docker-multi-stage-builds-python-development/
 RUN wget -q -P /usr/local/src/ --no-check-certificate https://github.com/opencv/opencv/archive/4.0.1.zip
 RUN cd /usr/local/src/ \
  && unzip 4.0.1.zip \
@@ -75,35 +76,32 @@ RUN cd /usr/local/src/ \
  && cmake -D CMAKE_INSTALL_TYPE=Release -D CMAKE_INSTALL_PREFIX=/usr/local/ .. \
  && make -j4 \
  && make install \
- && ldconfig \
  && rm -rf /usr/local/src/opencv-4.0.1
 
-# Download and install EdgeTPU libraries for Coral
-RUN wget https://dl.google.com/coral/edgetpu_api/edgetpu_api_latest.tar.gz -O edgetpu_api.tar.gz --trust-server-names \
-  && tar xzf edgetpu_api.tar.gz
+# Download and install EdgeTPU libraries
+RUN wget -q -O edgetpu_api.tar.gz --no-check-certificate http://storage.googleapis.com/cloud-iot-edge-pretrained-models/edgetpu_api.tar.gz
 
-COPY scripts/install_edgetpu_api.sh edgetpu_api/install.sh
-
-RUN cd edgetpu_api \
-  && /bin/bash install.sh
-
-# Copy a python 3.6 version
-RUN cd /usr/local/lib/python3.6/dist-packages/edgetpu/swig/ \
-  && ln -s _edgetpu_cpp_wrapper.cpython-35m-arm-linux-gnueabihf.so _edgetpu_cpp_wrapper.cpython-36m-arm-linux-gnueabihf.so
-
-# symlink the model and labels
-RUN wget https://dl.google.com/coral/canned_models/mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite -O mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite --trust-server-names
-RUN wget https://dl.google.com/coral/canned_models/coco_labels.txt -O coco_labels.txt --trust-server-names
-RUN ln -s mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite /frozen_inference_graph.pb
-RUN ln -s /coco_labels.txt /label_map.pbtext
+RUN tar xzf edgetpu_api.tar.gz \
+  && cd python-tflite-source \
+  && cp -p libedgetpu/libedgetpu_x86_64.so /lib/x86_64-linux-gnu/libedgetpu.so \
+  && cp edgetpu/swig/compiled_so/_edgetpu_cpp_wrapper_x86_64.so edgetpu/swig/_edgetpu_cpp_wrapper.so \
+  && cp edgetpu/swig/compiled_so/edgetpu_cpp_wrapper.py edgetpu/swig/ \
+  && python3 setup.py develop --user
 
 # Minimize image size 
 RUN (apt-get autoremove -y; \
      apt-get autoclean -y)
 
+# symlink the model and labels
+RUN ln -s /python-tflite-source/edgetpu/test_data/mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite /frozen_inference_graph.pb
+RUN ln -s /python-tflite-source/edgetpu/test_data/coco_labels.txt /label_map.pbtext
+
+# Set TF object detection available
+ENV PYTHONPATH "$PYTHONPATH:/usr/local/lib/python3.5/dist-packages/tensorflow/models/research:/usr/local/lib/python3.5/dist-packages/tensorflow/models/research/slim"
+RUN cd /usr/local/lib/python3.5/dist-packages/tensorflow/models/research && protoc object_detection/protos/*.proto --python_out=.
+
 WORKDIR /opt/frigate/
 ADD frigate frigate/
 COPY detect_objects.py .
-COPY benchmark.py .
 
 CMD ["python3", "-u", "detect_objects.py"]

README.md

@@ -1,7 +1,7 @@
-# Frigate - Realtime Object Detection for IP Cameras
+# Frigate - Realtime Object Detection for RTSP Cameras
 **Note:** This version requires the use of a [Google Coral USB Accelerator](https://coral.withgoogle.com/products/accelerator/)
 
-Uses OpenCV and Tensorflow to perform realtime object detection locally for IP cameras. Designed for integration with HomeAssistant or others via MQTT.
+Uses OpenCV and Tensorflow to perform realtime object detection locally for RTSP cameras. Designed for integration with HomeAssistant or others via MQTT.
 
 - Leverages multiprocessing and threads heavily with an emphasis on realtime over processing every frame
 - Allows you to define specific regions (squares) in the image to look for objects
@@ -30,9 +30,8 @@ docker run --rm \
 --privileged \
 -v /dev/bus/usb:/dev/bus/usb \
 -v <path_to_config_dir>:/config:ro \
--v /etc/localtime:/etc/localtime:ro \
 -p 5000:5000 \
--e FRIGATE_RTSP_PASSWORD='password' \
+-e RTSP_PASSWORD='password' \
 frigate:latest
 ```
 
@@ -45,58 +44,35 @@ Example docker-compose:
     image: frigate:latest
     volumes:
       - /dev/bus/usb:/dev/bus/usb
-      - /etc/localtime:/etc/localtime:ro
       - <path_to_config>:/config
     ports:
       - "5000:5000"
     environment:
-      FRIGATE_RTSP_PASSWORD: "password"
+      RTSP_PASSWORD: "password"
 ```
 
-A `config.yml` file must exist in the `config` directory. See example [here](config/config.example.yml) and device specific info can be found [here](docs/DEVICES.md).
+A `config.yml` file must exist in the `config` directory. See example [here](config/config.yml).
 
-Access the mjpeg stream at `http://localhost:5000/<camera_name>` and the best snapshot for any object type with at `http://localhost:5000/<camera_name>/<object_name>/best.jpg`
+Access the mjpeg stream at `http://localhost:5000/<camera_name>` and the best person snapshot at `http://localhost:5000/<camera_name>/best_person.jpg`
 
 ## Integration with HomeAssistant
 ```
 camera:
   - name: Camera Last Person
-    platform: mqtt
-    topic: frigate/<camera_name>/person/snapshot
-  - name: Camera Last Car
-    platform: mqtt
-    topic: frigate/<camera_name>/car/snapshot
+    platform: generic
+    still_image_url: http://<ip>:5000/<camera_name>/best_person.jpg
 
-binary_sensor:
+sensor:
   - name: Camera Person
     platform: mqtt
-    state_topic: "frigate/<camera_name>/person"
-    device_class: motion
+    state_topic: "frigate/<camera_name>/objects"
+    value_template: '{{ value_json.person }}'
+    device_class: moving
     availability_topic: "frigate/available"
-
-automation:
-  - alias: Alert me if a person is detected while armed away
-    trigger: 
-      platform: state
-      entity_id: binary_sensor.camera_person
-      from: 'off'
-      to: 'on'
-    condition:
-      - condition: state
-        entity_id: alarm_control_panel.home_alarm
-        state: armed_away
-    action:
-      - service: notify.user_telegram
-        data:
-          message: "A person was detected."
-          data:
-            photo:
-              - url: http://<ip>:5000/<camera_name>/person/best.jpg
-                caption: A person was detected.
 ```
 
 ## Tips
-- Lower the framerate of the video feed on the camera to reduce the CPU usage for capturing the feed
+- Lower the framerate of the RTSP feed on the camera to reduce the CPU usage for capturing the feed
 
 ## Future improvements
 - [x] Remove motion detection for now

benchmark.py

@@ -1,20 +0,0 @@
-import statistics
-import numpy as np
-from edgetpu.detection.engine import DetectionEngine
-
-# Path to frozen detection graph. This is the actual model that is used for the object detection.
-PATH_TO_CKPT = '/frozen_inference_graph.pb'
-
-# Load the edgetpu engine and labels
-engine = DetectionEngine(PATH_TO_CKPT)
-
-frame = np.zeros((300,300,3), np.uint8)
-flattened_frame = np.expand_dims(frame, axis=0).flatten()
-
-detection_times = []
-
-for x in range(0, 1000):
-    objects = engine.DetectWithInputTensor(flattened_frame, threshold=0.1, top_k=3)
-    detection_times.append(engine.get_inference_time())
-
-print("Average inference time: " + str(statistics.mean(detection_times)))

config/config.example.yml

@@ -1,128 +0,0 @@
-web_port: 5000
-
-mqtt:
-  host: mqtt.server.com
-  topic_prefix: frigate
-#  client_id: frigate # Optional -- set to override default client id of 'frigate' if running multiple instances
-#  user: username # Optional -- Uncomment for use
-#  password: password # Optional -- Uncomment for use
-
-#################
-# Default ffmpeg args. Optional and can be overwritten per camera.
-# Should work with most RTSP cameras that send h264 video
-# Built from the properties below with:
-# "ffmpeg" + global_args + input_args + "-i" + input + output_args
-#################
-# ffmpeg:
-#   global_args:
-#     - -hide_banner
-#     - -loglevel
-#     - panic
-#   hwaccel_args: []
-#   input_args:
-#     - -avoid_negative_ts
-#     - make_zero
-#     - -fflags
-#     - nobuffer
-#     - -flags
-#     - low_delay
-#     - -strict
-#     - experimental
-#     - -fflags
-#     - +genpts+discardcorrupt
-#     - -vsync
-#     - drop
-#     - -rtsp_transport
-#     - tcp
-#     - -stimeout
-#     - '5000000'
-#     - -use_wallclock_as_timestamps
-#     - '1'
-#   output_args:
-#     - -vf
-#     - mpdecimate
-#     - -f
-#     - rawvideo
-#     - -pix_fmt
-#     - rgb24
-
-####################
-# Global object configuration. Applies to all cameras and regions
-# unless overridden at the camera/region levels.
-# Keys must be valid labels. By default, the model uses coco (https://dl.google.com/coral/canned_models/coco_labels.txt).
-# All labels from the model are reported over MQTT. These values are used to filter out false positives.
-####################
-objects:
-  person:
-    min_area: 5000
-    max_area: 100000
-    threshold: 0.5
-
-cameras:
-  back:
-    ffmpeg:
-      ################
-      # Source passed to ffmpeg after the -i parameter. Supports anything compatible with OpenCV and FFmpeg.
-      # Environment variables that begin with 'FRIGATE_' may be referenced in {}
-      ################
-      input: rtsp://viewer:{FRIGATE_RTSP_PASSWORD}@10.0.10.10:554/cam/realmonitor?channel=1&subtype=2
-      #################
-      # These values will override default values for just this camera
-      #################
-      # global_args: []
-      # hwaccel_args: []
-      # input_args: []
-      # output_args: []
-
-    ################
-    ## Optional mask. Must be the same dimensions as your video feed.
-    ## The mask works by looking at the bottom center of the bounding box for the detected
-    ## person in the image. If that pixel in the mask is a black pixel, it ignores it as a
-    ## false positive. In my mask, the grass and driveway visible from my backdoor camera 
-    ## are white. The garage doors, sky, and trees (anywhere it would be impossible for a 
-    ## person to stand) are black.
-    ################
-    # mask: back-mask.bmp
-
-    ################
-    # Allows you to limit the framerate within frigate for cameras that do not support
-    # custom framerates. A value of 1 tells frigate to look at every frame, 2 every 2nd frame, 
-    # 3 every 3rd frame, etc.
-    ################
-    take_frame: 1
-
-    objects:
-      person:
-        min_area: 5000
-        max_area: 100000
-        threshold: 0.5
-    
-    ################
-    # size: size of the region in pixels
-    # x_offset/y_offset: position of the upper left corner of your region (top left of image is 0,0)
-    # min_person_area (optional): minimum width*height of the bounding box for the detected person
-    # max_person_area (optional): maximum width*height of the bounding box for the detected person
-    # threshold (optional): The minimum decimal percentage (50% hit = 0.5) for the confidence from tensorflow
-    # Tips: All regions are resized to 300x300 before detection because the model is trained on that size.
-    #       Resizing regions takes CPU power. Ideally, all regions should be as close to 300x300 as possible.
-    #       Defining a region that goes outside the bounds of the image will result in errors.
-    ################
-    regions:
-      - size: 350
-        x_offset: 0
-        y_offset: 300
-        objects:
-          car:
-            threshold: 0.2
-      - size: 400
-        x_offset: 350
-        y_offset: 250
-        objects:
-          person:
-            min_area: 2000
-      - size: 400
-        x_offset: 750
-        y_offset: 250
-        objects:
-          person:
-            min_area: 2000

config/config.yml

@@ -0,0 +1,42 @@
+web_port: 5000
+
+mqtt:
+  host: mqtt.server.com
+  topic_prefix: frigate
+
+cameras:
+  back:
+    rtsp:
+      user: viewer
+      host: 10.0.10.10
+      port: 554
+      # values that begin with a "$" will be replaced with environment variable
+      password: $RTSP_PASSWORD
+      path: /cam/realmonitor?channel=1&subtype=2
+    regions:
+      - size: 350
+        x_offset: 0
+        y_offset: 300
+      - size: 400
+        x_offset: 350
+        y_offset: 250
+      - size: 400
+        x_offset: 750
+        y_offset: 250
+    mask: back-mask.bmp
+    known_sizes:
+      - y: 300
+        min: 700
+        max: 1800
+      - y: 400
+        min: 3000
+        max: 7200
+      - y: 500
+        min: 8500
+        max: 20400
+      - y: 600
+        min: 10000
+        max: 50000
+      - y: 700
+        min: 10000
+        max: 125000

detect_objects.py

@@ -17,32 +17,6 @@ MQTT_PORT = CONFIG.get('mqtt', {}).get('port', 1883)
 MQTT_TOPIC_PREFIX = CONFIG.get('mqtt', {}).get('topic_prefix', 'frigate')
 MQTT_USER = CONFIG.get('mqtt', {}).get('user')
 MQTT_PASS = CONFIG.get('mqtt', {}).get('password')
-MQTT_CLIENT_ID = CONFIG.get('mqtt', {}).get('client_id', 'frigate')
-
-# Set the default FFmpeg config
-FFMPEG_CONFIG = CONFIG.get('ffmpeg', {})
-FFMPEG_DEFAULT_CONFIG = {
-    'global_args': FFMPEG_CONFIG.get('global_args', 
-        ['-hide_banner','-loglevel','panic']),
-    'hwaccel_args': FFMPEG_CONFIG.get('hwaccel_args', 
-        []),
-    'input_args': FFMPEG_CONFIG.get('input_args',
-        ['-avoid_negative_ts', 'make_zero',
-         '-fflags', 'nobuffer',
-         '-flags', 'low_delay',
-         '-strict', 'experimental',
-         '-fflags', '+genpts+discardcorrupt',
-         '-vsync', 'drop',
-         '-rtsp_transport', 'tcp',
-         '-stimeout', '5000000',
-         '-use_wallclock_as_timestamps', '1']),
-    'output_args': FFMPEG_CONFIG.get('output_args',
-        ['-vf', 'mpdecimate',
-         '-f', 'rawvideo',
-         '-pix_fmt', 'rgb24'])
-}
-
-GLOBAL_OBJECT_CONFIG = CONFIG.get('objects', {})
 
 WEB_PORT = CONFIG.get('web_port', 5000)
 DEBUG = (CONFIG.get('debug', '0') == '1')
@@ -51,18 +25,9 @@ def main():
     # connect to mqtt and setup last will
     def on_connect(client, userdata, flags, rc):
         print("On connect called")
-        if rc != 0:
-            if rc == 3:
-                print ("MQTT Server unavailable")
-            elif rc == 4:
-                print ("MQTT Bad username or password")
-            elif rc == 5:
-                print ("MQTT Not authorized")
-            else:
-                print ("Unable to connect to MQTT: Connection refused. Error code: " + str(rc))
         # publish a message to signal that the service is running
         client.publish(MQTT_TOPIC_PREFIX+'/available', 'online', retain=True)
-    client = mqtt.Client(client_id=MQTT_CLIENT_ID)
+    client = mqtt.Client()
     client.on_connect = on_connect
     client.will_set(MQTT_TOPIC_PREFIX+'/available', payload='offline', qos=1, retain=True)
     if not MQTT_USER is None:
@@ -71,12 +36,12 @@ def main():
     client.loop_start()
     
     # Queue for prepped frames, max size set to (number of cameras * 5)
-    max_queue_size = len(CONFIG['cameras'].items())*5
+    max_queue_size = len(CONFIG['cameras'].items())*10
     prepped_frame_queue = queue.Queue(max_queue_size)
 
     cameras = {}
     for name, config in CONFIG['cameras'].items():
-        cameras[name] = Camera(name, FFMPEG_DEFAULT_CONFIG, GLOBAL_OBJECT_CONFIG, config, prepped_frame_queue, client, MQTT_TOPIC_PREFIX)
+        cameras[name] = Camera(name, config, prepped_frame_queue, client, MQTT_TOPIC_PREFIX, DEBUG)
 
     prepped_queue_processor = PreppedQueueProcessor(
         cameras,
@@ -91,32 +56,21 @@ def main():
     # create a flask app that encodes frames a mjpeg on demand
     app = Flask(__name__)
 
-    @app.route('/')
-    def ishealthy():
-        # return a healh
-        return "Frigate is running. Alive and healthy!"
-
-    @app.route('/<camera_name>/<label>/best.jpg')
-    def best(camera_name, label):
-        if camera_name in cameras:
-            best_frame = cameras[camera_name].get_best(label)
-            if best_frame is None:
-                best_frame = np.zeros((720,1280,3), np.uint8)
-            ret, jpg = cv2.imencode('.jpg', best_frame)
-            response = make_response(jpg.tobytes())
-            response.headers['Content-Type'] = 'image/jpg'
-            return response
-        else:
-            return f'Camera named {camera_name} not found', 404
+    @app.route('/<camera_name>/best_person.jpg')
+    def best_person(camera_name):
+        best_person_frame = cameras[camera_name].get_best_person()
+        if best_person_frame is None:
+            best_person_frame = np.zeros((720,1280,3), np.uint8)
+        ret, jpg = cv2.imencode('.jpg', best_person_frame)
+        response = make_response(jpg.tobytes())
+        response.headers['Content-Type'] = 'image/jpg'
+        return response
 
     @app.route('/<camera_name>')
     def mjpeg_feed(camera_name):
-        if camera_name in cameras:
-            # return a multipart response
-            return Response(imagestream(camera_name),
-                            mimetype='multipart/x-mixed-replace; boundary=frame')
-        else:
-            return f'Camera named {camera_name} not found', 404
+        # return a multipart response
+        return Response(imagestream(camera_name),
+                        mimetype='multipart/x-mixed-replace; boundary=frame')
 
     def imagestream(camera_name):
         while True:
@@ -133,4 +87,4 @@ def main():
     camera.join()
 
 if __name__ == '__main__':
-    main()
+    main()

docs/DEVICES.md

@@ -1,74 +0,0 @@
-# Configuration Examples
-
-### Default (most RTSP cameras)
-This is the default ffmpeg command and should work with most RTSP cameras that send h264 video
-```yaml
-ffmpeg:
-  global_args:
-    - -hide_banner
-    - -loglevel
-    - panic
-  hwaccel_args: []
-  input_args:
-    - -avoid_negative_ts
-    - make_zero 
-    - -fflags
-    - nobuffer
-    - -flags
-    - low_delay
-    - -strict
-    - experimental
-    - -fflags
-    - +genpts+discardcorrupt
-    - -vsync
-    - drop
-    - -rtsp_transport
-    - tcp
-    - -stimeout
-    - '5000000' 
-    - -use_wallclock_as_timestamps
-    - '1'
-  output_args:
-    - -vf
-    - mpdecimate
-    - -f
-    - rawvideo
-    - -pix_fmt
-    - rgb24
-```
-
-### RTMP Cameras
-The input parameters need to be adjusted for RTMP cameras
-```yaml
-ffmpeg:
-input_args:
-    - -avoid_negative_ts
-    - make_zero
-    - -fflags
-    - nobuffer
-    - -flags
-    - low_delay
-    - -strict
-    - experimental
-    - -fflags
-    - +genpts+discardcorrupt
-    - -vsync
-    - drop
-    - -use_wallclock_as_timestamps
-    - '1'
-```
-
-
-### Hardware Acceleration
-
-Intel Quicksync
-```yaml
-ffmpeg:
-  hwaccel_args:
-    - -hwaccel
-    - vaapi
-    - -hwaccel_device
-    - /dev/dri/renderD128
-    - -hwaccel_output_format
-    - yuv420p
-```

frigate/mqtt.py

@@ -1,46 +1,33 @@
 import json
-import cv2
 import threading
-from collections import Counter, defaultdict
 
 class MqttObjectPublisher(threading.Thread):
-    def __init__(self, client, topic_prefix, objects_parsed, detected_objects, best_frames):
+    def __init__(self, client, topic_prefix, objects_parsed, detected_objects):
         threading.Thread.__init__(self)
         self.client = client
         self.topic_prefix = topic_prefix
         self.objects_parsed = objects_parsed
         self._detected_objects = detected_objects
-        self.best_frames = best_frames
 
     def run(self):
-        current_object_status = defaultdict(lambda: 'OFF')
+        last_sent_payload = ""
         while True:
+
+            # initialize the payload
+            payload = {}
+
             # wait until objects have been parsed
             with self.objects_parsed:
                 self.objects_parsed.wait()
 
-            # make a copy of detected objects
+            # add all the person scores in detected objects
             detected_objects = self._detected_objects.copy()
+            person_score = sum([obj['score'] for obj in detected_objects if obj['name'] == 'person'])
+            # if the person score is more than 100, set person to ON
+            payload['person'] = 'ON' if int(person_score*100) > 100 else 'OFF'
 
-            # total up all scores by object type
-            obj_counter = Counter()
-            for obj in detected_objects:
-                obj_counter[obj['name']] += obj['score']
-            
-            # report on detected objects
-            for obj_name, total_score in obj_counter.items():
-                new_status = 'ON' if int(total_score*100) > 100 else 'OFF'
-                if new_status != current_object_status[obj_name]:
-                    current_object_status[obj_name] = new_status
-                    self.client.publish(self.topic_prefix+'/'+obj_name, new_status, retain=False)
-                    # send the snapshot over mqtt as well
-                    if not self.best_frames.best_frames[obj_name] is None:
-                        ret, jpg = cv2.imencode('.jpg', self.best_frames.best_frames[obj_name])
-                        if ret:
-                            jpg_bytes = jpg.tobytes()
-                            self.client.publish(self.topic_prefix+'/'+obj_name+'/snapshot', jpg_bytes, retain=True)
-
-            # expire any objects that are ON and no longer detected
-            expired_objects = [obj_name for obj_name, status in current_object_status.items() if status == 'ON' and not obj_name in obj_counter]
-            for obj_name in expired_objects:
-                self.client.publish(self.topic_prefix+'/'+obj_name, 'OFF', retain=False)
+            # send message for objects if different
+            new_payload = json.dumps(payload, sort_keys=True)
+            if new_payload != last_sent_payload:
+                last_sent_payload = new_payload
+                self.client.publish(self.topic_prefix+'/objects', new_payload, retain=False)

frigate/object_detection.py

@@ -38,18 +38,19 @@ class PreppedQueueProcessor(threading.Thread):
             frame = self.prepped_frame_queue.get()
 
             # Actual detection.
-            objects = self.engine.DetectWithInputTensor(frame['frame'], threshold=0.5, top_k=5)
-            # print(self.engine.get_inference_time())
-
+            objects = self.engine.DetectWithInputTensor(frame['frame'], threshold=0.5, top_k=3)
             # parse and pass detected objects back to the camera
             parsed_objects = []
             for obj in objects:
+                box = obj.bounding_box.flatten().tolist()
                 parsed_objects.append({
-                            'region_id': frame['region_id'],
                             'frame_time': frame['frame_time'],
                             'name': str(self.labels[obj.label_id]),
                             'score': float(obj.score),
-                            'box': obj.bounding_box.flatten().tolist()
+                            'xmin': int((box[0] * frame['region_size']) + frame['region_x_offset']),
+                            'ymin': int((box[1] * frame['region_size']) + frame['region_y_offset']),
+                            'xmax': int((box[2] * frame['region_size']) + frame['region_x_offset']),
+                            'ymax': int((box[3] * frame['region_size']) + frame['region_y_offset'])
                         })
             self.cameras[frame['camera_name']].add_objects(parsed_objects)
 
@@ -58,7 +59,7 @@ class PreppedQueueProcessor(threading.Thread):
 class FramePrepper(threading.Thread):
     def __init__(self, camera_name, shared_frame, frame_time, frame_ready, 
         frame_lock,
-        region_size, region_x_offset, region_y_offset, region_id,
+        region_size, region_x_offset, region_y_offset,
         prepped_frame_queue):
 
         threading.Thread.__init__(self)
@@ -70,7 +71,6 @@ class FramePrepper(threading.Thread):
         self.region_size = region_size
         self.region_x_offset = region_x_offset
         self.region_y_offset = region_y_offset
-        self.region_id = region_id
         self.prepped_frame_queue = prepped_frame_queue
 
     def run(self):
@@ -88,11 +88,13 @@ class FramePrepper(threading.Thread):
                 cropped_frame = self.shared_frame[self.region_y_offset:self.region_y_offset+self.region_size, self.region_x_offset:self.region_x_offset+self.region_size].copy()
                 frame_time = self.frame_time.value
             
+            # convert to RGB
+            cropped_frame_rgb = cv2.cvtColor(cropped_frame, cv2.COLOR_BGR2RGB)
             # Resize to 300x300 if needed
-            if cropped_frame.shape != (300, 300, 3):
-                cropped_frame = cv2.resize(cropped_frame, dsize=(300, 300), interpolation=cv2.INTER_LINEAR)
+            if cropped_frame_rgb.shape != (300, 300, 3):
+                cropped_frame_rgb = cv2.resize(cropped_frame_rgb, dsize=(300, 300), interpolation=cv2.INTER_LINEAR)
             # Expand dimensions since the model expects images to have shape: [1, 300, 300, 3]
-            frame_expanded = np.expand_dims(cropped_frame, axis=0)
+            frame_expanded = np.expand_dims(cropped_frame_rgb, axis=0)
 
             # add the frame to the queue
             if not self.prepped_frame_queue.full():
@@ -101,7 +103,6 @@ class FramePrepper(threading.Thread):
                     'frame_time': frame_time,
                     'frame': frame_expanded.flatten().copy(),
                     'region_size': self.region_size,
-                    'region_id': self.region_id,
                     'region_x_offset': self.region_x_offset,
                     'region_y_offset': self.region_y_offset
                 })

frigate/objects.py

@@ -2,8 +2,7 @@ import time
 import datetime
 import threading
 import cv2
-import numpy as np
-from . util import draw_box_with_label
+from object_detection.utils import visualization_utils as vis_util
 
 class ObjectCleaner(threading.Thread):
     def __init__(self, objects_parsed, detected_objects):
@@ -36,15 +35,16 @@ class ObjectCleaner(threading.Thread):
                     self._objects_parsed.notify_all()
 
 
-# Maintains the frame and object with the highest score
-class BestFrames(threading.Thread):
+# Maintains the frame and person with the highest score from the most recent
+# motion event
+class BestPersonFrame(threading.Thread):
     def __init__(self, objects_parsed, recent_frames, detected_objects):
         threading.Thread.__init__(self)
         self.objects_parsed = objects_parsed
         self.recent_frames = recent_frames
         self.detected_objects = detected_objects
-        self.best_objects = {}
-        self.best_frames = {}
+        self.best_person = None
+        self.best_frame = None
 
     def run(self):
         while True:
@@ -55,30 +55,42 @@ class BestFrames(threading.Thread):
 
             # make a copy of detected objects
             detected_objects = self.detected_objects.copy()
+            detected_people = [obj for obj in detected_objects if obj['name'] == 'person']
 
-            for obj in detected_objects:
-                if obj['name'] in self.best_objects:
-                    now = datetime.datetime.now().timestamp()
-                    # if the object is a higher score than the current best score 
-                    # or the current object is more than 1 minute old, use the new object
-                    if obj['score'] > self.best_objects[obj['name']]['score'] or (now - self.best_objects[obj['name']]['frame_time']) > 60:
-                        self.best_objects[obj['name']] = obj
-                else:
-                    self.best_objects[obj['name']] = obj
+            # get the highest scoring person
+            new_best_person = max(detected_people, key=lambda x:x['score'], default=self.best_person)
+
+            # if there isnt a person, continue
+            if new_best_person is None:
+                continue
+
+            # if there is no current best_person
+            if self.best_person is None:
+                self.best_person = new_best_person
+            # if there is already a best_person
+            else:
+                now = datetime.datetime.now().timestamp()
+                # if the new best person is a higher score than the current best person 
+                # or the current person is more than 1 minute old, use the new best person
+                if new_best_person['score'] > self.best_person['score'] or (now - self.best_person['frame_time']) > 60:
+                    self.best_person = new_best_person
             
             # make a copy of the recent frames
             recent_frames = self.recent_frames.copy()
+            
+            if not self.best_person is None and self.best_person['frame_time'] in recent_frames:
+                best_frame = recent_frames[self.best_person['frame_time']]
+                best_frame = cv2.cvtColor(best_frame, cv2.COLOR_BGR2RGB)
+                # draw the bounding box on the frame
+                vis_util.draw_bounding_box_on_image_array(best_frame,
+                    self.best_person['ymin'],
+                    self.best_person['xmin'],
+                    self.best_person['ymax'],
+                    self.best_person['xmax'],
+                    color='red',
+                    thickness=2,
+                    display_str_list=["{}: {}%".format(self.best_person['name'],int(self.best_person['score']*100))],
+                    use_normalized_coordinates=False)
 
-            for name, obj in self.best_objects.items():
-                if obj['frame_time'] in recent_frames:
-                    best_frame = recent_frames[obj['frame_time']] #, np.zeros((720,1280,3), np.uint8))
-
-                    label = "{}: {}% {}".format(name,int(obj['score']*100),int(obj['area']))
-                    draw_box_with_label(best_frame, obj['xmin'], obj['ymin'], 
-                        obj['xmax'], obj['ymax'], label)
-                    
-                    # print a timestamp
-                    time_to_show = datetime.datetime.fromtimestamp(obj['frame_time']).strftime("%m/%d/%Y %H:%M:%S")
-                    cv2.putText(best_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
-                    
-                    self.best_frames[name] = cv2.cvtColor(best_frame, cv2.COLOR_RGB2BGR)
+                # convert back to BGR
+                self.best_frame = cv2.cvtColor(best_frame, cv2.COLOR_RGB2BGR)

frigate/util.py

@@ -1,26 +1,5 @@
 import numpy as np
-import cv2
 
 # convert shared memory array into numpy array
 def tonumpyarray(mp_arr):
-    return np.frombuffer(mp_arr.get_obj(), dtype=np.uint8)
-
-def draw_box_with_label(frame, x_min, y_min, x_max, y_max, label):
-    color = (255,0,0)
-    cv2.rectangle(frame, (x_min, y_min), 
-        (x_max, y_max), 
-        color, 2)
-    font_scale = 0.5
-    font = cv2.FONT_HERSHEY_SIMPLEX
-    # get the width and height of the text box
-    size = cv2.getTextSize(label, font, fontScale=font_scale, thickness=2)
-    text_width = size[0][0]
-    text_height = size[0][1]
-    line_height = text_height + size[1]
-    # set the text start position
-    text_offset_x = x_min
-    text_offset_y = 0 if y_min < line_height else y_min - (line_height+8)
-    # make the coords of the box with a small padding of two pixels
-    textbox_coords = ((text_offset_x, text_offset_y), (text_offset_x + text_width + 2, text_offset_y + line_height))
-    cv2.rectangle(frame, textbox_coords[0], textbox_coords[1], color, cv2.FILLED)
-    cv2.putText(frame, label, (text_offset_x, text_offset_y + line_height - 3), font, fontScale=font_scale, color=(0, 0, 0), thickness=2)
+    return np.frombuffer(mp_arr.get_obj(), dtype=np.uint8)

frigate/video.py

@@ -5,14 +5,64 @@ import cv2
 import threading
 import ctypes
 import multiprocessing as mp
-import subprocess as sp
 import numpy as np
-from collections import defaultdict
-from . util import tonumpyarray, draw_box_with_label
+from object_detection.utils import visualization_utils as vis_util
+from . util import tonumpyarray
 from . object_detection import FramePrepper
-from . objects import ObjectCleaner, BestFrames
+from . objects import ObjectCleaner, BestPersonFrame
 from . mqtt import MqttObjectPublisher
 
+# fetch the frames as fast a possible and store current frame in a shared memory array
+def fetch_frames(shared_arr, shared_frame_time, frame_lock, frame_ready, frame_shape, rtsp_url, take_frame=1):
+    # convert shared memory array into numpy and shape into image array
+    arr = tonumpyarray(shared_arr).reshape(frame_shape)
+
+    # start the video capture
+    video = cv2.VideoCapture()
+    video.open(rtsp_url)
+    # keep the buffer small so we minimize old data
+    video.set(cv2.CAP_PROP_BUFFERSIZE,1)
+
+    bad_frame_counter = 0
+    frame_num = 0
+    while True:
+        # check if the video stream is still open, and reopen if needed
+        if not video.isOpened():
+            success = video.open(rtsp_url)
+            if not success:
+                time.sleep(1)
+                continue
+        # grab the frame, but dont decode it yet
+        ret = video.grab()
+        # snapshot the time the frame was grabbed
+        frame_time = datetime.datetime.now()
+        if ret:
+            frame_num += 1
+            if (frame_num % take_frame) != 0:
+                continue
+            # go ahead and decode the current frame
+            ret, frame = video.retrieve()
+            if ret:
+                # Lock access and update frame
+                with frame_lock:
+                    arr[:] = frame
+                    shared_frame_time.value = frame_time.timestamp()
+                # Notify with the condition that a new frame is ready
+                with frame_ready:
+                    frame_ready.notify_all()
+                bad_frame_counter = 0
+            else:
+                print("Unable to decode frame")
+                bad_frame_counter += 1
+        else:
+            print("Unable to grab a frame")
+            bad_frame_counter += 1
+        
+        if bad_frame_counter > 100:
+            video.release()
+    
+    video.release()
+
 # Stores 2 seconds worth of frames when motion is detected so they can be used for other threads
 class FrameTracker(threading.Thread):
     def __init__(self, shared_frame, frame_time, frame_ready, frame_lock, recent_frames):
@@ -47,97 +97,93 @@ class FrameTracker(threading.Thread):
                 if (now - k) > 2:
                     del self.recent_frames[k]
 
-def get_frame_shape(source):
+def get_frame_shape(rtsp_url):
     # capture a single frame and check the frame shape so the correct array
     # size can be allocated in memory
-    video = cv2.VideoCapture(source)
+    video = cv2.VideoCapture(rtsp_url)
     ret, frame = video.read()
     frame_shape = frame.shape
     video.release()
     return frame_shape
 
-def get_ffmpeg_input(ffmpeg_input):
-    frigate_vars = {k: v for k, v in os.environ.items() if k.startswith('FRIGATE_')}
-    return ffmpeg_input.format(**frigate_vars)
+def get_rtsp_url(rtsp_config):
+    if (rtsp_config['password'].startswith('$')):
+        rtsp_config['password'] = os.getenv(rtsp_config['password'][1:])
+    return 'rtsp://{}:{}@{}:{}{}'.format(rtsp_config['user'], 
+        rtsp_config['password'], rtsp_config['host'], rtsp_config['port'],
+        rtsp_config['path'])
 
-class CameraWatchdog(threading.Thread):
-    def __init__(self, camera):
-        threading.Thread.__init__(self)
-        self.camera = camera
+def compute_sizes(frame_shape, known_sizes, mask):
+    # create a 3 dimensional numpy array to store estimated sizes
+    estimated_sizes = np.zeros((frame_shape[0], frame_shape[1], 2), np.uint32)
 
-    def run(self):
+    sorted_positions = sorted(known_sizes, key=lambda s: s['y'])
 
-        while True:
-            # wait a bit before checking
-            time.sleep(10)
+    last_position = {'y': 0, 'min': 0, 'max': 0}
+    next_position = sorted_positions.pop(0)
+    # if the next position has the same y coordinate, skip
+    while next_position['y'] == last_position['y']:
+        next_position = sorted_positions.pop(0)
+    y_change = next_position['y']-last_position['y']
+    min_size_change = next_position['min']-last_position['min']
+    max_size_change = next_position['max']-last_position['max']
+    min_step_size = min_size_change/y_change
+    max_step_size = max_size_change/y_change
 
-            if (datetime.datetime.now().timestamp() - self.camera.frame_time.value) > 300:
-                print("last frame is more than 5 minutes old, restarting camera capture...")
-                self.camera.start_or_restart_capture()
-                time.sleep(5)
+    min_current_size = 0
+    max_current_size = 0
 
-# Thread to read the stdout of the ffmpeg process and update the current frame
-class CameraCapture(threading.Thread):
-    def __init__(self, camera):
-        threading.Thread.__init__(self)
-        self.camera = camera
+    for y_position in range(frame_shape[0]):
+        # fill the row with the estimated size
+        estimated_sizes[y_position,:] = [min_current_size, max_current_size]
 
-    def run(self):
-        frame_num = 0
-        while True:
-            if self.camera.ffmpeg_process.poll() != None:
-                print("ffmpeg process is not running. exiting capture thread...")
-                break
+        # if you have reached the next size
+        if y_position == next_position['y']:
+            last_position = next_position
+            # if there are still positions left
+            if len(sorted_positions) > 0:
+                next_position = sorted_positions.pop(0)
+                # if the next position has the same y coordinate, skip
+                while next_position['y'] == last_position['y']:
+                    next_position = sorted_positions.pop(0)
+                y_change = next_position['y']-last_position['y']
+                min_size_change = next_position['min']-last_position['min']
+                max_size_change = next_position['max']-last_position['max']
+                min_step_size = min_size_change/y_change
+                max_step_size = max_size_change/y_change
+            else:
+                min_step_size = 0
+                max_step_size = 0
+        
+        min_current_size += min_step_size
+        max_current_size += max_step_size
 
-            raw_image = self.camera.ffmpeg_process.stdout.read(self.camera.frame_size)
+    # apply mask by filling 0s for all locations a person could not be standing
+    if mask is not None:
+        pass
 
-            if len(raw_image) == 0:
-                print("ffmpeg didnt return a frame. something is wrong. exiting capture thread...")
-                break
-
-            frame_num += 1
-            if (frame_num % self.camera.take_frame) != 0:
-                continue
-
-            with self.camera.frame_lock:
-                self.camera.frame_time.value = datetime.datetime.now().timestamp()
-                
-                self.camera.current_frame[:] = (
-                    np
-                    .frombuffer(raw_image, np.uint8)
-                    .reshape(self.camera.frame_shape)
-                )
-            # Notify with the condition that a new frame is ready
-            with self.camera.frame_ready:
-                self.camera.frame_ready.notify_all()
+    return estimated_sizes
 
 class Camera:
-    def __init__(self, name, ffmpeg_config, global_objects_config, config, prepped_frame_queue, mqtt_client, mqtt_prefix):
+    def __init__(self, name, config, prepped_frame_queue, mqtt_client, mqtt_prefix, debug=False):
         self.name = name
         self.config = config
         self.detected_objects = []
         self.recent_frames = {}
-
-        self.ffmpeg = config.get('ffmpeg', {})
-        self.ffmpeg_input = get_ffmpeg_input(self.ffmpeg['input'])
-        self.ffmpeg_global_args = self.ffmpeg.get('global_args', ffmpeg_config['global_args'])
-        self.ffmpeg_hwaccel_args = self.ffmpeg.get('hwaccel_args', ffmpeg_config['hwaccel_args'])
-        self.ffmpeg_input_args = self.ffmpeg.get('input_args', ffmpeg_config['input_args'])
-        self.ffmpeg_output_args = self.ffmpeg.get('output_args', ffmpeg_config['output_args'])
-
-        camera_objects_config = config.get('objects', {})
-
+        self.rtsp_url = get_rtsp_url(self.config['rtsp'])
         self.take_frame = self.config.get('take_frame', 1)
         self.regions = self.config['regions']
-        self.frame_shape = get_frame_shape(self.ffmpeg_input)
-        self.frame_size = self.frame_shape[0] * self.frame_shape[1] * self.frame_shape[2]
+        self.frame_shape = get_frame_shape(self.rtsp_url)
         self.mqtt_client = mqtt_client
         self.mqtt_topic_prefix = '{}/{}'.format(mqtt_prefix, self.name)
+        self.debug = debug
 
-        # create a numpy array for the current frame in initialize to zeros
-        self.current_frame = np.zeros(self.frame_shape, np.uint8)
+        # compute the flattened array length from the shape of the frame
+        flat_array_length = self.frame_shape[0] * self.frame_shape[1] * self.frame_shape[2]
+        # create shared array for storing the full frame image data
+        self.shared_frame_array = mp.Array(ctypes.c_uint8, flat_array_length)
         # create shared value for storing the frame_time
-        self.frame_time = mp.Value('d', 0.0)
+        self.shared_frame_time = mp.Value('d', 0.0)
         # Lock to control access to the frame
         self.frame_lock = mp.Lock()
         # Condition for notifying that a new frame is ready
@@ -145,197 +191,139 @@ class Camera:
         # Condition for notifying that objects were parsed
         self.objects_parsed = mp.Condition()
 
-        self.ffmpeg_process = None
-        self.capture_thread = None
+        # shape current frame so it can be treated as a numpy image
+        self.shared_frame_np = tonumpyarray(self.shared_frame_array).reshape(self.frame_shape)
+
+        # create the process to capture frames from the RTSP stream and store in a shared array
+        self.capture_process = mp.Process(target=fetch_frames, args=(self.shared_frame_array, 
+            self.shared_frame_time, self.frame_lock, self.frame_ready, self.frame_shape,
+            self.rtsp_url, self.take_frame))
+        self.capture_process.daemon = True
 
         # for each region, create a separate thread to resize the region and prep for detection
         self.detection_prep_threads = []
-        for index, region in enumerate(self.config['regions']):
-            region_objects = region.get('objects', {})
-            # build objects config for region
-            objects_with_config = set().union(global_objects_config.keys(), camera_objects_config.keys(), region_objects.keys())
-            merged_objects_config = defaultdict(lambda: {})
-            for obj in objects_with_config:
-                merged_objects_config[obj] = {**global_objects_config.get(obj,{}), **camera_objects_config.get(obj, {}), **region_objects.get(obj, {})}
-            
-            region['objects'] = merged_objects_config
-
+        for region in self.config['regions']:
             self.detection_prep_threads.append(FramePrepper(
                 self.name,
-                self.current_frame,
-                self.frame_time,
+                self.shared_frame_np,
+                self.shared_frame_time,
                 self.frame_ready,
                 self.frame_lock,
-                region['size'], region['x_offset'], region['y_offset'], index,
+                region['size'], region['x_offset'], region['y_offset'],
                 prepped_frame_queue
             ))
         
         # start a thread to store recent motion frames for processing
-        self.frame_tracker = FrameTracker(self.current_frame, self.frame_time, 
+        self.frame_tracker = FrameTracker(self.shared_frame_np, self.shared_frame_time, 
             self.frame_ready, self.frame_lock, self.recent_frames)
         self.frame_tracker.start()
 
-        # start a thread to store the highest scoring recent frames for monitored object types
-        self.best_frames = BestFrames(self.objects_parsed, self.recent_frames, self.detected_objects)
-        self.best_frames.start()
+        # start a thread to store the highest scoring recent person frame
+        self.best_person_frame = BestPersonFrame(self.objects_parsed, self.recent_frames, self.detected_objects)
+        self.best_person_frame.start()
 
         # start a thread to expire objects from the detected objects list
         self.object_cleaner = ObjectCleaner(self.objects_parsed, self.detected_objects)
         self.object_cleaner.start()
 
-        # start a thread to publish object scores
-        mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self.objects_parsed, self.detected_objects, self.best_frames)
+        # start a thread to publish object scores (currently only person)
+        mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self.objects_parsed, self.detected_objects)
         mqtt_publisher.start()
 
-        # create a watchdog thread for capture process
-        self.watchdog = CameraWatchdog(self)
-
-        # load in the mask for object detection
+        # load in the mask for person detection
         if 'mask' in self.config:
             self.mask = cv2.imread("/config/{}".format(self.config['mask']), cv2.IMREAD_GRAYSCALE)
         else:
-            self.mask = None
-
-        if self.mask is None:
             self.mask = np.zeros((self.frame_shape[0], self.frame_shape[1], 1), np.uint8)
             self.mask[:] = 255
 
-
-    def start_or_restart_capture(self):
-        if not self.ffmpeg_process is None:
-            print("Terminating the existing ffmpeg process...")
-            self.ffmpeg_process.terminate()
-            try:
-                print("Waiting for ffmpeg to exit gracefully...")
-                self.ffmpeg_process.wait(timeout=30)
-            except sp.TimeoutExpired:
-                print("FFmpeg didnt exit. Force killing...")
-                self.ffmpeg_process.kill()
-                self.ffmpeg_process.wait()
-
-            print("Waiting for the capture thread to exit...")
-            self.capture_thread.join()
-            self.ffmpeg_process = None
-            self.capture_thread = None
-            
-        # create the process to capture frames from the input stream and store in a shared array
-        print("Creating a new ffmpeg process...")
-        self.start_ffmpeg()
-        
-        print("Creating a new capture thread...")
-        self.capture_thread = CameraCapture(self)
-        print("Starting a new capture thread...")
-        self.capture_thread.start()
-    
-    def start_ffmpeg(self):
-        ffmpeg_cmd = (['ffmpeg'] +
-            self.ffmpeg_global_args +
-            self.ffmpeg_hwaccel_args +
-            self.ffmpeg_input_args +
-            ['-i', self.ffmpeg_input] +
-            self.ffmpeg_output_args +
-            ['pipe:'])
-
-        print(" ".join(ffmpeg_cmd))
-        
-        self.ffmpeg_process = sp.Popen(ffmpeg_cmd, stdout = sp.PIPE, bufsize=self.frame_size)
+        # pre-compute estimated person size for every pixel in the image
+        if 'known_sizes' in self.config:
+            self.calculated_person_sizes = compute_sizes((self.frame_shape[0], self.frame_shape[1]), 
+                self.config['known_sizes'], None)
+        else:
+            self.calculated_person_sizes = None
     
     def start(self):
-        self.start_or_restart_capture()
+        self.capture_process.start()
         # start the object detection prep threads
         for detection_prep_thread in self.detection_prep_threads:
             detection_prep_thread.start()
-        self.watchdog.start()
     
     def join(self):
-        self.capture_thread.join()
+        self.capture_process.join()
     
     def get_capture_pid(self):
-        return self.ffmpeg_process.pid
+        return self.capture_process.pid
     
     def add_objects(self, objects):
         if len(objects) == 0:
             return
 
         for obj in objects:
-            # find the matching region
-            region = self.regions[obj['region_id']]
-
-            # Compute some extra properties
-            obj.update({
-                'xmin': int((obj['box'][0] * region['size']) + region['x_offset']),
-                'ymin': int((obj['box'][1] * region['size']) + region['y_offset']),
-                'xmax': int((obj['box'][2] * region['size']) + region['x_offset']),
-                'ymax': int((obj['box'][3] * region['size']) + region['y_offset'])
-            })
+            if self.debug:
+                # print out the detected objects, scores and locations
+                print(self.name, obj['name'], obj['score'], obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'])
             
-            # Compute the area
-            obj['area'] = (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin'])
+            location = (int(obj['ymax']), int((obj['xmax']-obj['xmin'])/2))
 
-            object_name = obj['name']
+            # if the person is in a masked location, continue
+            if self.mask[location[0]][location[1]] == [0]:
+                continue
 
-            if object_name in region['objects']:
-                obj_settings = region['objects'][object_name]
+            if self.calculated_person_sizes is not None and obj['name'] == 'person':
+                person_size_range = self.calculated_person_sizes[location[0]][location[1]]
 
-                # if the min area is larger than the
-                # detected object, don't add it to detected objects
-                if obj_settings.get('min_area',-1) > obj['area']:
-                    continue
-                
-                # if the detected object is larger than the
-                # max area, don't add it to detected objects
-                if obj_settings.get('max_area', region['size']**2) < obj['area']:
+                # if the person isnt on the ground, continue
+                if(person_size_range[0] == 0 and person_size_range[1] == 0):
                     continue
 
-                # if the score is lower than the threshold, skip
-                if obj_settings.get('threshold', 0) > obj['score']:
-                    continue
-            
-                # compute the coordinates of the object and make sure
-                # the location isnt outside the bounds of the image (can happen from rounding)
-                y_location = min(int(obj['ymax']), len(self.mask)-1)
-                x_location = min(int((obj['xmax']-obj['xmin'])/2.0)+obj['xmin'], len(self.mask[0])-1)
+                person_size = (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin'])
 
-                # if the object is in a masked location, don't add it to detected objects
-                if self.mask[y_location][x_location] == [0]:
+                # if the person is not within 20% of the estimated size for that location, continue
+                if person_size < person_size_range[0] or person_size > person_size_range[1]:
                     continue
 
             self.detected_objects.append(obj)
 
         with self.objects_parsed:
             self.objects_parsed.notify_all()
-    
-    def get_best(self, label):
-        return self.best_frames.best_frames.get(label)
+
+    def get_best_person(self):
+        return self.best_person_frame.best_frame
     
     def get_current_frame_with_objects(self):
         # make a copy of the current detected objects
         detected_objects = self.detected_objects.copy()
         # lock and make a copy of the current frame
         with self.frame_lock:
-            frame = self.current_frame.copy()
-            frame_time = self.frame_time.value
+            frame = self.shared_frame_np.copy()
 
+        # convert to RGB for drawing
+        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
         # draw the bounding boxes on the screen
         for obj in detected_objects:
-            label = "{}: {}% {}".format(obj['name'],int(obj['score']*100),int(obj['area']))
-            draw_box_with_label(frame, obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'], label)
+            vis_util.draw_bounding_box_on_image_array(frame,
+                obj['ymin'],
+                obj['xmin'],
+                obj['ymax'],
+                obj['xmax'],
+                color='red',
+                thickness=2,
+                display_str_list=["{}: {}%".format(obj['name'],int(obj['score']*100))],
+                use_normalized_coordinates=False)
 
         for region in self.regions:
             color = (255,255,255)
             cv2.rectangle(frame, (region['x_offset'], region['y_offset']), 
                 (region['x_offset']+region['size'], region['y_offset']+region['size']), 
                 color, 2)
-        
-        # print a timestamp
-        time_to_show = datetime.datetime.fromtimestamp(frame_time).strftime("%m/%d/%Y %H:%M:%S")
-        cv2.putText(frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
 
-        # convert to BGR
+        # convert back to BGR
         frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
 
         return frame
 
 
     
-        
+        

scripts/install_edgetpu_api.sh

@@ -1,50 +0,0 @@
-#!/bin/bash
-set -e
-
-CPU_ARCH=$(uname -m)
-OS_VERSION=$(uname -v)
-
-echo "CPU_ARCH ${CPU_ARCH}"
-echo "OS_VERSION ${OS_VERSION}"
-
-if [[ "${CPU_ARCH}" == "x86_64" ]]; then
-  echo "Recognized as Linux on x86_64."
-  LIBEDGETPU_SUFFIX=x86_64
-  HOST_GNU_TYPE=x86_64-linux-gnu
-elif [[ "${CPU_ARCH}" == "armv7l" ]]; then
-  echo "Recognized as Linux on ARM32 platform."
-  LIBEDGETPU_SUFFIX=arm32
-  HOST_GNU_TYPE=arm-linux-gnueabihf
-elif [[ "${CPU_ARCH}" == "aarch64" ]]; then
-  echo "Recognized as generic ARM64 platform."
-  LIBEDGETPU_SUFFIX=arm64
-  HOST_GNU_TYPE=aarch64-linux-gnu
-fi
-
-if [[ -z "${HOST_GNU_TYPE}" ]]; then
-  echo "Your platform is not supported."
-  exit 1
-fi
-
-echo "Using maximum operating frequency."
-LIBEDGETPU_SRC="libedgetpu/libedgetpu_${LIBEDGETPU_SUFFIX}.so"
-LIBEDGETPU_DST="/usr/lib/${HOST_GNU_TYPE}/libedgetpu.so.1.0"
-
-# Runtime library.
-echo "Installing Edge TPU runtime library [${LIBEDGETPU_DST}]..."
-if [[ -f "${LIBEDGETPU_DST}" ]]; then
-  echo "File already exists. Replacing it..."
-  rm -f "${LIBEDGETPU_DST}"
-fi
-
-cp -p "${LIBEDGETPU_SRC}" "${LIBEDGETPU_DST}"
-ldconfig
-echo "Done."
-
-# Python API.
-WHEEL=$(ls edgetpu-*-py3-none-any.whl 2>/dev/null)
-if [[ $? == 0 ]]; then
-  echo "Installing Edge TPU Python API..."
-  python3 -m pip install --no-deps "${WHEEL}"
-  echo "Done."
-fi

scripts/install_odroid_repo.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-apt-key adv --keyserver keyserver.ubuntu.com --recv-keys D986B59D
-
-echo "deb http://deb.odroid.in/5422-s bionic main" > /etc/apt/sources.list.d/odroid.list