use a thread to capture frames from the subprocess so it can be killed properly

* Add object size to the bounding box

Remove script from Dockerfile

Fix framerate command

Move default value for framerate

update dockerfile

dockerfile changes

Add person_area label to surrounding box


Update dockerfile


ffmpeg config bug


Add `person_area` label to `best_person` frame


Resolve debug view showing area label for non-persons


Add object size to the bounding box


Add object size to the bounding box

* Move object area outside of conditional to work with all object types

.dockerignore

@@ -1 +1,6 @@
-README.md
+README.md
+diagram.png
+.gitignore
+debug
+config/
+*.pyc

.github/FUNDING.yml

@@ -0,0 +1 @@
+ko_fi: blakeblackshear

Dockerfile

@@ -1,71 +1,70 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
-# Install system packages
-RUN apt-get -qq update && apt-get -qq install --no-install-recommends -y python3 \ 
- python3-dev \
- python-pil \
- python-lxml \
- python-tk \
+ARG DEVICE
+
+# Install packages for apt repo
+RUN apt-get -qq update && apt-get -qq install --no-install-recommends -y \
+    apt-transport-https \
+    ca-certificates \
+    curl \
+    wget \
+    gnupg-agent \
+    dirmngr \
+    software-properties-common \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY scripts/install_odroid_repo.sh .
+
+RUN if [ "$DEVICE" = "odroid" ]; then \
+      sh /install_odroid_repo.sh; \
+    fi
+
+RUN apt-get -qq update && apt-get -qq install --no-install-recommends -y \
+ python3 \ 
+ # OpenCV dependencies
+ ffmpeg \
  build-essential \
- cmake \ 
- git \ 
- libgtk2.0-dev \ 
- pkg-config \ 
- libavcodec-dev \ 
- libavformat-dev \ 
- libswscale-dev \ 
- libtbb2 \
- libtbb-dev \ 
+ cmake \
+ unzip \
+ pkg-config \
  libjpeg-dev \
  libpng-dev \
  libtiff-dev \
- libjasper-dev \
- libdc1394-22-dev \
- x11-apps \
- wget \
- vim \
- ffmpeg \
- unzip \
- libusb-1.0-0-dev \
- python3-setuptools \
+ libavcodec-dev \
+ libavformat-dev \
+ libswscale-dev \
+ libv4l-dev \
+ libxvidcore-dev \
+ libx264-dev \
+ libgtk-3-dev \
+ libatlas-base-dev \
+ gfortran \
+ python3-dev \
+ # Coral USB Python API Dependencies
+ libusb-1.0-0 \
+ python3-pip \
+ python3-pil \
  python3-numpy \
- zlib1g-dev \
- libgoogle-glog-dev \
- swig \
- libunwind-dev \
- libc++-dev \
- libc++abi-dev \
- build-essential \
+ libc++1 \
+ libc++abi1 \
+ libunwind8 \
+ libgcc1 \
+ # VAAPI drivers for Intel hardware accel
+ libva-drm2 libva2 i965-va-driver vainfo \
  && rm -rf /var/lib/apt/lists/* 
 
 # Install core packages 
 RUN wget -q -O /tmp/get-pip.py --no-check-certificate https://bootstrap.pypa.io/get-pip.py && python3 /tmp/get-pip.py
 RUN  pip install -U pip \
  numpy \
- pillow \
- matplotlib \
- notebook \
  Flask \
- imutils \
  paho-mqtt \
  PyYAML
 
-# Install tensorflow models object detection
-RUN GIT_SSL_NO_VERIFY=true git clone -q https://github.com/tensorflow/models /usr/local/lib/python3.5/dist-packages/tensorflow/models
-RUN wget -q -P /usr/local/src/ --no-check-certificate https://github.com/google/protobuf/releases/download/v3.5.1/protobuf-python-3.5.1.tar.gz
-
-# Download & build protobuf-python
-RUN cd /usr/local/src/ \
- && tar xf protobuf-python-3.5.1.tar.gz \
- && rm protobuf-python-3.5.1.tar.gz \
- && cd /usr/local/src/protobuf-3.5.1/ \
- && ./configure \
- && make \
- && make install \
- && ldconfig \
- && rm -rf /usr/local/src/protobuf-3.5.1/
-
 # Download & build OpenCV
+# TODO: use multistage build to reduce image size: 
+#   https://medium.com/@denismakogon/pain-and-gain-running-opencv-application-with-golang-and-docker-on-alpine-3-7-435aa11c7aec
+#   https://www.merixstudio.com/blog/docker-multi-stage-builds-python-development/
 RUN wget -q -P /usr/local/src/ --no-check-certificate https://github.com/opencv/opencv/archive/4.0.1.zip
 RUN cd /usr/local/src/ \
  && unzip 4.0.1.zip \
@@ -76,32 +75,35 @@ RUN cd /usr/local/src/ \
  && cmake -D CMAKE_INSTALL_TYPE=Release -D CMAKE_INSTALL_PREFIX=/usr/local/ .. \
  && make -j4 \
  && make install \
+ && ldconfig \
  && rm -rf /usr/local/src/opencv-4.0.1
 
-# Download and install EdgeTPU libraries
-RUN wget -q -O edgetpu_api.tar.gz --no-check-certificate http://storage.googleapis.com/cloud-iot-edge-pretrained-models/edgetpu_api.tar.gz
+# Download and install EdgeTPU libraries for Coral
+RUN wget https://dl.google.com/coral/edgetpu_api/edgetpu_api_latest.tar.gz -O edgetpu_api.tar.gz --trust-server-names \
+  && tar xzf edgetpu_api.tar.gz
 
-RUN tar xzf edgetpu_api.tar.gz \
-  && cd python-tflite-source \
-  && cp -p libedgetpu/libedgetpu_x86_64.so /lib/x86_64-linux-gnu/libedgetpu.so \
-  && cp edgetpu/swig/compiled_so/_edgetpu_cpp_wrapper_x86_64.so edgetpu/swig/_edgetpu_cpp_wrapper.so \
-  && cp edgetpu/swig/compiled_so/edgetpu_cpp_wrapper.py edgetpu/swig/ \
-  && python3 setup.py develop --user
+COPY scripts/install_edgetpu_api.sh edgetpu_api/install.sh
+
+RUN cd edgetpu_api \
+  && /bin/bash install.sh
+
+# Copy a python 3.6 version
+RUN cd /usr/local/lib/python3.6/dist-packages/edgetpu/swig/ \
+  && ln -s _edgetpu_cpp_wrapper.cpython-35m-arm-linux-gnueabihf.so _edgetpu_cpp_wrapper.cpython-36m-arm-linux-gnueabihf.so
+
+# symlink the model and labels
+RUN wget https://dl.google.com/coral/canned_models/mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite -O mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite --trust-server-names
+RUN wget https://dl.google.com/coral/canned_models/coco_labels.txt -O coco_labels.txt --trust-server-names
+RUN ln -s mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite /frozen_inference_graph.pb
+RUN ln -s /coco_labels.txt /label_map.pbtext
 
 # Minimize image size 
 RUN (apt-get autoremove -y; \
      apt-get autoclean -y)
 
-# symlink the model and labels
-RUN ln -s /python-tflite-source/edgetpu/test_data/mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite /frozen_inference_graph.pb
-RUN ln -s /python-tflite-source/edgetpu/test_data/coco_labels.txt /label_map.pbtext
-
-# Set TF object detection available
-ENV PYTHONPATH "$PYTHONPATH:/usr/local/lib/python3.5/dist-packages/tensorflow/models/research:/usr/local/lib/python3.5/dist-packages/tensorflow/models/research/slim"
-RUN cd /usr/local/lib/python3.5/dist-packages/tensorflow/models/research && protoc object_detection/protos/*.proto --python_out=.
-
 WORKDIR /opt/frigate/
 ADD frigate frigate/
 COPY detect_objects.py .
+COPY benchmark.py .
 
 CMD ["python3", "-u", "detect_objects.py"]

README.md

@@ -1,3 +1,5 @@
+<a href='https://ko-fi.com/P5P7XGO9' target='_blank'><img height='36' style='border:0px;height:36px;' src='https://az743702.vo.msecnd.net/cdn/kofi4.png?v=2' border='0' alt='Buy Me a Coffee at ko-fi.com' /></a>
+
 # Frigate - Realtime Object Detection for RTSP Cameras
 **Note:** This version requires the use of a [Google Coral USB Accelerator](https://coral.withgoogle.com/products/accelerator/)
 
@@ -62,12 +64,12 @@ camera:
     platform: generic
     still_image_url: http://<ip>:5000/<camera_name>/best_person.jpg
 
-sensor:
+binary_sensor:
   - name: Camera Person
     platform: mqtt
     state_topic: "frigate/<camera_name>/objects"
     value_template: '{{ value_json.person }}'
-    device_class: moving
+    device_class: motion
     availability_topic: "frigate/available"
 ```
 

benchmark.py

@@ -0,0 +1,20 @@
+import statistics
+import numpy as np
+from edgetpu.detection.engine import DetectionEngine
+
+# Path to frozen detection graph. This is the actual model that is used for the object detection.
+PATH_TO_CKPT = '/frozen_inference_graph.pb'
+
+# Load the edgetpu engine and labels
+engine = DetectionEngine(PATH_TO_CKPT)
+
+frame = np.zeros((300,300,3), np.uint8)
+flattened_frame = np.expand_dims(frame, axis=0).flatten()
+
+detection_times = []
+
+for x in range(0, 1000):
+    objects = engine.DetectWithInputTensor(flattened_frame, threshold=0.1, top_k=3)
+    detection_times.append(engine.get_inference_time())
+
+print("Average inference time: " + str(statistics.mean(detection_times)))

config/config.yml

@@ -3,6 +3,8 @@ web_port: 5000
 mqtt:
   host: mqtt.server.com
   topic_prefix: frigate
+#  user: username # Optional -- Uncomment for use
+#  password: password # Optional -- Uncomment for use
 
 cameras:
   back:
@@ -13,37 +15,51 @@ cameras:
       # values that begin with a "$" will be replaced with environment variable
       password: $RTSP_PASSWORD
       path: /cam/realmonitor?channel=1&subtype=2
+
+    ################
+    ## Optional mask. Must be the same dimensions as your video feed.
+    ## The mask works by looking at the bottom center of the bounding box for the detected
+    ## person in the image. If that pixel in the mask is a black pixel, it ignores it as a
+    ## false positive. In my mask, the grass and driveway visible from my backdoor camera 
+    ## are white. The garage doors, sky, and trees (anywhere it would be impossible for a 
+    ## person to stand) are black.
+    ################
+    # mask: back-mask.bmp
+
+    ################
+    # Allows you to limit the framerate within frigate for cameras that do not support
+    # custom framerates. A value of 1 tells frigate to look at every frame, 2 every 2nd frame, 
+    # 3 every 3rd frame, etc.
+    ################
+    take_frame: 1
+
+    ################
+    # Optional hardware acceleration parameters for ffmpeg. If your hardware supports it, it can
+    # greatly reduce the CPU power used to decode the video stream. You will need to determine which
+    # parameters work for your specific hardware. These may work for those with Intel hardware that
+    # supports QuickSync.
+    ################
+    # ffmpeg_hwaccel_args:
+    #   - -hwaccel
+    #   - vaapi
+    #   - -hwaccel_device
+    #   - /dev/dri/renderD128
+    #   - -hwaccel_output_format
+    #   - yuv420p
+    
     regions:
       - size: 350
         x_offset: 0
         y_offset: 300
         min_person_area: 5000
+        threshold: 0.5
       - size: 400
         x_offset: 350
         y_offset: 250
         min_person_area: 2000
+        threshold: 0.5
       - size: 400
         x_offset: 750
         y_offset: 250
         min_person_area: 2000
-  back2:
-    rtsp:
-      user: viewer
-      host: 10.0.10.10
-      port: 554
-      # values that begin with a "$" will be replaced with environment variable
-      password: $RTSP_PASSWORD
-      path: /cam/realmonitor?channel=1&subtype=2
-    regions:
-      - size: 350
-        x_offset: 0
-        y_offset: 300
-        min_person_area: 5000
-      - size: 400
-        x_offset: 350
-        y_offset: 250
-        min_person_area: 2000
-      - size: 400
-        x_offset: 750
-        y_offset: 250
-        min_person_area: 2000
+        threshold: 0.5

detect_objects.py

@@ -25,9 +25,18 @@ def main():
     # connect to mqtt and setup last will
     def on_connect(client, userdata, flags, rc):
         print("On connect called")
+        if rc != 0:
+            if rc == 3:
+                print ("MQTT Server unavailable")
+            elif rc == 4:
+                print ("MQTT Bad username or password")
+            elif rc == 5:
+                print ("MQTT Not authorized")
+            else:
+                print ("Unable to connect to MQTT: Connection refused. Error code: " + str(rc))
         # publish a message to signal that the service is running
         client.publish(MQTT_TOPIC_PREFIX+'/available', 'online', retain=True)
-    client = mqtt.Client()
+    client = mqtt.Client(client_id="frigate")
     client.on_connect = on_connect
     client.will_set(MQTT_TOPIC_PREFIX+'/available', payload='offline', qos=1, retain=True)
     if not MQTT_USER is None:
@@ -87,4 +96,4 @@ def main():
     camera.join()
 
 if __name__ == '__main__':
-    main()
+    main()

frigate/object_detection.py

@@ -38,7 +38,9 @@ class PreppedQueueProcessor(threading.Thread):
             frame = self.prepped_frame_queue.get()
 
             # Actual detection.
-            objects = self.engine.DetectWithInputTensor(frame['frame'], threshold=0.5, top_k=3)
+            objects = self.engine.DetectWithInputTensor(frame['frame'], threshold=frame['region_threshold'], top_k=3)
+            # print(self.engine.get_inference_time())
+
             # parse and pass detected objects back to the camera
             parsed_objects = []
             for obj in objects:
@@ -59,7 +61,7 @@ class PreppedQueueProcessor(threading.Thread):
 class FramePrepper(threading.Thread):
     def __init__(self, camera_name, shared_frame, frame_time, frame_ready, 
         frame_lock,
-        region_size, region_x_offset, region_y_offset,
+        region_size, region_x_offset, region_y_offset, region_threshold,
         prepped_frame_queue):
 
         threading.Thread.__init__(self)
@@ -71,6 +73,7 @@ class FramePrepper(threading.Thread):
         self.region_size = region_size
         self.region_x_offset = region_x_offset
         self.region_y_offset = region_y_offset
+        self.region_threshold = region_threshold
         self.prepped_frame_queue = prepped_frame_queue
 
     def run(self):
@@ -88,13 +91,11 @@ class FramePrepper(threading.Thread):
                 cropped_frame = self.shared_frame[self.region_y_offset:self.region_y_offset+self.region_size, self.region_x_offset:self.region_x_offset+self.region_size].copy()
                 frame_time = self.frame_time.value
             
-            # convert to RGB
-            cropped_frame_rgb = cv2.cvtColor(cropped_frame, cv2.COLOR_BGR2RGB)
             # Resize to 300x300 if needed
-            if cropped_frame_rgb.shape != (300, 300, 3):
-                cropped_frame_rgb = cv2.resize(cropped_frame_rgb, dsize=(300, 300), interpolation=cv2.INTER_LINEAR)
+            if cropped_frame.shape != (300, 300, 3):
+                cropped_frame = cv2.resize(cropped_frame, dsize=(300, 300), interpolation=cv2.INTER_LINEAR)
             # Expand dimensions since the model expects images to have shape: [1, 300, 300, 3]
-            frame_expanded = np.expand_dims(cropped_frame_rgb, axis=0)
+            frame_expanded = np.expand_dims(cropped_frame, axis=0)
 
             # add the frame to the queue
             if not self.prepped_frame_queue.full():
@@ -103,6 +104,7 @@ class FramePrepper(threading.Thread):
                     'frame_time': frame_time,
                     'frame': frame_expanded.flatten().copy(),
                     'region_size': self.region_size,
+                    'region_threshold': self.region_threshold,
                     'region_x_offset': self.region_x_offset,
                     'region_y_offset': self.region_y_offset
                 })

frigate/objects.py

@@ -2,7 +2,7 @@ import time
 import datetime
 import threading
 import cv2
-from object_detection.utils import visualization_utils as vis_util
+from . util import draw_box_with_label
 
 class ObjectCleaner(threading.Thread):
     def __init__(self, objects_parsed, detected_objects):
@@ -80,17 +80,9 @@ class BestPersonFrame(threading.Thread):
             
             if not self.best_person is None and self.best_person['frame_time'] in recent_frames:
                 best_frame = recent_frames[self.best_person['frame_time']]
-                best_frame = cv2.cvtColor(best_frame, cv2.COLOR_BGR2RGB)
-                # draw the bounding box on the frame
-                vis_util.draw_bounding_box_on_image_array(best_frame,
-                    self.best_person['ymin'],
-                    self.best_person['xmin'],
-                    self.best_person['ymax'],
-                    self.best_person['xmax'],
-                    color='red',
-                    thickness=2,
-                    display_str_list=["{}: {}%".format(self.best_person['name'],int(self.best_person['score']*100))],
-                    use_normalized_coordinates=False)
 
-                # convert back to BGR
+                label = "{}: {}% {}".format(self.best_person['name'],int(self.best_person['score']*100),int(self.best_person['area']))
+                draw_box_with_label(best_frame, self.best_person['xmin'], self.best_person['ymin'], 
+                    self.best_person['xmax'], self.best_person['ymax'], label)
+                
                 self.best_frame = cv2.cvtColor(best_frame, cv2.COLOR_RGB2BGR)

frigate/util.py

@@ -1,5 +1,26 @@
 import numpy as np
+import cv2
 
 # convert shared memory array into numpy array
 def tonumpyarray(mp_arr):
-    return np.frombuffer(mp_arr.get_obj(), dtype=np.uint8)
+    return np.frombuffer(mp_arr.get_obj(), dtype=np.uint8)
+
+def draw_box_with_label(frame, x_min, y_min, x_max, y_max, label):
+    color = (255,0,0)
+    cv2.rectangle(frame, (x_min, y_min), 
+        (x_max, y_max), 
+        color, 2)
+    font_scale = 0.5
+    font = cv2.FONT_HERSHEY_SIMPLEX
+    # get the width and height of the text box
+    size = cv2.getTextSize(label, font, fontScale=font_scale, thickness=2)
+    text_width = size[0][0]
+    text_height = size[0][1]
+    line_height = text_height + size[1]
+    # set the text start position
+    text_offset_x = x_min
+    text_offset_y = 0 if y_min < line_height else y_min - (line_height+8)
+    # make the coords of the box with a small padding of two pixels
+    textbox_coords = ((text_offset_x, text_offset_y), (text_offset_x + text_width + 2, text_offset_y + line_height))
+    cv2.rectangle(frame, textbox_coords[0], textbox_coords[1], color, cv2.FILLED)
+    cv2.putText(frame, label, (text_offset_x, text_offset_y + line_height - 3), font, fontScale=font_scale, color=(0, 0, 0), thickness=2)

frigate/video.py

@@ -5,59 +5,13 @@ import cv2
 import threading
 import ctypes
 import multiprocessing as mp
-from object_detection.utils import visualization_utils as vis_util
-from . util import tonumpyarray
+import subprocess as sp
+import numpy as np
+from . util import tonumpyarray, draw_box_with_label
 from . object_detection import FramePrepper
 from . objects import ObjectCleaner, BestPersonFrame
 from . mqtt import MqttObjectPublisher
 
-# fetch the frames as fast a possible and store current frame in a shared memory array
-def fetch_frames(shared_arr, shared_frame_time, frame_lock, frame_ready, frame_shape, rtsp_url):
-    # convert shared memory array into numpy and shape into image array
-    arr = tonumpyarray(shared_arr).reshape(frame_shape)
-
-    # start the video capture
-    video = cv2.VideoCapture()
-    video.open(rtsp_url)
-    # keep the buffer small so we minimize old data
-    video.set(cv2.CAP_PROP_BUFFERSIZE,1)
-
-    bad_frame_counter = 0
-    while True:
-        # check if the video stream is still open, and reopen if needed
-        if not video.isOpened():
-            success = video.open(rtsp_url)
-            if not success:
-                time.sleep(1)
-                continue
-        # grab the frame, but dont decode it yet
-        ret = video.grab()
-        # snapshot the time the frame was grabbed
-        frame_time = datetime.datetime.now()
-        if ret:
-            # go ahead and decode the current frame
-            ret, frame = video.retrieve()
-            if ret:
-                # Lock access and update frame
-                with frame_lock:
-                    arr[:] = frame
-                    shared_frame_time.value = frame_time.timestamp()
-                # Notify with the condition that a new frame is ready
-                with frame_ready:
-                    frame_ready.notify_all()
-                bad_frame_counter = 0
-            else:
-                print("Unable to decode frame")
-                bad_frame_counter += 1
-        else:
-            print("Unable to grab a frame")
-            bad_frame_counter += 1
-        
-        if bad_frame_counter > 100:
-            video.release()
-    
-    video.release()
-
 # Stores 2 seconds worth of frames when motion is detected so they can be used for other threads
 class FrameTracker(threading.Thread):
     def __init__(self, shared_frame, frame_time, frame_ready, frame_lock, recent_frames):
@@ -108,6 +62,57 @@ def get_rtsp_url(rtsp_config):
         rtsp_config['password'], rtsp_config['host'], rtsp_config['port'],
         rtsp_config['path'])
 
+class CameraWatchdog(threading.Thread):
+    def __init__(self, camera):
+        threading.Thread.__init__(self)
+        self.camera = camera
+
+    def run(self):
+
+        while True:
+            # wait a bit before checking
+            time.sleep(10)
+
+            if (datetime.datetime.now().timestamp() - self.camera.frame_time.value) > 2:
+                print("last frame is more than 2 seconds old, restarting camera capture...")
+                self.camera.start_or_restart_capture()
+                time.sleep(5)
+
+# Thread to read the stdout of the ffmpeg process and update the current frame
+class CameraCapture(threading.Thread):
+    def __init__(self, camera):
+        threading.Thread.__init__(self)
+        self.camera = camera
+
+    def run(self):
+        frame_num = 0
+        while True:
+            if self.camera.ffmpeg_process.poll() != None:
+                print("ffmpeg process is not running. exiting capture thread...")
+                break
+
+            raw_image = self.camera.ffmpeg_process.stdout.read(self.camera.frame_size)
+
+            if len(raw_image) == 0:
+                print("ffmpeg didnt return a frame. something is wrong. exiting capture thread...")
+                break
+
+            frame_num += 1
+            if (frame_num % self.camera.take_frame) != 0:
+                continue
+
+            with self.camera.frame_lock:
+                self.camera.frame_time.value = datetime.datetime.now().timestamp()
+                
+                self.camera.current_frame[:] = (
+                    np
+                    .frombuffer(raw_image, np.uint8)
+                    .reshape(self.camera.frame_shape)
+                )
+            # Notify with the condition that a new frame is ready
+            with self.camera.frame_ready:
+                self.camera.frame_ready.notify_all()
+
 class Camera:
     def __init__(self, name, config, prepped_frame_queue, mqtt_client, mqtt_prefix):
         self.name = name
@@ -115,17 +120,18 @@ class Camera:
         self.detected_objects = []
         self.recent_frames = {}
         self.rtsp_url = get_rtsp_url(self.config['rtsp'])
+        self.take_frame = self.config.get('take_frame', 1)
+        self.ffmpeg_hwaccel_args = self.config.get('ffmpeg_hwaccel_args', [])
         self.regions = self.config['regions']
         self.frame_shape = get_frame_shape(self.rtsp_url)
+        self.frame_size = self.frame_shape[0] * self.frame_shape[1] * self.frame_shape[2]
         self.mqtt_client = mqtt_client
         self.mqtt_topic_prefix = '{}/{}'.format(mqtt_prefix, self.name)
 
-        # compute the flattened array length from the shape of the frame
-        flat_array_length = self.frame_shape[0] * self.frame_shape[1] * self.frame_shape[2]
-        # create shared array for storing the full frame image data
-        self.shared_frame_array = mp.Array(ctypes.c_uint8, flat_array_length)
+        # create a numpy array for the current frame in initialize to zeros
+        self.current_frame = np.zeros(self.frame_shape, np.uint8)
         # create shared value for storing the frame_time
-        self.shared_frame_time = mp.Value('d', 0.0)
+        self.frame_time = mp.Value('d', 0.0)
         # Lock to control access to the frame
         self.frame_lock = mp.Lock()
         # Condition for notifying that a new frame is ready
@@ -133,29 +139,30 @@ class Camera:
         # Condition for notifying that objects were parsed
         self.objects_parsed = mp.Condition()
 
-        # shape current frame so it can be treated as a numpy image
-        self.shared_frame_np = tonumpyarray(self.shared_frame_array).reshape(self.frame_shape)
-
-        # create the process to capture frames from the RTSP stream and store in a shared array
-        self.capture_process = mp.Process(target=fetch_frames, args=(self.shared_frame_array, 
-            self.shared_frame_time, self.frame_lock, self.frame_ready, self.frame_shape, self.rtsp_url))
-        self.capture_process.daemon = True
+        self.ffmpeg_process = None
+        self.capture_thread = None
 
         # for each region, create a separate thread to resize the region and prep for detection
         self.detection_prep_threads = []
         for region in self.config['regions']:
+            # set a default threshold of 0.5 if not defined
+            if not 'threshold' in region:
+                region['threshold'] = 0.5
+            if not isinstance(region['threshold'], float):
+                print('Threshold is not a float. Setting to 0.5 default.')
+                region['threshold'] = 0.5
             self.detection_prep_threads.append(FramePrepper(
                 self.name,
-                self.shared_frame_np,
-                self.shared_frame_time,
+                self.current_frame,
+                self.frame_time,
                 self.frame_ready,
                 self.frame_lock,
-                region['size'], region['x_offset'], region['y_offset'],
+                region['size'], region['x_offset'], region['y_offset'], region['threshold'],
                 prepped_frame_queue
             ))
         
         # start a thread to store recent motion frames for processing
-        self.frame_tracker = FrameTracker(self.shared_frame_np, self.shared_frame_time, 
+        self.frame_tracker = FrameTracker(self.current_frame, self.frame_time, 
             self.frame_ready, self.frame_lock, self.recent_frames)
         self.frame_tracker.start()
 
@@ -170,26 +177,90 @@ class Camera:
         # start a thread to publish object scores (currently only person)
         mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self.objects_parsed, self.detected_objects)
         mqtt_publisher.start()
+
+        # create a watchdog thread for capture process
+        self.watchdog = CameraWatchdog(self)
+
+        # load in the mask for person detection
+        if 'mask' in self.config:
+            self.mask = cv2.imread("/config/{}".format(self.config['mask']), cv2.IMREAD_GRAYSCALE)
+        else:
+            self.mask = None
+
+        if self.mask is None:
+            self.mask = np.zeros((self.frame_shape[0], self.frame_shape[1], 1), np.uint8)
+            self.mask[:] = 255
+
+
+    def start_or_restart_capture(self):
+        if not self.ffmpeg_process is None:
+            print("Killing the existing ffmpeg process...")
+            self.ffmpeg_process.kill()
+            self.ffmpeg_process.wait()
+            print("Waiting for the capture thread to exit...")
+            self.capture_thread.join()
+            self.ffmpeg_process = None
+            self.capture_thread = None
+            
+        # create the process to capture frames from the RTSP stream and store in a shared array
+        print("Creating a new ffmpeg process...")
+        self.start_ffmpeg()
+        
+        print("Creating a new capture thread...")
+        self.capture_thread = CameraCapture(self)
+        print("Starting a new capture thread...")
+        self.capture_thread.start()
+    
+    def start_ffmpeg(self):
+        ffmpeg_global_args = [
+            '-hide_banner', '-loglevel', 'panic'
+        ]
+        ffmpeg_input_args = [
+            '-avoid_negative_ts', 'make_zero', 
+            '-fflags', 'nobuffer',
+            '-flags', 'low_delay',
+            '-strict', 'experimental',
+            '-fflags', '+genpts', 
+            '-rtsp_transport', 'tcp', 
+            '-stimeout', '5000000', 
+            '-use_wallclock_as_timestamps', '1'
+        ]
+
+        ffmpeg_cmd = (['ffmpeg'] +
+            ffmpeg_global_args +
+            self.ffmpeg_hwaccel_args +
+            ffmpeg_input_args +
+            ['-i', self.rtsp_url,
+            '-f', 'rawvideo',
+            '-pix_fmt', 'rgb24',
+            'pipe:'])
+
+        print(" ".join(ffmpeg_cmd))
+        
+        self.ffmpeg_process = sp.Popen(ffmpeg_cmd, stdout = sp.PIPE, bufsize=self.frame_size)
     
     def start(self):
-        self.capture_process.start()
+        self.start_or_restart_capture()
         # start the object detection prep threads
         for detection_prep_thread in self.detection_prep_threads:
             detection_prep_thread.start()
+        self.watchdog.start()
     
     def join(self):
-        self.capture_process.join()
+        self.capture_thread.join()
     
     def get_capture_pid(self):
-        return self.capture_process.pid
+        return self.ffmpeg_process.pid
     
     def add_objects(self, objects):
         if len(objects) == 0:
             return
 
         for obj in objects:
+            # Store object area to use in bounding box labels
+            obj['area'] = (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin'])
+
             if obj['name'] == 'person':
-                person_area = (obj['xmax']-obj['xmin'])*(obj['ymax']-obj['ymin'])
                 # find the matching region
                 region = None
                 for r in self.regions:
@@ -204,7 +275,16 @@ class Camera:
                 
                 # if the min person area is larger than the
                 # detected person, don't add it to detected objects
-                if region and region['min_person_area'] > person_area:
+                if region and 'min_person_area' in region and region['min_person_area'] > obj['area']:
+                    continue
+            
+                # compute the coordinates of the person and make sure
+                # the location isnt outside the bounds of the image (can happen from rounding)
+                y_location = min(int(obj['ymax']), len(self.mask)-1)
+                x_location = min(int((obj['xmax']-obj['xmin'])/2.0)+obj['xmin'], len(self.mask[0])-1)
+
+                # if the person is in a masked location, continue
+                if self.mask[y_location][x_location] == [0]:
                     continue
 
             self.detected_objects.append(obj)
@@ -220,21 +300,12 @@ class Camera:
         detected_objects = self.detected_objects.copy()
         # lock and make a copy of the current frame
         with self.frame_lock:
-            frame = self.shared_frame_np.copy()
+            frame = self.current_frame.copy()
 
-        # convert to RGB for drawing
-        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
         # draw the bounding boxes on the screen
         for obj in detected_objects:
-            vis_util.draw_bounding_box_on_image_array(frame,
-                obj['ymin'],
-                obj['xmin'],
-                obj['ymax'],
-                obj['xmax'],
-                color='red',
-                thickness=2,
-                display_str_list=["{}: {}%".format(obj['name'],int(obj['score']*100))],
-                use_normalized_coordinates=False)
+            label = "{}: {}% {}".format(obj['name'],int(obj['score']*100),int(obj['area']))
+            draw_box_with_label(frame, obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'], label)
 
         for region in self.regions:
             color = (255,255,255)
@@ -242,11 +313,11 @@ class Camera:
                 (region['x_offset']+region['size'], region['y_offset']+region['size']), 
                 color, 2)
 
-        # convert back to BGR
+        # convert to BGR
         frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
 
         return frame
 
 
     
-        
+        

scripts/install_edgetpu_api.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -e
+
+CPU_ARCH=$(uname -m)
+OS_VERSION=$(uname -v)
+
+echo "CPU_ARCH ${CPU_ARCH}"
+echo "OS_VERSION ${OS_VERSION}"
+
+if [[ "${CPU_ARCH}" == "x86_64" ]]; then
+  echo "Recognized as Linux on x86_64."
+  LIBEDGETPU_SUFFIX=x86_64
+  HOST_GNU_TYPE=x86_64-linux-gnu
+elif [[ "${CPU_ARCH}" == "armv7l" ]]; then
+  echo "Recognized as Linux on ARM32 platform."
+  LIBEDGETPU_SUFFIX=arm32
+  HOST_GNU_TYPE=arm-linux-gnueabihf
+elif [[ "${CPU_ARCH}" == "aarch64" ]]; then
+  echo "Recognized as generic ARM64 platform."
+  LIBEDGETPU_SUFFIX=arm64
+  HOST_GNU_TYPE=aarch64-linux-gnu
+fi
+
+if [[ -z "${HOST_GNU_TYPE}" ]]; then
+  echo "Your platform is not supported."
+  exit 1
+fi
+
+echo "Using maximum operating frequency."
+LIBEDGETPU_SRC="libedgetpu/libedgetpu_${LIBEDGETPU_SUFFIX}.so"
+LIBEDGETPU_DST="/usr/lib/${HOST_GNU_TYPE}/libedgetpu.so.1.0"
+
+# Runtime library.
+echo "Installing Edge TPU runtime library [${LIBEDGETPU_DST}]..."
+if [[ -f "${LIBEDGETPU_DST}" ]]; then
+  echo "File already exists. Replacing it..."
+  rm -f "${LIBEDGETPU_DST}"
+fi
+
+cp -p "${LIBEDGETPU_SRC}" "${LIBEDGETPU_DST}"
+ldconfig
+echo "Done."
+
+# Python API.
+WHEEL=$(ls edgetpu-*-py3-none-any.whl 2>/dev/null)
+if [[ $? == 0 ]]; then
+  echo "Installing Edge TPU Python API..."
+  python3 -m pip install --no-deps "${WHEEL}"
+  echo "Done."
+fi

scripts/install_odroid_repo.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+apt-key adv --keyserver keyserver.ubuntu.com --recv-keys D986B59D
+
+echo "deb http://deb.odroid.in/5422-s bionic main" > /etc/apt/sources.list.d/odroid.list