Compare commits

..

17 Commits

Author SHA1 Message Date
Blake Blackshear
5c01720567 Update README.md 2019-12-12 08:08:32 -06:00
Blake Blackshear
262f45c8bc Update sponsorship option 2019-12-11 06:35:17 -06:00
tubalainen
22bb17b2fd Filename updated but not the reference 2019-12-09 06:01:27 -06:00
Blake Blackshear
3a3afe14bf change the ffmpeg config for global defaults and overrides 2019-12-08 16:03:23 -06:00
Blake Blackshear
01f058a482 clarify optional properties 2019-12-08 16:03:23 -06:00
Blake Blackshear
d899ef158e fix datestamp positioning 2019-12-08 16:03:23 -06:00
Blake Blackshear
39d64f7ba7 add health check and handle bad camera names 2019-12-08 16:03:23 -06:00
Blake Blackshear
f148eb5a7b add some comments for regions 2019-12-08 16:03:23 -06:00
Blake Blackshear
297e2f1c0c allow mqtt client_id to be set for multi frigate setups 2019-12-08 16:03:23 -06:00
Blake Blackshear
e818744d81 print the frame time on the image 2019-12-08 08:55:54 -06:00
Blake Blackshear
ceedfae993 add max person area 2019-12-08 07:17:18 -06:00
Blake Blackshear
e13563770d allow full customization of input 2019-12-08 07:06:52 -06:00
Blake Blackshear
a659019d1a move config example 2019-12-08 07:06:52 -06:00
blakeblackshear
ba71927d53 allow setting custom output params and setting the log level for ffmpeg 2019-08-25 08:54:19 -05:00
blakeblackshear
04fed31eac increase watchdog timeout to 10 seconds 2019-08-25 08:54:19 -05:00
blakeblackshear
ebaa8fac01 tweak input params and gracefully kill ffmpeg 2019-08-25 08:54:19 -05:00
blakeblackshear
2ec45cd1b6 send the best person frame over mqtt for faster updates in homeassistant 2019-08-25 08:54:19 -05:00
10 changed files with 322 additions and 129 deletions

2
.github/FUNDING.yml vendored
View File

@@ -1 +1 @@
ko_fi: blakeblackshear
github: blakeblackshear

2
.gitignore vendored
View File

@@ -1,2 +1,4 @@
*.pyc
debug
.vscode
config/config.yml

View File

@@ -1,9 +1,7 @@
<a href='https://ko-fi.com/P5P7XGO9' target='_blank'><img height='36' style='border:0px;height:36px;' src='https://az743702.vo.msecnd.net/cdn/kofi4.png?v=2' border='0' alt='Buy Me a Coffee at ko-fi.com' /></a>
# Frigate - Realtime Object Detection for RTSP Cameras
# Frigate - Realtime Object Detection for IP Cameras
**Note:** This version requires the use of a [Google Coral USB Accelerator](https://coral.withgoogle.com/products/accelerator/)
Uses OpenCV and Tensorflow to perform realtime object detection locally for RTSP cameras. Designed for integration with HomeAssistant or others via MQTT.
Uses OpenCV and Tensorflow to perform realtime object detection locally for IP cameras. Designed for integration with HomeAssistant or others via MQTT.
- Leverages multiprocessing and threads heavily with an emphasis on realtime over processing every frame
- Allows you to define specific regions (squares) in the image to look for objects
@@ -32,8 +30,9 @@ docker run --rm \
--privileged \
-v /dev/bus/usb:/dev/bus/usb \
-v <path_to_config_dir>:/config:ro \
-v /etc/localtime:/etc/localtime:ro \
-p 5000:5000 \
-e RTSP_PASSWORD='password' \
-e FRIGATE_RTSP_PASSWORD='password' \
frigate:latest
```
@@ -46,14 +45,15 @@ Example docker-compose:
image: frigate:latest
volumes:
- /dev/bus/usb:/dev/bus/usb
- /etc/localtime:/etc/localtime:ro
- <path_to_config>:/config
ports:
- "5000:5000"
environment:
RTSP_PASSWORD: "password"
FRIGATE_RTSP_PASSWORD: "password"
```
A `config.yml` file must exist in the `config` directory. See example [here](config/config.yml).
A `config.yml` file must exist in the `config` directory. See example [here](config/config.example.yml) and device specific info can be found [here](docs/DEVICES.md).
Access the mjpeg stream at `http://localhost:5000/<camera_name>` and the best person snapshot at `http://localhost:5000/<camera_name>/best_person.jpg`
@@ -61,8 +61,8 @@ Access the mjpeg stream at `http://localhost:5000/<camera_name>` and the best pe
```
camera:
- name: Camera Last Person
platform: generic
still_image_url: http://<ip>:5000/<camera_name>/best_person.jpg
platform: mqtt
topic: frigate/<camera_name>/snapshot
binary_sensor:
- name: Camera Person
@@ -71,10 +71,30 @@ binary_sensor:
value_template: '{{ value_json.person }}'
device_class: motion
availability_topic: "frigate/available"
automation:
- alias: Alert me if a person is detected while armed away
trigger:
platform: state
entity_id: binary_sensor.camera_person
from: 'off'
to: 'on'
condition:
- condition: state
entity_id: alarm_control_panel.home_alarm
state: armed_away
action:
- service: notify.user_telegram
data:
message: "A person was detected."
data:
photo:
- url: http://<ip>:5000/<camera_name>/best_person.jpg
caption: A person was detected.
```
## Tips
- Lower the framerate of the RTSP feed on the camera to reduce the CPU usage for capturing the feed
- Lower the framerate of the video feed on the camera to reduce the CPU usage for capturing the feed
## Future improvements
- [x] Remove motion detection for now

110
config/config.example.yml Normal file
View File

@@ -0,0 +1,110 @@
web_port: 5000
mqtt:
host: mqtt.server.com
topic_prefix: frigate
# client_id: frigate # Optional -- set to override default client id of 'frigate' if running multiple instances
# user: username # Optional -- Uncomment for use
# password: password # Optional -- Uncomment for use
#################
# Default ffmpeg args. Optional and can be overwritten per camera.
# Should work with most RTSP cameras that send h264 video
# Built from the properties below with:
# "ffmpeg" + global_args + input_args + "-i" + input + output_args
#################
# ffmpeg:
# global_args:
# - -hide_banner
# - -loglevel
# - panic
# hwaccel_args: []
# input_args:
# - -avoid_negative_ts
# - make_zero
# - -fflags
# - nobuffer
# - -flags
# - low_delay
# - -strict
# - experimental
# - -fflags
# - +genpts+discardcorrupt
# - -vsync
# - drop
# - -rtsp_transport
# - tcp
# - -stimeout
# - '5000000'
# - -use_wallclock_as_timestamps
# - '1'
# output_args:
# - -vf
# - mpdecimate
# - -f
# - rawvideo
# - -pix_fmt
# - rgb24
cameras:
back:
ffmpeg:
################
# Source passed to ffmpeg after the -i parameter. Supports anything compatible with OpenCV and FFmpeg.
# Environment variables that begin with 'FRIGATE_' may be referenced in {}
################
input: rtsp://viewer:{FRIGATE_RTSP_PASSWORD}@10.0.10.10:554/cam/realmonitor?channel=1&subtype=2
#################
# These values will override default values for just this camera
#################
# global_args: []
# hwaccel_args: []
# input_args: []
# output_args: []
################
## Optional mask. Must be the same dimensions as your video feed.
## The mask works by looking at the bottom center of the bounding box for the detected
## person in the image. If that pixel in the mask is a black pixel, it ignores it as a
## false positive. In my mask, the grass and driveway visible from my backdoor camera
## are white. The garage doors, sky, and trees (anywhere it would be impossible for a
## person to stand) are black.
################
# mask: back-mask.bmp
################
# Allows you to limit the framerate within frigate for cameras that do not support
# custom framerates. A value of 1 tells frigate to look at every frame, 2 every 2nd frame,
# 3 every 3rd frame, etc.
################
take_frame: 1
################
# size: size of the region in pixels
# x_offset/y_offset: position of the upper left corner of your region (top left of image is 0,0)
# min_person_area (optional): minimum width*height of the bounding box for the detected person
# max_person_area (optional): maximum width*height of the bounding box for the detected person
# threshold (optional): The minimum decimal percentage (50% hit = 0.5) for the confidence from tensorflow
# Tips: All regions are resized to 300x300 before detection because the model is trained on that size.
# Resizing regions takes CPU power. Ideally, all regions should be as close to 300x300 as possible.
# Defining a region that goes outside the bounds of the image will result in errors.
################
regions:
- size: 350
x_offset: 0
y_offset: 300
min_person_area: 5000
max_person_area: 100000
threshold: 0.5
- size: 400
x_offset: 350
y_offset: 250
min_person_area: 2000
max_person_area: 100000
threshold: 0.5
- size: 400
x_offset: 750
y_offset: 250
min_person_area: 2000
max_person_area: 100000
threshold: 0.5

View File

@@ -1,65 +0,0 @@
web_port: 5000
mqtt:
host: mqtt.server.com
topic_prefix: frigate
# user: username # Optional -- Uncomment for use
# password: password # Optional -- Uncomment for use
cameras:
back:
rtsp:
user: viewer
host: 10.0.10.10
port: 554
# values that begin with a "$" will be replaced with environment variable
password: $RTSP_PASSWORD
path: /cam/realmonitor?channel=1&subtype=2
################
## Optional mask. Must be the same dimensions as your video feed.
## The mask works by looking at the bottom center of the bounding box for the detected
## person in the image. If that pixel in the mask is a black pixel, it ignores it as a
## false positive. In my mask, the grass and driveway visible from my backdoor camera
## are white. The garage doors, sky, and trees (anywhere it would be impossible for a
## person to stand) are black.
################
# mask: back-mask.bmp
################
# Allows you to limit the framerate within frigate for cameras that do not support
# custom framerates. A value of 1 tells frigate to look at every frame, 2 every 2nd frame,
# 3 every 3rd frame, etc.
################
take_frame: 1
################
# Optional hardware acceleration parameters for ffmpeg. If your hardware supports it, it can
# greatly reduce the CPU power used to decode the video stream. You will need to determine which
# parameters work for your specific hardware. These may work for those with Intel hardware that
# supports QuickSync.
################
# ffmpeg_hwaccel_args:
# - -hwaccel
# - vaapi
# - -hwaccel_device
# - /dev/dri/renderD128
# - -hwaccel_output_format
# - yuv420p
regions:
- size: 350
x_offset: 0
y_offset: 300
min_person_area: 5000
threshold: 0.5
- size: 400
x_offset: 350
y_offset: 250
min_person_area: 2000
threshold: 0.5
- size: 400
x_offset: 750
y_offset: 250
min_person_area: 2000
threshold: 0.5

View File

@@ -17,6 +17,30 @@ MQTT_PORT = CONFIG.get('mqtt', {}).get('port', 1883)
MQTT_TOPIC_PREFIX = CONFIG.get('mqtt', {}).get('topic_prefix', 'frigate')
MQTT_USER = CONFIG.get('mqtt', {}).get('user')
MQTT_PASS = CONFIG.get('mqtt', {}).get('password')
MQTT_CLIENT_ID = CONFIG.get('mqtt', {}).get('client_id', 'frigate')
# Set the default FFmpeg config
FFMPEG_CONFIG = CONFIG.get('ffmpeg', {})
FFMPEG_DEFAULT_CONFIG = {
'global_args': FFMPEG_CONFIG.get('global_args',
['-hide_banner','-loglevel','panic']),
'hwaccel_args': FFMPEG_CONFIG.get('hwaccel_args',
[]),
'input_args': FFMPEG_CONFIG.get('input_args',
['-avoid_negative_ts', 'make_zero',
'-fflags', 'nobuffer',
'-flags', 'low_delay',
'-strict', 'experimental',
'-fflags', '+genpts+discardcorrupt',
'-vsync', 'drop',
'-rtsp_transport', 'tcp',
'-stimeout', '5000000',
'-use_wallclock_as_timestamps', '1']),
'output_args': FFMPEG_CONFIG.get('output_args',
['-vf', 'mpdecimate',
'-f', 'rawvideo',
'-pix_fmt', 'rgb24'])
}
WEB_PORT = CONFIG.get('web_port', 5000)
DEBUG = (CONFIG.get('debug', '0') == '1')
@@ -36,7 +60,7 @@ def main():
print ("Unable to connect to MQTT: Connection refused. Error code: " + str(rc))
# publish a message to signal that the service is running
client.publish(MQTT_TOPIC_PREFIX+'/available', 'online', retain=True)
client = mqtt.Client(client_id="frigate")
client = mqtt.Client(client_id=MQTT_CLIENT_ID)
client.on_connect = on_connect
client.will_set(MQTT_TOPIC_PREFIX+'/available', payload='offline', qos=1, retain=True)
if not MQTT_USER is None:
@@ -50,7 +74,7 @@ def main():
cameras = {}
for name, config in CONFIG['cameras'].items():
cameras[name] = Camera(name, config, prepped_frame_queue, client, MQTT_TOPIC_PREFIX)
cameras[name] = Camera(name, FFMPEG_DEFAULT_CONFIG, config, prepped_frame_queue, client, MQTT_TOPIC_PREFIX)
prepped_queue_processor = PreppedQueueProcessor(
cameras,
@@ -65,21 +89,32 @@ def main():
# create a flask app that encodes frames a mjpeg on demand
app = Flask(__name__)
@app.route('/')
def ishealthy():
# return a healh
return "Frigate is running. Alive and healthy!"
@app.route('/<camera_name>/best_person.jpg')
def best_person(camera_name):
best_person_frame = cameras[camera_name].get_best_person()
if best_person_frame is None:
best_person_frame = np.zeros((720,1280,3), np.uint8)
ret, jpg = cv2.imencode('.jpg', best_person_frame)
response = make_response(jpg.tobytes())
response.headers['Content-Type'] = 'image/jpg'
return response
if camera_name in cameras:
best_person_frame = cameras[camera_name].get_best_person()
if best_person_frame is None:
best_person_frame = np.zeros((720,1280,3), np.uint8)
ret, jpg = cv2.imencode('.jpg', best_person_frame)
response = make_response(jpg.tobytes())
response.headers['Content-Type'] = 'image/jpg'
return response
else:
return f'Camera named {camera_name} not found', 404
@app.route('/<camera_name>')
def mjpeg_feed(camera_name):
# return a multipart response
return Response(imagestream(camera_name),
mimetype='multipart/x-mixed-replace; boundary=frame')
if camera_name in cameras:
# return a multipart response
return Response(imagestream(camera_name),
mimetype='multipart/x-mixed-replace; boundary=frame')
else:
return f'Camera named {camera_name} not found', 404
def imagestream(camera_name):
while True:

74
docs/DEVICES.md Normal file
View File

@@ -0,0 +1,74 @@
# Configuration Examples
### Default (most RTSP cameras)
This is the default ffmpeg command and should work with most RTSP cameras that send h264 video
```yaml
ffmpeg:
global_args:
- -hide_banner
- -loglevel
- panic
hwaccel_args: []
input_args:
- -avoid_negative_ts
- make_zero
- -fflags
- nobuffer
- -flags
- low_delay
- -strict
- experimental
- -fflags
- +genpts+discardcorrupt
- -vsync
- drop
- -rtsp_transport
- tcp
- -stimeout
- '5000000'
- -use_wallclock_as_timestamps
- '1'
output_args:
- -vf
- mpdecimate
- -f
- rawvideo
- -pix_fmt
- rgb24
```
### RTMP Cameras
The input parameters need to be adjusted for RTMP cameras
```yaml
ffmpeg:
input_args:
- -avoid_negative_ts
- make_zero
- -fflags
- nobuffer
- -flags
- low_delay
- -strict
- experimental
- -fflags
- +genpts+discardcorrupt
- -vsync
- drop
- -use_wallclock_as_timestamps
- '1'
```
### Hardware Acceleration
Intel Quicksync
```yaml
ffmpeg:
hwaccel_args:
- -hwaccel
- vaapi
- -hwaccel_device
- /dev/dri/renderD128
- -hwaccel_output_format
- yuv420p
```

View File

@@ -1,13 +1,15 @@
import json
import cv2
import threading
class MqttObjectPublisher(threading.Thread):
def __init__(self, client, topic_prefix, objects_parsed, detected_objects):
def __init__(self, client, topic_prefix, objects_parsed, detected_objects, best_person_frame):
threading.Thread.__init__(self)
self.client = client
self.topic_prefix = topic_prefix
self.objects_parsed = objects_parsed
self._detected_objects = detected_objects
self.best_person_frame = best_person_frame
def run(self):
last_sent_payload = ""
@@ -30,4 +32,10 @@ class MqttObjectPublisher(threading.Thread):
new_payload = json.dumps(payload, sort_keys=True)
if new_payload != last_sent_payload:
last_sent_payload = new_payload
self.client.publish(self.topic_prefix+'/objects', new_payload, retain=False)
self.client.publish(self.topic_prefix+'/objects', new_payload, retain=False)
# send the snapshot over mqtt as well
if not self.best_person_frame.best_frame is None:
ret, jpg = cv2.imencode('.jpg', self.best_person_frame.best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(self.topic_prefix+'/snapshot', jpg_bytes, retain=True)

View File

@@ -85,4 +85,8 @@ class BestPersonFrame(threading.Thread):
draw_box_with_label(best_frame, self.best_person['xmin'], self.best_person['ymin'],
self.best_person['xmax'], self.best_person['ymax'], label)
# print a timestamp
time_to_show = datetime.datetime.fromtimestamp(self.best_person['frame_time']).strftime("%m/%d/%Y %H:%M:%S")
cv2.putText(best_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
self.best_frame = cv2.cvtColor(best_frame, cv2.COLOR_RGB2BGR)

View File

@@ -46,21 +46,18 @@ class FrameTracker(threading.Thread):
if (now - k) > 2:
del self.recent_frames[k]
def get_frame_shape(rtsp_url):
def get_frame_shape(source):
# capture a single frame and check the frame shape so the correct array
# size can be allocated in memory
video = cv2.VideoCapture(rtsp_url)
video = cv2.VideoCapture(source)
ret, frame = video.read()
frame_shape = frame.shape
video.release()
return frame_shape
def get_rtsp_url(rtsp_config):
if (rtsp_config['password'].startswith('$')):
rtsp_config['password'] = os.getenv(rtsp_config['password'][1:])
return 'rtsp://{}:{}@{}:{}{}'.format(rtsp_config['user'],
rtsp_config['password'], rtsp_config['host'], rtsp_config['port'],
rtsp_config['path'])
def get_ffmpeg_input(ffmpeg_input):
frigate_vars = {k: v for k, v in os.environ.items() if k.startswith('FRIGATE_')}
return ffmpeg_input.format(**frigate_vars)
class CameraWatchdog(threading.Thread):
def __init__(self, camera):
@@ -73,8 +70,8 @@ class CameraWatchdog(threading.Thread):
# wait a bit before checking
time.sleep(10)
if (datetime.datetime.now().timestamp() - self.camera.frame_time.value) > 2:
print("last frame is more than 2 seconds old, restarting camera capture...")
if (datetime.datetime.now().timestamp() - self.camera.frame_time.value) > 10:
print("last frame is more than 10 seconds old, restarting camera capture...")
self.camera.start_or_restart_capture()
time.sleep(5)
@@ -114,16 +111,22 @@ class CameraCapture(threading.Thread):
self.camera.frame_ready.notify_all()
class Camera:
def __init__(self, name, config, prepped_frame_queue, mqtt_client, mqtt_prefix):
def __init__(self, name, ffmpeg_config, config, prepped_frame_queue, mqtt_client, mqtt_prefix):
self.name = name
self.config = config
self.detected_objects = []
self.recent_frames = {}
self.rtsp_url = get_rtsp_url(self.config['rtsp'])
self.ffmpeg = config.get('ffmpeg', {})
self.ffmpeg_input = get_ffmpeg_input(self.ffmpeg['input'])
self.ffmpeg_global_args = self.ffmpeg.get('global_args', ffmpeg_config['global_args'])
self.ffmpeg_hwaccel_args = self.ffmpeg.get('hwaccel_args', ffmpeg_config['hwaccel_args'])
self.ffmpeg_input_args = self.ffmpeg.get('input_args', ffmpeg_config['input_args'])
self.ffmpeg_output_args = self.ffmpeg.get('output_args', ffmpeg_config['output_args'])
self.take_frame = self.config.get('take_frame', 1)
self.ffmpeg_hwaccel_args = self.config.get('ffmpeg_hwaccel_args', [])
self.regions = self.config['regions']
self.frame_shape = get_frame_shape(self.rtsp_url)
self.frame_shape = get_frame_shape(self.ffmpeg_input)
self.frame_size = self.frame_shape[0] * self.frame_shape[1] * self.frame_shape[2]
self.mqtt_client = mqtt_client
self.mqtt_topic_prefix = '{}/{}'.format(mqtt_prefix, self.name)
@@ -175,7 +178,7 @@ class Camera:
self.object_cleaner.start()
# start a thread to publish object scores (currently only person)
mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self.objects_parsed, self.detected_objects)
mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self.objects_parsed, self.detected_objects, self.best_person_frame)
mqtt_publisher.start()
# create a watchdog thread for capture process
@@ -194,15 +197,22 @@ class Camera:
def start_or_restart_capture(self):
if not self.ffmpeg_process is None:
print("Killing the existing ffmpeg process...")
self.ffmpeg_process.kill()
self.ffmpeg_process.wait()
print("Terminating the existing ffmpeg process...")
self.ffmpeg_process.terminate()
try:
print("Waiting for ffmpeg to exit gracefully...")
self.ffmpeg_process.wait(timeout=30)
except sp.TimeoutExpired:
print("FFmpeg didnt exit. Force killing...")
self.ffmpeg_process.kill()
self.ffmpeg_process.wait()
print("Waiting for the capture thread to exit...")
self.capture_thread.join()
self.ffmpeg_process = None
self.capture_thread = None
# create the process to capture frames from the RTSP stream and store in a shared array
# create the process to capture frames from the input stream and store in a shared array
print("Creating a new ffmpeg process...")
self.start_ffmpeg()
@@ -212,28 +222,13 @@ class Camera:
self.capture_thread.start()
def start_ffmpeg(self):
ffmpeg_global_args = [
'-hide_banner', '-loglevel', 'panic'
]
ffmpeg_input_args = [
'-avoid_negative_ts', 'make_zero',
'-fflags', 'nobuffer',
'-flags', 'low_delay',
'-strict', 'experimental',
'-fflags', '+genpts',
'-rtsp_transport', 'tcp',
'-stimeout', '5000000',
'-use_wallclock_as_timestamps', '1'
]
ffmpeg_cmd = (['ffmpeg'] +
ffmpeg_global_args +
self.ffmpeg_global_args +
self.ffmpeg_hwaccel_args +
ffmpeg_input_args +
['-i', self.rtsp_url,
'-f', 'rawvideo',
'-pix_fmt', 'rgb24',
'pipe:'])
self.ffmpeg_input_args +
['-i', self.ffmpeg_input] +
self.ffmpeg_output_args +
['pipe:'])
print(" ".join(ffmpeg_cmd))
@@ -277,6 +272,11 @@ class Camera:
# detected person, don't add it to detected objects
if region and 'min_person_area' in region and region['min_person_area'] > obj['area']:
continue
# if the detected person is larger than the
# max person area, don't add it to detected objects
if region and 'max_person_area' in region and region['max_person_area'] < obj['area']:
continue
# compute the coordinates of the person and make sure
# the location isnt outside the bounds of the image (can happen from rounding)
@@ -301,6 +301,7 @@ class Camera:
# lock and make a copy of the current frame
with self.frame_lock:
frame = self.current_frame.copy()
frame_time = self.frame_time.value
# draw the bounding boxes on the screen
for obj in detected_objects:
@@ -312,6 +313,10 @@ class Camera:
cv2.rectangle(frame, (region['x_offset'], region['y_offset']),
(region['x_offset']+region['size'], region['y_offset']+region['size']),
color, 2)
# print a timestamp
time_to_show = datetime.datetime.fromtimestamp(frame_time).strftime("%m/%d/%Y %H:%M:%S")
cv2.putText(frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
# convert to BGR
frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)