chris/frigate
1
0
Fork 0
forked from Github/frigate
Code Pull Requests Packages Activity

Autotracking optimizations (#7109)

Browse Source 
* much improved motion estimation and tracking

* docs updates

* move ptz specific mp values to ptz_metrics dict

* only check if moving at frame time

* pass full dict instead of individual values
This commit is contained in:
Josh Hawkins
2023-07-11 06:23:20 -05:00
committed by GitHub
parent a94346e17e
commit 7a2d09dc35
8 changed files with 188 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
frigate/ptz/autotrack.py
View File

@@ -2,6 +2,7 @@
import copy
import logging
import math
import queue
import threading
import time
@@ -14,33 +15,48 @@ from norfair.camera_motion import MotionEstimator, TranslationTransformationGett
from frigate.config import CameraConfig, FrigateConfig
from frigate.ptz.onvif import OnvifController
from frigate.types import CameraMetricsTypes
from frigate.types import PTZMetricsTypes
from frigate.util.image import SharedMemoryFrameManager, intersection_over_union
logger = logging.getLogger(__name__)
def ptz_moving_at_frame_time(frame_time, ptz_start_time, ptz_stop_time):
# Determine if the PTZ was in motion at the set frame time
# for non ptz/autotracking cameras, this will always return False
# ptz_start_time is initialized to 0 on startup and only changes
# when autotracking movements are made
# the offset "primes" the motion estimator with a few frames before movement
offset = 0.5
return (ptz_start_time != 0.0 and frame_time >= ptz_start_time - offset) and (
ptz_stop_time == 0.0 or (ptz_start_time - offset <= frame_time <= ptz_stop_time)
)
class PtzMotionEstimator:
def __init__(self, config: CameraConfig, ptz_stopped) -> None:
def __init__(self, config: CameraConfig, ptz_metrics: PTZMetricsTypes) -> None:
self.frame_manager = SharedMemoryFrameManager()
# homography is nice (zooming) but slow, translation is pan/tilt only but fast.
self.norfair_motion_estimator = MotionEstimator(
transformations_getter=TranslationTransformationGetter(),
min_distance=30,
max_points=500,
max_points=900,
)
self.camera_config = config
self.coord_transformations = None
self.ptz_stopped = ptz_stopped
self.ptz_metrics = ptz_metrics
self.ptz_start_time = self.ptz_metrics["ptz_start_time"]
self.ptz_stop_time = self.ptz_metrics["ptz_stop_time"]
logger.debug(f"Motion estimator init for cam: {config.name}")
def motion_estimator(self, detections, frame_time, camera_name):
if (
self.camera_config.onvif.autotracking.enabled
and not self.ptz_stopped.is_set()
if ptz_moving_at_frame_time(
frame_time, self.ptz_start_time.value, self.ptz_stop_time.value
):
logger.debug(
f"Motion estimator running for {camera_name} - frame time: {frame_time}"
f"Motion estimator running for {camera_name} - frame time: {frame_time}, {self.ptz_start_time.value}, {self.ptz_stop_time.value}"
)
frame_id = f"{camera_name}{frame_time}"
@@ -74,9 +90,7 @@ class PtzMotionEstimator:
f"Motion estimator transformation: {self.coord_transformations.rel_to_abs((0,0))}"
)
return self.coord_transformations
return None
return self.coord_transformations
class PtzAutoTrackerThread(threading.Thread):
@@ -84,12 +98,12 @@ class PtzAutoTrackerThread(threading.Thread):
self,
config: FrigateConfig,
onvif: OnvifController,
camera_metrics: dict[str, CameraMetricsTypes],
ptz_metrics: dict[str, PTZMetricsTypes],
stop_event: MpEvent,
) -> None:
threading.Thread.__init__(self)
self.name = "ptz_autotracker"
self.ptz_autotracker = PtzAutoTracker(config, onvif, camera_metrics)
self.ptz_autotracker = PtzAutoTracker(config, onvif, ptz_metrics)
self.stop_event = stop_event
self.config = config
@@ -112,11 +126,11 @@ class PtzAutoTracker:
self,
config: FrigateConfig,
onvif: OnvifController,
camera_metrics: CameraMetricsTypes,
ptz_metrics: PTZMetricsTypes,
) -> None:
self.config = config
self.onvif = onvif
self.camera_metrics = camera_metrics
self.ptz_metrics = ptz_metrics
self.tracked_object: dict[str, object] = {}
self.tracked_object_previous: dict[str, object] = {}
self.object_types = {}
@@ -146,17 +160,13 @@ class PtzAutoTracker:
if not self.onvif._init_onvif(camera_name):
logger.warning(f"Unable to initialize onvif for {camera_name}")
cam.onvif.autotracking.enabled = False
self.camera_metrics[camera_name][
"ptz_autotracker_enabled"
].value = False
self.ptz_metrics[camera_name]["ptz_autotracker_enabled"].value = False
return
if not self.onvif.cams[camera_name]["relative_fov_supported"]:
cam.onvif.autotracking.enabled = False
self.camera_metrics[camera_name][
"ptz_autotracker_enabled"
].value = False
self.ptz_metrics[camera_name]["ptz_autotracker_enabled"].value = False
logger.warning(
f"Disabling autotracking for {camera_name}: FOV relative movement not supported"
)
@@ -198,16 +208,10 @@ class PtzAutoTracker:
move_data = self.move_queues[camera].get()
pan, tilt = move_data
# check if ptz is moving
self.onvif.get_camera_status(camera)
# Wait until the camera finishes moving
self.camera_metrics[camera]["ptz_stopped"].wait()
self.onvif._move_relative(camera, pan, tilt, 1)
# Wait until the camera finishes moving
while not self.camera_metrics[camera]["ptz_stopped"].is_set():
while not self.ptz_metrics[camera]["ptz_stopped"].is_set():
# check if ptz is moving
self.onvif.get_camera_status(camera)
time.sleep(1 / (self.config.cameras[camera].detect.fps / 2))
@@ -237,10 +241,7 @@ class PtzAutoTracker:
def autotrack_object(self, camera, obj):
camera_config = self.config.cameras[camera]
if (
camera_config.onvif.autotracking.enabled
and self.camera_metrics[camera]["ptz_stopped"].is_set()
):
if camera_config.onvif.autotracking.enabled:
# either this is a brand new object that's on our camera, has our label, entered the zone, is not a false positive,
# and is not initially motionless - or one we're already tracking, which assumes all those things are already true
if (
@@ -259,7 +260,11 @@ class PtzAutoTracker:
)
self.tracked_object[camera] = obj
self.tracked_object_previous[camera] = copy.deepcopy(obj)
self._autotrack_move_ptz(camera, obj)
# only enqueue another move if the camera isn't moving
if self.ptz_metrics[camera]["ptz_stopped"].is_set():
self.ptz_metrics[camera]["ptz_stopped"].clear()
logger.debug("Autotrack: New object, moving ptz")
self._autotrack_move_ptz(camera, obj)
return
@@ -271,28 +276,57 @@ class PtzAutoTracker:
and obj.obj_data["frame_time"]
!= self.tracked_object_previous[camera].obj_data["frame_time"]
):
# don't move the ptz if we're relatively close to the existing box
# should we use iou or euclidean distance or both?
# distance = math.sqrt((obj.obj_data["centroid"][0] - camera_width/2)**2 + (obj.obj_data["centroid"][1] - obj.camera_height/2)**2)
# if distance <= (self.camera_width * .15) or distance <= (self.camera_height * .15)
if (
intersection_over_union(
self.tracked_object_previous[camera].obj_data["box"],
obj.obj_data["box"],
)
> 0.5
):
# Don't move ptz if Euclidean distance from object to center of frame is
# less than 15% of the of the larger dimension (width or height) of the frame,
# multiplied by a scaling factor for object size.
# Adjusting this percentage slightly lower will effectively cause the camera to move
# more often to keep the object in the center. Raising the percentage will cause less
# movement and will be more flexible with objects not quite being centered.
# TODO: there's probably a better way to approach this
distance = math.sqrt(
(obj.obj_data["centroid"][0] - camera_config.detect.width / 2) ** 2
+ (obj.obj_data["centroid"][1] - camera_config.detect.height / 2)
** 2
)
obj_width = obj.obj_data["box"][2] - obj.obj_data["box"][0]
obj_height = obj.obj_data["box"][3] - obj.obj_data["box"][1]
max_obj = max(obj_width, obj_height)
max_frame = max(camera_config.detect.width, camera_config.detect.height)
# larger objects should lower the threshold, smaller objects should raise it
scaling_factor = 1 - (max_obj / max_frame)
distance_threshold = 0.15 * (max_frame) * scaling_factor
iou = intersection_over_union(
self.tracked_object_previous[camera].obj_data["box"],
obj.obj_data["box"],
)
logger.debug(
f"Distance: {distance}, threshold: {distance_threshold}, iou: {iou}"
)
if (distance < distance_threshold or iou > 0.5) and self.ptz_metrics[
camera
]["ptz_stopped"].is_set():
logger.debug(
f"Autotrack: Existing object (do NOT move ptz): {obj.obj_data['id']} {obj.obj_data['box']} {obj.obj_data['frame_time']}"
)
self.tracked_object_previous[camera] = copy.deepcopy(obj)
return
logger.debug(
f"Autotrack: Existing object (move ptz): {obj.obj_data['id']} {obj.obj_data['box']} {obj.obj_data['frame_time']}"
f"Autotrack: Existing object (need to move ptz): {obj.obj_data['id']} {obj.obj_data['box']} {obj.obj_data['frame_time']}"
)
self.tracked_object_previous[camera] = copy.deepcopy(obj)
self._autotrack_move_ptz(camera, obj)
# only enqueue another move if the camera isn't moving
if self.ptz_metrics[camera]["ptz_stopped"].is_set():
self.ptz_metrics[camera]["ptz_stopped"].clear()
logger.debug("Autotrack: Existing object, moving ptz")
self._autotrack_move_ptz(camera, obj)
return
@@ -320,7 +354,11 @@ class PtzAutoTracker:
)
self.tracked_object[camera] = obj
self.tracked_object_previous[camera] = copy.deepcopy(obj)
self._autotrack_move_ptz(camera, obj)
# only enqueue another move if the camera isn't moving
if self.ptz_metrics[camera]["ptz_stopped"].is_set():
self.ptz_metrics[camera]["ptz_stopped"].clear()
logger.debug("Autotrack: Reacquired object, moving ptz")
self._autotrack_move_ptz(camera, obj)
return
@@ -334,7 +372,6 @@ class PtzAutoTracker:
f"Autotrack: End object: {obj.obj_data['id']} {obj.obj_data['box']}"
)
self.tracked_object[camera] = None
self.onvif.get_camera_status(camera)
def camera_maintenance(self, camera):
# calls get_camera_status to check/update ptz movement
@@ -344,7 +381,7 @@ class PtzAutoTracker:
if not self.autotracker_init[camera]:
self._autotracker_setup(self.config.cameras[camera], camera)
# regularly update camera status
if not self.camera_metrics[camera]["ptz_stopped"].is_set():
if not self.ptz_metrics[camera]["ptz_stopped"].is_set():
self.onvif.get_camera_status(camera)
# return to preset if tracking is over
@@ -359,7 +396,7 @@ class PtzAutoTracker:
)
and autotracker_config.return_preset
):
self.camera_metrics[camera]["ptz_stopped"].wait()
self.ptz_metrics[camera]["ptz_stopped"].wait()
logger.debug(
f"Autotrack: Time is {time.time()}, returning to preset: {autotracker_config.return_preset}"
)

38
frigate/ptz/onvif.py
View File

@@ -1,5 +1,6 @@
"""Configure and control camera via onvif."""
import datetime
import logging
import site
from enum import Enum
@@ -8,7 +9,7 @@ import numpy
from onvif import ONVIFCamera, ONVIFError
from frigate.config import FrigateConfig
from frigate.types import CameraMetricsTypes
from frigate.types import PTZMetricsTypes
logger = logging.getLogger(__name__)
@@ -29,10 +30,10 @@ class OnvifCommandEnum(str, Enum):
class OnvifController:
def __init__(
self, config: FrigateConfig, camera_metrics: dict[str, CameraMetricsTypes]
self, config: FrigateConfig, ptz_metrics: dict[str, PTZMetricsTypes]
) -> None:
self.cams: dict[str, ONVIFCamera] = {}
self.camera_metrics = camera_metrics
self.ptz_metrics = ptz_metrics
for cam_name, cam in config.cameras.items():
if not cam.enabled:
@@ -207,7 +208,6 @@ class OnvifController:
return
logger.debug(f"{camera_name} called RelativeMove: pan: {pan} tilt: {tilt}")
self.get_camera_status(camera_name)
if self.cams[camera_name]["active"]:
logger.warning(
@@ -216,7 +216,12 @@ class OnvifController:
return
self.cams[camera_name]["active"] = True
self.camera_metrics[camera_name]["ptz_stopped"].clear()
self.ptz_metrics[camera_name]["ptz_stopped"].clear()
logger.debug(f"PTZ start time: {datetime.datetime.now().timestamp()}")
self.ptz_metrics[camera_name][
"ptz_start_time"
].value = datetime.datetime.now().timestamp()
self.ptz_metrics[camera_name]["ptz_stop_time"].value = 0
onvif: ONVIFCamera = self.cams[camera_name]["onvif"]
move_request = self.cams[camera_name]["relative_move_request"]
@@ -261,7 +266,7 @@ class OnvifController:
return
self.cams[camera_name]["active"] = True
self.camera_metrics[camera_name]["ptz_stopped"].clear()
self.ptz_metrics[camera_name]["ptz_stopped"].clear()
move_request = self.cams[camera_name]["move_request"]
onvif: ONVIFCamera = self.cams[camera_name]["onvif"]
preset_token = self.cams[camera_name]["presets"][preset]
@@ -271,7 +276,7 @@ class OnvifController:
"PresetToken": preset_token,
}
)
self.camera_metrics[camera_name]["ptz_stopped"].set()
self.ptz_metrics[camera_name]["ptz_stopped"].set()
self.cams[camera_name]["active"] = False
def _zoom(self, camera_name: str, command: OnvifCommandEnum) -> None:
@@ -345,10 +350,25 @@ class OnvifController:
if status.MoveStatus.PanTilt == "IDLE" and status.MoveStatus.Zoom == "IDLE":
self.cams[camera_name]["active"] = False
self.camera_metrics[camera_name]["ptz_stopped"].set()
if not self.ptz_metrics[camera_name]["ptz_stopped"].is_set():
self.ptz_metrics[camera_name]["ptz_stopped"].set()
logger.debug(f"PTZ stop time: {datetime.datetime.now().timestamp()}")
self.ptz_metrics[camera_name][
"ptz_stop_time"
].value = datetime.datetime.now().timestamp()
else:
self.cams[camera_name]["active"] = True
self.camera_metrics[camera_name]["ptz_stopped"].clear()
if self.ptz_metrics[camera_name]["ptz_stopped"].is_set():
self.ptz_metrics[camera_name]["ptz_stopped"].clear()
logger.debug(f"PTZ start time: {datetime.datetime.now().timestamp()}")
self.ptz_metrics[camera_name][
"ptz_start_time"
].value = datetime.datetime.now().timestamp()
self.ptz_metrics[camera_name]["ptz_stop_time"].value = 0
return {
"pan": status.Position.PanTilt.x,