Basic PTZ object autotracking functionality (#6913)

* Basic functionality

* Threaded motion estimator

* Revert "Threaded motion estimator"

This reverts commit 3171801607.

* Don't detect motion when ptz is moving

* fix motion logic

* fix mypy error

* Add threaded queue for movement for slower ptzs

* Move queues per camera

* Move autotracker start to app.py

* iou value for tracked object

* mqtt callback

* tracked object should be initially motionless

* only draw thicker box if autotracking is enabled

* Init if enabled when initially disabled in config

* Fix init

* Thread names

* Always use motion estimator

* docs

* clarify fov support

* remove size ratio

* use mp event instead of value for ptz status

* update autotrack at half fps

* fix merge conflict

* fix event type for mypy

* clean up

* Clean up

* remove unused code

* merge conflict fix

* docs: update link to object_detectors page

* Update docs/docs/configuration/autotracking.md

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>

* clarify wording

* pass actual instances directly

* default return preset

* fix type

* Error message when onvif init fails

* disable autotracking if onvif init fails

* disable autotracking if onvif init fails

* ptz module

* verify required_zones in config

* update util after dev merge

---------

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>

frigate/app.py

@@ -42,7 +42,8 @@ from frigate.object_detection import ObjectDetectProcess
 from frigate.object_processing import TrackedObjectProcessor
 from frigate.output import output_frames
 from frigate.plus import PlusApi
-from frigate.ptz import OnvifController
+from frigate.ptz.autotrack import PtzAutoTrackerThread
+from frigate.ptz.onvif import OnvifController
 from frigate.record.record import manage_recordings
 from frigate.stats import StatsEmitter, stats_init
 from frigate.storage import StorageMaintainer
@@ -134,6 +135,13 @@ class FrigateApp:
                     "i",
                     self.config.cameras[camera_name].motion.improve_contrast,
                 ),
+                "ptz_autotracker_enabled": mp.Value(  # type: ignore[typeddict-item]
+                    # issue https://github.com/python/typeshed/issues/8799
+                    # from mypy 0.981 onwards
+                    "i",
+                    self.config.cameras[camera_name].onvif.autotracking.enabled,
+                ),
+                "ptz_stopped": mp.Event(),
                 "motion_threshold": mp.Value(  # type: ignore[typeddict-item]
                     # issue https://github.com/python/typeshed/issues/8799
                     # from mypy 0.981 onwards
@@ -162,6 +170,7 @@ class FrigateApp:
                 "capture_process": None,
                 "process": None,
             }
+            self.camera_metrics[camera_name]["ptz_stopped"].set()
             self.feature_metrics[camera_name] = {
                 "audio_enabled": mp.Value(  # type: ignore[typeddict-item]
                     # issue https://github.com/python/typeshed/issues/8799
@@ -308,7 +317,7 @@ class FrigateApp:
         )
 
     def init_onvif(self) -> None:
-        self.onvif_controller = OnvifController(self.config)
+        self.onvif_controller = OnvifController(self.config, self.camera_metrics)
 
     def init_dispatcher(self) -> None:
         comms: list[Communicator] = []
@@ -362,6 +371,15 @@ class FrigateApp:
                 detector_config,
             )
 
+    def start_ptz_autotracker(self) -> None:
+        self.ptz_autotracker_thread = PtzAutoTrackerThread(
+            self.config,
+            self.onvif_controller,
+            self.camera_metrics,
+            self.stop_event,
+        )
+        self.ptz_autotracker_thread.start()
+
     def start_detected_frames_processor(self) -> None:
         self.detected_frames_processor = TrackedObjectProcessor(
             self.config,
@@ -371,6 +389,7 @@ class FrigateApp:
             self.event_processed_queue,
             self.video_output_queue,
             self.recordings_info_queue,
+            self.ptz_autotracker_thread,
             self.stop_event,
         )
         self.detected_frames_processor.start()
@@ -535,6 +554,7 @@ class FrigateApp:
             sys.exit(1)
         self.start_detectors()
         self.start_video_output_processor()
+        self.start_ptz_autotracker()
         self.start_detected_frames_processor()
         self.start_camera_processors()
         self.start_camera_capture_processes()
@@ -579,6 +599,7 @@ class FrigateApp:
 
         self.dispatcher.stop()
         self.detected_frames_processor.join()
+        self.ptz_autotracker_thread.join()
         self.event_processor.join()
         self.event_cleanup.join()
         self.stats_emitter.join()

frigate/comms/dispatcher.py

@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod
 from typing import Any, Callable
 
 from frigate.config import FrigateConfig
-from frigate.ptz import OnvifCommandEnum, OnvifController
+from frigate.ptz.onvif import OnvifCommandEnum, OnvifController
 from frigate.types import CameraMetricsTypes, FeatureMetricsTypes
 from frigate.util.services import restart_frigate
 
@@ -55,6 +55,7 @@ class Dispatcher:
             "audio": self._on_audio_command,
             "detect": self._on_detect_command,
             "improve_contrast": self._on_motion_improve_contrast_command,
+            "ptz_autotracker": self._on_ptz_autotracker_command,
             "motion": self._on_motion_command,
             "motion_contour_area": self._on_motion_contour_area_command,
             "motion_threshold": self._on_motion_threshold_command,
@@ -159,6 +160,25 @@ class Dispatcher:
 
         self.publish(f"{camera_name}/improve_contrast/state", payload, retain=True)
 
+    def _on_ptz_autotracker_command(self, camera_name: str, payload: str) -> None:
+        """Callback for ptz_autotracker topic."""
+        ptz_autotracker_settings = self.config.cameras[camera_name].onvif.autotracking
+
+        if payload == "ON":
+            if not self.camera_metrics[camera_name]["ptz_autotracker_enabled"].value:
+                logger.info(f"Turning on ptz autotracker for {camera_name}")
+                self.camera_metrics[camera_name]["ptz_autotracker_enabled"].value = True
+                ptz_autotracker_settings.enabled = True
+        elif payload == "OFF":
+            if self.camera_metrics[camera_name]["ptz_autotracker_enabled"].value:
+                logger.info(f"Turning off ptz autotracker for {camera_name}")
+                self.camera_metrics[camera_name][
+                    "ptz_autotracker_enabled"
+                ].value = False
+                ptz_autotracker_settings.enabled = False
+
+        self.publish(f"{camera_name}/ptz_autotracker/state", payload, retain=True)
+
     def _on_motion_contour_area_command(self, camera_name: str, payload: int) -> None:
         """Callback for motion contour topic."""
         try:

frigate/comms/mqtt.py

@@ -69,6 +69,11 @@ class MqttClient(Communicator):  # type: ignore[misc]
                 "ON" if camera.motion.improve_contrast else "OFF",  # type: ignore[union-attr]
                 retain=True,
             )
+            self.publish(
+                f"{camera_name}/ptz_autotracker/state",
+                "ON" if camera.onvif.autotracking.enabled else "OFF",
+                retain=True,
+            )
             self.publish(
                 f"{camera_name}/motion_threshold/state",
                 camera.motion.threshold,  # type: ignore[union-attr]
@@ -152,6 +157,7 @@ class MqttClient(Communicator):  # type: ignore[misc]
             "audio",
             "motion",
             "improve_contrast",
+            "ptz_autotracker",
             "motion_threshold",
             "motion_contour_area",
         ]

frigate/config.py

@@ -128,11 +128,31 @@ class MqttConfig(FrigateBaseModel):
         return v
 
 
+class PtzAutotrackConfig(FrigateBaseModel):
+    enabled: bool = Field(default=False, title="Enable PTZ object autotracking.")
+    track: List[str] = Field(default=DEFAULT_TRACKED_OBJECTS, title="Objects to track.")
+    required_zones: List[str] = Field(
+        default_factory=list,
+        title="List of required zones to be entered in order to begin autotracking.",
+    )
+    return_preset: str = Field(
+        default="home",
+        title="Name of camera preset to return to when object tracking is over.",
+    )
+    timeout: int = Field(
+        default=10, title="Seconds to delay before returning to preset."
+    )
+
+
 class OnvifConfig(FrigateBaseModel):
     host: str = Field(default="", title="Onvif Host")
     port: int = Field(default=8000, title="Onvif Port")
     user: Optional[str] = Field(title="Onvif Username")
     password: Optional[str] = Field(title="Onvif Password")
+    autotracking: PtzAutotrackConfig = Field(
+        default_factory=PtzAutotrackConfig,
+        title="PTZ auto tracking config.",
+    )
 
 
 class RetainModeEnum(str, Enum):
@@ -892,6 +912,17 @@ def verify_zone_objects_are_tracked(camera_config: CameraConfig) -> None:
                 )
 
 
+def verify_autotrack_zones(camera_config: CameraConfig) -> ValueError | None:
+    """Verify that required_zones are specified when autotracking is enabled."""
+    if (
+        camera_config.onvif.autotracking.enabled
+        and not camera_config.onvif.autotracking.required_zones
+    ):
+        raise ValueError(
+            f"Camera {camera_config.name} has autotracking enabled, required_zones must be set to at least one of the camera's zones."
+        )
+
+
 class FrigateConfig(FrigateBaseModel):
     mqtt: MqttConfig = Field(title="MQTT Configuration.")
     database: DatabaseConfig = Field(
@@ -1067,6 +1098,7 @@ class FrigateConfig(FrigateBaseModel):
             verify_recording_retention(camera_config)
             verify_recording_segments_setup_with_reasonable_time(camera_config)
             verify_zone_objects_are_tracked(camera_config)
+            verify_autotrack_zones(camera_config)
 
             if camera_config.rtmp.enabled:
                 logger.warning(

frigate/http.py

@@ -35,7 +35,7 @@ from frigate.events.external import ExternalEventProcessor
 from frigate.models import Event, Recordings, Timeline
 from frigate.object_processing import TrackedObject
 from frigate.plus import PlusApi
-from frigate.ptz import OnvifController
+from frigate.ptz.onvif import OnvifController
 from frigate.record.export import PlaybackFactorEnum, RecordingExporter
 from frigate.stats import stats_snapshot
 from frigate.storage import StorageMaintainer

frigate/object_processing.py

@@ -22,6 +22,7 @@ from frigate.config import (
 )
 from frigate.const import CLIPS_DIR
 from frigate.events.maintainer import EventTypeEnum
+from frigate.ptz.autotrack import PtzAutoTrackerThread
 from frigate.util.image import (
     SharedMemoryFrameManager,
     area,
@@ -143,6 +144,7 @@ class TrackedObject:
     def update(self, current_frame_time, obj_data):
         thumb_update = False
         significant_change = False
+        autotracker_update = False
         # if the object is not in the current frame, add a 0.0 to the score history
         if obj_data["frame_time"] != current_frame_time:
             self.score_history.append(0.0)
@@ -236,9 +238,15 @@ class TrackedObject:
             if self.obj_data["frame_time"] - self.previous["frame_time"] > 60:
                 significant_change = True
 
+            # update autotrack at half fps
+            if self.obj_data["frame_time"] - self.previous["frame_time"] > (
+                1 / (self.camera_config.detect.fps / 2)
+            ):
+                autotracker_update = True
+
         self.obj_data.update(obj_data)
         self.current_zones = current_zones
-        return (thumb_update, significant_change)
+        return (thumb_update, significant_change, autotracker_update)
 
     def to_dict(self, include_thumbnail: bool = False):
         (self.thumbnail_data["frame_time"] if self.thumbnail_data is not None else 0.0)
@@ -437,7 +445,11 @@ def zone_filtered(obj: TrackedObject, object_config):
 # Maintains the state of a camera
 class CameraState:
     def __init__(
-        self, name, config: FrigateConfig, frame_manager: SharedMemoryFrameManager
+        self,
+        name,
+        config: FrigateConfig,
+        frame_manager: SharedMemoryFrameManager,
+        ptz_autotracker_thread: PtzAutoTrackerThread,
     ):
         self.name = name
         self.config = config
@@ -455,6 +467,7 @@ class CameraState:
         self.regions = []
         self.previous_frame_id = None
         self.callbacks = defaultdict(list)
+        self.ptz_autotracker_thread = ptz_autotracker_thread
 
     def get_current_frame(self, draw_options={}):
         with self.current_frame_lock:
@@ -476,6 +489,21 @@ class CameraState:
                     thickness = 1
                     color = (255, 0, 0)
 
+                # draw thicker box around ptz autotracked object
+                if (
+                    self.camera_config.onvif.autotracking.enabled
+                    and self.ptz_autotracker_thread.ptz_autotracker.tracked_object[
+                        self.name
+                    ]
+                    is not None
+                    and obj["id"]
+                    == self.ptz_autotracker_thread.ptz_autotracker.tracked_object[
+                        self.name
+                    ].obj_data["id"]
+                ):
+                    thickness = 5
+                    color = self.config.model.colormap[obj["label"]]
+
                 # draw the bounding boxes on the frame
                 box = obj["box"]
                 draw_box_with_label(
@@ -589,10 +617,14 @@ class CameraState:
 
         for id in updated_ids:
             updated_obj = tracked_objects[id]
-            thumb_update, significant_update = updated_obj.update(
+            thumb_update, significant_update, autotracker_update = updated_obj.update(
                 frame_time, current_detections[id]
             )
 
+            if autotracker_update or significant_update:
+                for c in self.callbacks["autotrack"]:
+                    c(self.name, updated_obj, frame_time)
+
             if thumb_update:
                 # ensure this frame is stored in the cache
                 if (
@@ -733,6 +765,7 @@ class TrackedObjectProcessor(threading.Thread):
         event_processed_queue,
         video_output_queue,
         recordings_info_queue,
+        ptz_autotracker_thread,
         stop_event,
     ):
         threading.Thread.__init__(self)
@@ -748,6 +781,7 @@ class TrackedObjectProcessor(threading.Thread):
         self.camera_states: dict[str, CameraState] = {}
         self.frame_manager = SharedMemoryFrameManager()
         self.last_motion_detected: dict[str, float] = {}
+        self.ptz_autotracker_thread = ptz_autotracker_thread
 
         def start(camera, obj: TrackedObject, current_frame_time):
             self.event_queue.put(
@@ -774,6 +808,9 @@ class TrackedObjectProcessor(threading.Thread):
                 )
             )
 
+        def autotrack(camera, obj: TrackedObject, current_frame_time):
+            self.ptz_autotracker_thread.ptz_autotracker.autotrack_object(camera, obj)
+
         def end(camera, obj: TrackedObject, current_frame_time):
             # populate has_snapshot
             obj.has_snapshot = self.should_save_snapshot(camera, obj)
@@ -822,6 +859,7 @@ class TrackedObjectProcessor(threading.Thread):
                     "type": "end",
                 }
                 self.dispatcher.publish("events", json.dumps(message), retain=False)
+                self.ptz_autotracker_thread.ptz_autotracker.end_object(camera, obj)
 
             self.event_queue.put(
                 (
@@ -858,8 +896,11 @@ class TrackedObjectProcessor(threading.Thread):
             self.dispatcher.publish(f"{camera}/{object_name}", status, retain=False)
 
         for camera in self.config.cameras.keys():
-            camera_state = CameraState(camera, self.config, self.frame_manager)
+            camera_state = CameraState(
+                camera, self.config, self.frame_manager, self.ptz_autotracker_thread
+            )
             camera_state.on("start", start)
+            camera_state.on("autotrack", autotrack)
             camera_state.on("update", update)
             camera_state.on("end", end)
             camera_state.on("snapshot", snapshot)

frigate/ptz/autotrack.py

@@ -0,0 +1,362 @@
+"""Automatically pan, tilt, and zoom on detected objects via onvif."""
+
+import copy
+import logging
+import queue
+import threading
+import time
+from functools import partial
+from multiprocessing.synchronize import Event as MpEvent
+
+import cv2
+import numpy as np
+from norfair.camera_motion import MotionEstimator, TranslationTransformationGetter
+
+from frigate.config import CameraConfig, FrigateConfig
+from frigate.ptz.onvif import OnvifController
+from frigate.types import CameraMetricsTypes
+from frigate.util.image import SharedMemoryFrameManager, intersection_over_union
+
+logger = logging.getLogger(__name__)
+
+
+class PtzMotionEstimator:
+    def __init__(self, config: CameraConfig, ptz_stopped) -> None:
+        self.frame_manager = SharedMemoryFrameManager()
+        # homography is nice (zooming) but slow, translation is pan/tilt only but fast.
+        self.norfair_motion_estimator = MotionEstimator(
+            transformations_getter=TranslationTransformationGetter(),
+            min_distance=30,
+            max_points=500,
+        )
+        self.camera_config = config
+        self.coord_transformations = None
+        self.ptz_stopped = ptz_stopped
+        logger.debug(f"Motion estimator init for cam: {config.name}")
+
+    def motion_estimator(self, detections, frame_time, camera_name):
+        if (
+            self.camera_config.onvif.autotracking.enabled
+            and not self.ptz_stopped.is_set()
+        ):
+            logger.debug(
+                f"Motion estimator running for {camera_name} - frame time: {frame_time}"
+            )
+
+            frame_id = f"{camera_name}{frame_time}"
+            yuv_frame = self.frame_manager.get(
+                frame_id, self.camera_config.frame_shape_yuv
+            )
+
+            frame = cv2.cvtColor(yuv_frame, cv2.COLOR_YUV2GRAY_I420)
+
+            # mask out detections for better motion estimation
+            mask = np.ones(frame.shape[:2], frame.dtype)
+
+            detection_boxes = [x[2] for x in detections]
+            for detection in detection_boxes:
+                x1, y1, x2, y2 = detection
+                mask[y1:y2, x1:x2] = 0
+
+            # merge camera config motion mask with detections. Norfair function needs 0,1 mask
+            mask = np.bitwise_and(mask, self.camera_config.motion.mask).clip(max=1)
+
+            # Norfair estimator function needs color so it can convert it right back to gray
+            frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2BGRA)
+
+            self.coord_transformations = self.norfair_motion_estimator.update(
+                frame, mask
+            )
+
+            self.frame_manager.close(frame_id)
+
+            logger.debug(
+                f"Motion estimator transformation: {self.coord_transformations.rel_to_abs((0,0))}"
+            )
+
+            return self.coord_transformations
+
+        return None
+
+
+class PtzAutoTrackerThread(threading.Thread):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        onvif: OnvifController,
+        camera_metrics: dict[str, CameraMetricsTypes],
+        stop_event: MpEvent,
+    ) -> None:
+        threading.Thread.__init__(self)
+        self.name = "ptz_autotracker"
+        self.ptz_autotracker = PtzAutoTracker(config, onvif, camera_metrics)
+        self.stop_event = stop_event
+        self.config = config
+
+    def run(self):
+        while not self.stop_event.is_set():
+            for camera_name, cam in self.config.cameras.items():
+                if cam.onvif.autotracking.enabled:
+                    self.ptz_autotracker.camera_maintenance(camera_name)
+                    time.sleep(1)
+                else:
+                    # disabled dynamically by mqtt
+                    if self.ptz_autotracker.tracked_object.get(camera_name):
+                        self.ptz_autotracker.tracked_object[camera_name] = None
+                        self.ptz_autotracker.tracked_object_previous[camera_name] = None
+            time.sleep(0.1)
+        logger.info("Exiting autotracker...")
+
+
+class PtzAutoTracker:
+    def __init__(
+        self,
+        config: FrigateConfig,
+        onvif: OnvifController,
+        camera_metrics: CameraMetricsTypes,
+    ) -> None:
+        self.config = config
+        self.onvif = onvif
+        self.camera_metrics = camera_metrics
+        self.tracked_object: dict[str, object] = {}
+        self.tracked_object_previous: dict[str, object] = {}
+        self.object_types = {}
+        self.required_zones = {}
+        self.move_queues = {}
+        self.move_threads = {}
+        self.autotracker_init = {}
+
+        # if cam is set to autotrack, onvif should be set up
+        for camera_name, cam in self.config.cameras.items():
+            self.autotracker_init[camera_name] = False
+            if cam.onvif.autotracking.enabled:
+                self._autotracker_setup(cam, camera_name)
+
+    def _autotracker_setup(self, cam, camera_name):
+        logger.debug(f"Autotracker init for cam: {camera_name}")
+
+        self.object_types[camera_name] = cam.onvif.autotracking.track
+        self.required_zones[camera_name] = cam.onvif.autotracking.required_zones
+
+        self.tracked_object[camera_name] = None
+        self.tracked_object_previous[camera_name] = None
+
+        self.move_queues[camera_name] = queue.Queue()
+
+        if not self.onvif.cams[camera_name]["init"]:
+            if not self.onvif._init_onvif(camera_name):
+                logger.warning(f"Unable to initialize onvif for {camera_name}")
+                cam.onvif.autotracking.enabled = False
+                self.camera_metrics[camera_name][
+                    "ptz_autotracker_enabled"
+                ].value = False
+
+                return
+
+            if not self.onvif.cams[camera_name]["relative_fov_supported"]:
+                cam.onvif.autotracking.enabled = False
+                self.camera_metrics[camera_name][
+                    "ptz_autotracker_enabled"
+                ].value = False
+                logger.warning(
+                    f"Disabling autotracking for {camera_name}: FOV relative movement not supported"
+                )
+
+                return
+
+            # movement thread per camera
+            if not self.move_threads or not self.move_threads[camera_name]:
+                self.move_threads[camera_name] = threading.Thread(
+                    name=f"move_thread_{camera_name}",
+                    target=partial(self._process_move_queue, camera_name),
+                )
+                self.move_threads[camera_name].daemon = True
+                self.move_threads[camera_name].start()
+
+        self.autotracker_init[camera_name] = True
+
+    def _process_move_queue(self, camera):
+        while True:
+            try:
+                if self.move_queues[camera].qsize() > 1:
+                    # Accumulate values since last moved
+                    pan = 0
+                    tilt = 0
+
+                    while not self.move_queues[camera].empty():
+                        queued_pan, queued_tilt = self.move_queues[camera].queue[0]
+
+                        # If exceeding the movement range, keep it in the queue and move now
+                        if abs(pan + queued_pan) > 1.0 or abs(tilt + queued_tilt) > 1.0:
+                            logger.debug("Pan or tilt value exceeds 1.0")
+                            break
+
+                        queued_pan, queued_tilt = self.move_queues[camera].get()
+
+                        pan += queued_pan
+                        tilt += queued_tilt
+                else:
+                    move_data = self.move_queues[camera].get()
+                    pan, tilt = move_data
+
+                self.onvif._move_relative(camera, pan, tilt, 1)
+
+                # Wait until the camera finishes moving
+                self.camera_metrics[camera]["ptz_stopped"].wait()
+
+            except queue.Empty:
+                time.sleep(0.1)
+
+    def _enqueue_move(self, camera, pan, tilt):
+        move_data = (pan, tilt)
+        logger.debug(f"enqueue pan: {pan}, enqueue tilt: {tilt}")
+        self.move_queues[camera].put(move_data)
+
+    def _autotrack_move_ptz(self, camera, obj):
+        camera_config = self.config.cameras[camera]
+
+        # # frame width and height
+        camera_width = camera_config.frame_shape[1]
+        camera_height = camera_config.frame_shape[0]
+
+        # Normalize coordinates. top right of the fov is (1,1).
+        pan = 0.5 - (obj.obj_data["centroid"][0] / camera_width)
+        tilt = 0.5 - (obj.obj_data["centroid"][1] / camera_height)
+
+        # ideas: check object velocity for camera speed?
+        self._enqueue_move(camera, -pan, tilt)
+
+    def autotrack_object(self, camera, obj):
+        camera_config = self.config.cameras[camera]
+
+        # check if ptz is moving
+        self.onvif.get_camera_status(camera)
+
+        if camera_config.onvif.autotracking.enabled:
+            # either this is a brand new object that's on our camera, has our label, entered the zone, is not a false positive,
+            # and is not initially motionless - or one we're already tracking, which assumes all those things are already true
+            if (
+                # new object
+                self.tracked_object[camera] is None
+                and obj.camera == camera
+                and obj.obj_data["label"] in self.object_types[camera]
+                and set(obj.entered_zones) & set(self.required_zones[camera])
+                and not obj.previous["false_positive"]
+                and not obj.false_positive
+                and self.tracked_object_previous[camera] is None
+                and obj.obj_data["motionless_count"] == 0
+            ):
+                logger.debug(
+                    f"Autotrack: New object: {obj.obj_data['id']} {obj.obj_data['box']} {obj.obj_data['frame_time']}"
+                )
+                self.tracked_object[camera] = obj
+                self.tracked_object_previous[camera] = copy.deepcopy(obj)
+                self._autotrack_move_ptz(camera, obj)
+
+                return
+
+            if (
+                # already tracking an object
+                self.tracked_object[camera] is not None
+                and self.tracked_object_previous[camera] is not None
+                and obj.obj_data["id"] == self.tracked_object[camera].obj_data["id"]
+                and obj.obj_data["frame_time"]
+                != self.tracked_object_previous[camera].obj_data["frame_time"]
+            ):
+                # don't move the ptz if we're relatively close to the existing box
+                # should we use iou or euclidean distance or both?
+                # distance = math.sqrt((obj.obj_data["centroid"][0] - camera_width/2)**2 + (obj.obj_data["centroid"][1] - obj.camera_height/2)**2)
+                # if distance <= (self.camera_width * .15) or distance <= (self.camera_height * .15)
+                if (
+                    intersection_over_union(
+                        self.tracked_object_previous[camera].obj_data["box"],
+                        obj.obj_data["box"],
+                    )
+                    > 0.5
+                ):
+                    logger.debug(
+                        f"Autotrack: Existing object (do NOT move ptz): {obj.obj_data['id']} {obj.obj_data['box']} {obj.obj_data['frame_time']}"
+                    )
+                    self.tracked_object_previous[camera] = copy.deepcopy(obj)
+                    return
+
+                logger.debug(
+                    f"Autotrack: Existing object (move ptz): {obj.obj_data['id']} {obj.obj_data['box']} {obj.obj_data['frame_time']}"
+                )
+                self.tracked_object_previous[camera] = copy.deepcopy(obj)
+                self._autotrack_move_ptz(camera, obj)
+
+                return
+
+            if (
+                # The tracker lost an object, so let's check the previous object's region and compare it with the incoming object
+                # If it's within bounds, start tracking that object.
+                # Should we check region (maybe too broad) or expand the previous object's box a bit and check that?
+                self.tracked_object[camera] is None
+                and obj.camera == camera
+                and obj.obj_data["label"] in self.object_types[camera]
+                and not obj.previous["false_positive"]
+                and not obj.false_positive
+                and obj.obj_data["motionless_count"] == 0
+                and self.tracked_object_previous[camera] is not None
+            ):
+                if (
+                    intersection_over_union(
+                        self.tracked_object_previous[camera].obj_data["region"],
+                        obj.obj_data["box"],
+                    )
+                    < 0.2
+                ):
+                    logger.debug(
+                        f"Autotrack: Reacquired object: {obj.obj_data['id']} {obj.obj_data['box']} {obj.obj_data['frame_time']}"
+                    )
+                    self.tracked_object[camera] = obj
+                    self.tracked_object_previous[camera] = copy.deepcopy(obj)
+                    self._autotrack_move_ptz(camera, obj)
+
+                return
+
+    def end_object(self, camera, obj):
+        if self.config.cameras[camera].onvif.autotracking.enabled:
+            if (
+                self.tracked_object[camera] is not None
+                and obj.obj_data["id"] == self.tracked_object[camera].obj_data["id"]
+            ):
+                logger.debug(
+                    f"Autotrack: End object: {obj.obj_data['id']} {obj.obj_data['box']}"
+                )
+                self.tracked_object[camera] = None
+                self.onvif.get_camera_status(camera)
+
+    def camera_maintenance(self, camera):
+        # calls get_camera_status to check/update ptz movement
+        # returns camera to preset after timeout when tracking is over
+        autotracker_config = self.config.cameras[camera].onvif.autotracking
+
+        if not self.autotracker_init[camera]:
+            self._autotracker_setup(self.config.cameras[camera], camera)
+        # regularly update camera status
+        if not self.camera_metrics[camera]["ptz_stopped"].is_set():
+            self.onvif.get_camera_status(camera)
+
+        # return to preset if tracking is over
+        if (
+            self.tracked_object[camera] is None
+            and self.tracked_object_previous[camera] is not None
+            and (
+                # might want to use a different timestamp here?
+                time.time()
+                - self.tracked_object_previous[camera].obj_data["frame_time"]
+                > autotracker_config.timeout
+            )
+            and autotracker_config.return_preset
+        ):
+            self.camera_metrics[camera]["ptz_stopped"].wait()
+            logger.debug(
+                f"Autotrack: Time is {time.time()}, returning to preset: {autotracker_config.return_preset}"
+            )
+            self.onvif._move_to_preset(
+                camera,
+                autotracker_config.return_preset.lower(),
+            )
+            self.tracked_object_previous[camera] = None

frigate/ptz/onvif.py

@@ -4,9 +4,11 @@ import logging
 import site
 from enum import Enum
 
+import numpy
 from onvif import ONVIFCamera, ONVIFError
 
 from frigate.config import FrigateConfig
+from frigate.types import CameraMetricsTypes
 
 logger = logging.getLogger(__name__)
 
@@ -26,8 +28,11 @@ class OnvifCommandEnum(str, Enum):
 
 
 class OnvifController:
-    def __init__(self, config: FrigateConfig) -> None:
+    def __init__(
+        self, config: FrigateConfig, camera_metrics: dict[str, CameraMetricsTypes]
+    ) -> None:
         self.cams: dict[str, ONVIFCamera] = {}
+        self.camera_metrics = camera_metrics
 
         for cam_name, cam in config.cameras.items():
             if not cam.enabled:
@@ -68,12 +73,51 @@ class OnvifController:
         ptz = onvif.create_ptz_service()
         request = ptz.create_type("GetConfigurationOptions")
         request.ConfigurationToken = profile.PTZConfiguration.token
+        ptz_config = ptz.GetConfigurationOptions(request)
 
-        # setup moving request
+        fov_space_id = next(
+            (
+                i
+                for i, space in enumerate(
+                    ptz_config.Spaces.RelativePanTiltTranslationSpace
+                )
+                if "TranslationSpaceFov" in space["URI"]
+            ),
+            None,
+        )
+
+        # setup continuous moving request
         move_request = ptz.create_type("ContinuousMove")
         move_request.ProfileToken = profile.token
         self.cams[camera_name]["move_request"] = move_request
 
+        # setup relative moving request for autotracking
+        move_request = ptz.create_type("RelativeMove")
+        move_request.ProfileToken = profile.token
+        if move_request.Translation is None and fov_space_id is not None:
+            move_request.Translation = ptz.GetStatus(
+                {"ProfileToken": profile.token}
+            ).Position
+            move_request.Translation.PanTilt.space = ptz_config["Spaces"][
+                "RelativePanTiltTranslationSpace"
+            ][fov_space_id]["URI"]
+            move_request.Translation.Zoom.space = ptz_config["Spaces"][
+                "RelativeZoomTranslationSpace"
+            ][0]["URI"]
+        if move_request.Speed is None:
+            move_request.Speed = ptz.GetStatus({"ProfileToken": profile.token}).Position
+        self.cams[camera_name]["relative_move_request"] = move_request
+
+        # setup relative moving request for autotracking
+        move_request = ptz.create_type("AbsoluteMove")
+        move_request.ProfileToken = profile.token
+        self.cams[camera_name]["absolute_move_request"] = move_request
+
+        # status request for autotracking
+        status_request = ptz.create_type("GetStatus")
+        status_request.ProfileToken = profile.token
+        self.cams[camera_name]["status_request"] = status_request
+
         # setup existing presets
         try:
             presets: list[dict] = ptz.GetPresets({"ProfileToken": profile.token})
@@ -94,6 +138,20 @@ class OnvifController:
         if ptz_config.Spaces and ptz_config.Spaces.ContinuousZoomVelocitySpace:
             supported_features.append("zoom")
 
+        if ptz_config.Spaces and ptz_config.Spaces.RelativePanTiltTranslationSpace:
+            supported_features.append("pt-r")
+
+        if ptz_config.Spaces and ptz_config.Spaces.RelativeZoomTranslationSpace:
+            supported_features.append("zoom-r")
+
+        if fov_space_id is not None:
+            supported_features.append("pt-r-fov")
+            self.cams[camera_name][
+                "relative_fov_range"
+            ] = ptz_config.Spaces.RelativePanTiltTranslationSpace[fov_space_id]
+
+        self.cams[camera_name]["relative_fov_supported"] = fov_space_id is not None
+
         self.cams[camera_name]["features"] = supported_features
 
         self.cams[camera_name]["init"] = True
@@ -143,12 +201,74 @@ class OnvifController:
 
         onvif.get_service("ptz").ContinuousMove(move_request)
 
+    def _move_relative(self, camera_name: str, pan, tilt, speed) -> None:
+        if not self.cams[camera_name]["relative_fov_supported"]:
+            logger.error(f"{camera_name} does not support ONVIF RelativeMove (FOV).")
+            return
+
+        logger.debug(f"{camera_name} called RelativeMove: pan: {pan} tilt: {tilt}")
+        self.get_camera_status(camera_name)
+
+        if self.cams[camera_name]["active"]:
+            logger.warning(
+                f"{camera_name} is already performing an action, not moving..."
+            )
+            return
+
+        self.cams[camera_name]["active"] = True
+        self.camera_metrics[camera_name]["ptz_stopped"].clear()
+        onvif: ONVIFCamera = self.cams[camera_name]["onvif"]
+        move_request = self.cams[camera_name]["relative_move_request"]
+
+        # function takes in -1 to 1 for pan and tilt, interpolate to the values of the camera.
+        # The onvif spec says this can report as +INF and -INF, so this may need to be modified
+        pan = numpy.interp(
+            pan,
+            [-1, 1],
+            [
+                self.cams[camera_name]["relative_fov_range"]["XRange"]["Min"],
+                self.cams[camera_name]["relative_fov_range"]["XRange"]["Max"],
+            ],
+        )
+        tilt = numpy.interp(
+            tilt,
+            [-1, 1],
+            [
+                self.cams[camera_name]["relative_fov_range"]["YRange"]["Min"],
+                self.cams[camera_name]["relative_fov_range"]["YRange"]["Max"],
+            ],
+        )
+
+        move_request.Speed = {
+            "PanTilt": {
+                "x": speed,
+                "y": speed,
+            },
+            "Zoom": 0,
+        }
+
+        # move pan and tilt separately
+        move_request.Translation.PanTilt.x = pan
+        move_request.Translation.PanTilt.y = 0
+        move_request.Translation.Zoom.x = 0
+
+        onvif.get_service("ptz").RelativeMove(move_request)
+
+        move_request.Translation.PanTilt.x = 0
+        move_request.Translation.PanTilt.y = tilt
+        move_request.Translation.Zoom.x = 0
+
+        onvif.get_service("ptz").RelativeMove(move_request)
+
+        self.cams[camera_name]["active"] = False
+
     def _move_to_preset(self, camera_name: str, preset: str) -> None:
         if preset not in self.cams[camera_name]["presets"]:
             logger.error(f"{preset} is not a valid preset for {camera_name}")
             return
 
         self.cams[camera_name]["active"] = True
+        self.camera_metrics[camera_name]["ptz_stopped"].clear()
         move_request = self.cams[camera_name]["move_request"]
         onvif: ONVIFCamera = self.cams[camera_name]["onvif"]
         preset_token = self.cams[camera_name]["presets"][preset]
@@ -158,6 +278,7 @@ class OnvifController:
                 "PresetToken": preset_token,
             }
         )
+        self.camera_metrics[camera_name]["ptz_stopped"].set()
         self.cams[camera_name]["active"] = False
 
     def _zoom(self, camera_name: str, command: OnvifCommandEnum) -> None:
@@ -216,3 +337,30 @@ class OnvifController:
             "features": self.cams[camera_name]["features"],
             "presets": list(self.cams[camera_name]["presets"].keys()),
         }
+
+    def get_camera_status(self, camera_name: str) -> dict[str, any]:
+        if camera_name not in self.cams.keys():
+            logger.error(f"Onvif is not setup for {camera_name}")
+            return {}
+
+        if not self.cams[camera_name]["init"]:
+            self._init_onvif(camera_name)
+
+        onvif: ONVIFCamera = self.cams[camera_name]["onvif"]
+        status_request = self.cams[camera_name]["status_request"]
+        status = onvif.get_service("ptz").GetStatus(status_request)
+
+        if status.MoveStatus.PanTilt == "IDLE" or status.MoveStatus.Zoom == "IDLE":
+            self.cams[camera_name]["active"] = False
+            self.camera_metrics[camera_name]["ptz_stopped"].set()
+        else:
+            self.cams[camera_name]["active"] = True
+            self.camera_metrics[camera_name]["ptz_stopped"].clear()
+
+        return {
+            "pan": status.Position.PanTilt.x,
+            "tilt": status.Position.PanTilt.y,
+            "zoom": status.Position.Zoom.x,
+            "pantilt_moving": status.MoveStatus.PanTilt,
+            "zoom_moving": status.MoveStatus.Zoom,
+        }

frigate/track/norfair_tracker.py

@@ -5,7 +5,8 @@ import numpy as np
 from norfair import Detection, Drawable, Tracker, draw_boxes
 from norfair.drawing.drawer import Drawer
 
-from frigate.config import DetectConfig
+from frigate.config import CameraConfig
+from frigate.ptz.autotrack import PtzMotionEstimator
 from frigate.track import ObjectTracker
 from frigate.util.image import intersection_over_union
 
@@ -54,12 +55,16 @@ def frigate_distance(detection: Detection, tracked_object) -> float:
 
 
 class NorfairTracker(ObjectTracker):
-    def __init__(self, config: DetectConfig):
+    def __init__(self, config: CameraConfig, ptz_autotracker_enabled, ptz_stopped):
         self.tracked_objects = {}
         self.disappeared = {}
         self.positions = {}
-        self.max_disappeared = config.max_disappeared
-        self.detect_config = config
+        self.max_disappeared = config.detect.max_disappeared
+        self.camera_config = config
+        self.detect_config = config.detect
+        self.ptz_autotracker_enabled = ptz_autotracker_enabled.value
+        self.ptz_stopped = ptz_stopped
+        self.camera_name = config.name
         self.track_id_map = {}
         # TODO: could also initialize a tracker per object class if there
         #       was a good reason to have different distance calculations
@@ -69,6 +74,8 @@ class NorfairTracker(ObjectTracker):
             initialization_delay=0,
             hit_counter_max=self.max_disappeared,
         )
+        if self.ptz_autotracker_enabled:
+            self.ptz_motion_estimator = PtzMotionEstimator(config, self.ptz_stopped)
 
     def register(self, track_id, obj):
         rand_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=6))
@@ -230,7 +237,16 @@ class NorfairTracker(ObjectTracker):
                 )
             )
 
-        tracked_objects = self.tracker.update(detections=norfair_detections)
+        coord_transformations = None
+
+        if self.ptz_autotracker_enabled:
+            coord_transformations = self.ptz_motion_estimator.motion_estimator(
+                detections, frame_time, self.camera_name
+            )
+
+        tracked_objects = self.tracker.update(
+            detections=norfair_detections, coord_transformations=coord_transformations
+        )
 
         # update or create new tracks
         active_ids = []

frigate/types.py

@@ -1,5 +1,6 @@
 from multiprocessing.context import Process
 from multiprocessing.sharedctypes import Synchronized
+from multiprocessing.synchronize import Event
 from typing import Optional, TypedDict
 
 from faster_fifo import Queue
@@ -17,6 +18,8 @@ class CameraMetricsTypes(TypedDict):
     frame_queue: Queue
     motion_enabled: Synchronized
     improve_contrast_enabled: Synchronized
+    ptz_autotracker_enabled: Synchronized
+    ptz_stopped: Event
     motion_threshold: Synchronized
     motion_contour_area: Synchronized
     process: Optional[Process]

frigate/video.py

@@ -478,6 +478,8 @@ def track_camera(
     detection_enabled = process_info["detection_enabled"]
     motion_enabled = process_info["motion_enabled"]
     improve_contrast_enabled = process_info["improve_contrast_enabled"]
+    ptz_autotracker_enabled = process_info["ptz_autotracker_enabled"]
+    ptz_stopped = process_info["ptz_stopped"]
     motion_threshold = process_info["motion_threshold"]
     motion_contour_area = process_info["motion_contour_area"]
 
@@ -497,7 +499,7 @@ def track_camera(
         name, labelmap, detection_queue, result_connection, model_config, stop_event
     )
 
-    object_tracker = NorfairTracker(config.detect)
+    object_tracker = NorfairTracker(config, ptz_autotracker_enabled, ptz_stopped)
 
     frame_manager = SharedMemoryFrameManager()
 
@@ -518,6 +520,7 @@ def track_camera(
         detection_enabled,
         motion_enabled,
         stop_event,
+        ptz_stopped,
     )
 
     logger.info(f"{name}: exiting subprocess")
@@ -742,6 +745,7 @@ def process_frames(
     detection_enabled: mp.Value,
     motion_enabled: mp.Value,
     stop_event,
+    ptz_stopped: mp.Event,
     exit_on_empty: bool = False,
 ):
     fps = process_info["process_fps"]
@@ -778,7 +782,11 @@ def process_frames(
             continue
 
         # look for motion if enabled
-        motion_boxes = motion_detector.detect(frame) if motion_enabled.value else []
+        motion_boxes = (
+            motion_detector.detect(frame)
+            if motion_enabled.value and ptz_stopped.is_set()
+            else []
+        )
 
         regions = []
         consolidated_detections = []