Autotracking bugfixes and zooming updates (#8103)

* zoom in/out in search for lost objects

* predicted box should not be empty

* clean up and update zoom logic

* only zoom if enabled

* more cleanup

* check for valid velocity when zooming

* only try absolute zoom in if obj area has changed

* zoom logic

* don't enqueue lost object zoom if already at limit

* don't disable motion boxes during ptz moves

* velocity threshold based on move coefficients

* fix area zoom logic

* disable debug zoom

* don't process objects if ptz moving

* recalc with exponent

* change exponent

* remove lost object zooming

* increase distance threshold for stationary object

* increase distance threshold constant

* only zoom out if nonzero

* camera name in all debug logging

* add camera name to debug logging

* camera variable name consistency

* update calibration behavior and docs

* docs and better zooming

* more sensible target values

* docs wording

* fix velocity threshold variable

* zooming tweaks and remove iou for current objects

* debug and docs

* get valid velocity

* include zero

* additional debug statements

* add zoom hysteresis

* zoom on initial move if relative

* only update target box if we actually zoom

* merge dev

* use getattr instead of get

* increase distance threshold

* reverse logic

* get_camera_status after preset move to store zoom

* final tweaks and docs

* use constants and catch possible debug exception

* adjust zoom factor exponent

* don't run motion estimation when calling preset

* adjust dimension threshold

* use numpy for velocity estimate calcs

* more numpy conversion

* fix numpy shapes

* numpy zeros dimension

* more zoom out conditions

* fix velocity bug

* ensure init has been called in debug view

* ensure onvif init if enabling by mqtt

* change default hysteresis values

* recalc relative zoom value

* zoom out value

* try to zoom when object isn't moving

* try zoom when tracked object is not moving

* don't try to zoom every time

* negate zoom out condition when needed

* hysteresis constants for absolute zooming

* update zoom conditions

* don't recalc target box on zoom only

* zoom out if above area threshold

* don't print zooming debug for stationary obj

* revamp zooming to use area moving average

* zooming tweaks and expose property

* limit zoom with max target box

* use calibration to determine zoom levels

* zoom logic fix

* docs

* add tapo c200 camera

* fix initial absolute zoom

* small zoom logic fix

* better invalid velocity checks

* fix test

* really fix test this time

frigate/app.py

@@ -190,6 +190,12 @@ class FrigateApp:
                 "ptz_zoom_level": mp.Value("d", 0.0),  # type: ignore[typeddict-item]
                 # issue https://github.com/python/typeshed/issues/8799
                 # from mypy 0.981 onwards
+                "ptz_max_zoom": mp.Value("d", 0.0),  # type: ignore[typeddict-item]
+                # issue https://github.com/python/typeshed/issues/8799
+                # from mypy 0.981 onwards
+                "ptz_min_zoom": mp.Value("d", 0.0),  # type: ignore[typeddict-item]
+                # issue https://github.com/python/typeshed/issues/8799
+                # from mypy 0.981 onwards
             }
             self.ptz_metrics[camera_name]["ptz_stopped"].set()
             self.feature_metrics[camera_name] = {

frigate/comms/dispatcher.py

@@ -182,11 +182,13 @@ class Dispatcher:
             if not self.ptz_metrics[camera_name]["ptz_autotracker_enabled"].value:
                 logger.info(f"Turning on ptz autotracker for {camera_name}")
                 self.ptz_metrics[camera_name]["ptz_autotracker_enabled"].value = True
+                self.ptz_metrics[camera_name]["ptz_start_time"].value = 0
                 ptz_autotracker_settings.enabled = True
         elif payload == "OFF":
             if self.ptz_metrics[camera_name]["ptz_autotracker_enabled"].value:
                 logger.info(f"Turning off ptz autotracker for {camera_name}")
                 self.ptz_metrics[camera_name]["ptz_autotracker_enabled"].value = False
+                self.ptz_metrics[camera_name]["ptz_start_time"].value = 0
                 ptz_autotracker_settings.enabled = False
 
         self.publish(f"{camera_name}/ptz_autotracker/state", payload, retain=True)

frigate/config.py

@@ -188,8 +188,8 @@ class PtzAutotrackConfig(FrigateBaseModel):
         else:
             raise ValueError("Invalid type for movement_weights")
 
-        if len(weights) != 3:
-            raise ValueError("movement_weights must have exactly 3 floats")
+        if len(weights) != 5:
+            raise ValueError("movement_weights must have exactly 5 floats")
 
         return weights
 

frigate/const.py

@@ -57,3 +57,13 @@ MAX_PLAYLIST_SECONDS = 7200  # support 2 hour segments for a single playlist to
 
 INSERT_MANY_RECORDINGS = "insert_many_recordings"
 REQUEST_REGION_GRID = "request_region_grid"
+
+# Autotracking
+
+AUTOTRACKING_MAX_AREA_RATIO = 0.5
+AUTOTRACKING_MOTION_MIN_DISTANCE = 20
+AUTOTRACKING_MOTION_MAX_POINTS = 500
+AUTOTRACKING_MAX_MOVE_METRICS = 500
+AUTOTRACKING_ZOOM_OUT_HYSTERESIS = 1.2
+AUTOTRACKING_ZOOM_IN_HYSTERESIS = 0.9
+AUTOTRACKING_ZOOM_EDGE_THRESHOLD = 0.05

frigate/object_processing.py

@@ -499,6 +499,9 @@ class CameraState:
                 # draw thicker box around ptz autotracked object
                 if (
                     self.camera_config.onvif.autotracking.enabled
+                    and self.ptz_autotracker_thread.ptz_autotracker.autotracker_init[
+                        self.name
+                    ]
                     and self.ptz_autotracker_thread.ptz_autotracker.tracked_object[
                         self.name
                     ]
@@ -507,6 +510,7 @@ class CameraState:
                     == self.ptz_autotracker_thread.ptz_autotracker.tracked_object[
                         self.name
                     ].obj_data["id"]
+                    and obj["frame_time"] == frame_time
                 ):
                     thickness = 5
                     color = self.config.model.colormap[obj["label"]]

frigate/ptz/onvif.py

@@ -77,6 +77,7 @@ class OnvifController:
 
         request = ptz.create_type("GetConfigurations")
         configs = ptz.GetConfigurations(request)[0]
+        logger.debug(f"Onvif configs for {camera_name}: {configs}")
 
         request = ptz.create_type("GetConfigurationOptions")
         request.ConfigurationToken = profile.PTZConfiguration.token
@@ -196,6 +197,20 @@ class OnvifController:
 
         if ptz_config.Spaces and ptz_config.Spaces.RelativeZoomTranslationSpace:
             supported_features.append("zoom-r")
+            try:
+                # get camera's zoom limits from onvif config
+                self.cams[camera_name][
+                    "relative_zoom_range"
+                ] = ptz_config.Spaces.RelativeZoomTranslationSpace[0]
+            except Exception:
+                if (
+                    self.config.cameras[camera_name].onvif.autotracking.zooming
+                    == ZoomingModeEnum.relative
+                ):
+                    self.config.cameras[camera_name].onvif.autotracking.zooming = False
+                    logger.warning(
+                        f"Disabling autotracking zooming for {camera_name}: Relative zoom not supported"
+                    )
 
         if ptz_config.Spaces and ptz_config.Spaces.AbsoluteZoomPositionSpace:
             supported_features.append("zoom-a")
@@ -273,7 +288,9 @@ class OnvifController:
             logger.error(f"{camera_name} does not support ONVIF RelativeMove (FOV).")
             return
 
-        logger.debug(f"{camera_name} called RelativeMove: pan: {pan} tilt: {tilt}")
+        logger.debug(
+            f"{camera_name} called RelativeMove: pan: {pan} tilt: {tilt} zoom: {zoom}"
+        )
 
         if self.cams[camera_name]["active"]:
             logger.warning(
@@ -284,7 +301,7 @@ class OnvifController:
         self.cams[camera_name]["active"] = True
         self.ptz_metrics[camera_name]["ptz_stopped"].clear()
         logger.debug(
-            f"PTZ start time: {self.ptz_metrics[camera_name]['ptz_frame_time'].value}"
+            f"{camera_name}: PTZ start time: {self.ptz_metrics[camera_name]['ptz_frame_time'].value}"
         )
         self.ptz_metrics[camera_name]["ptz_start_time"].value = self.ptz_metrics[
             camera_name
@@ -350,6 +367,8 @@ class OnvifController:
 
         self.cams[camera_name]["active"] = True
         self.ptz_metrics[camera_name]["ptz_stopped"].clear()
+        self.ptz_metrics[camera_name]["ptz_start_time"].value = 0
+        self.ptz_metrics[camera_name]["ptz_stop_time"].value = 0
         move_request = self.cams[camera_name]["move_request"]
         onvif: ONVIFCamera = self.cams[camera_name]["onvif"]
         preset_token = self.cams[camera_name]["presets"][preset]
@@ -359,7 +378,7 @@ class OnvifController:
                 "PresetToken": preset_token,
             }
         )
-        self.ptz_metrics[camera_name]["ptz_stopped"].set()
+
         self.cams[camera_name]["active"] = False
 
     def _zoom(self, camera_name: str, command: OnvifCommandEnum) -> None:
@@ -396,7 +415,7 @@ class OnvifController:
         self.cams[camera_name]["active"] = True
         self.ptz_metrics[camera_name]["ptz_stopped"].clear()
         logger.debug(
-            f"PTZ start time: {self.ptz_metrics[camera_name]['ptz_frame_time'].value}"
+            f"{camera_name}: PTZ start time: {self.ptz_metrics[camera_name]['ptz_frame_time'].value}"
         )
         self.ptz_metrics[camera_name]["ptz_start_time"].value = self.ptz_metrics[
             camera_name
@@ -418,7 +437,7 @@ class OnvifController:
         move_request.Speed = {"Zoom": speed}
         move_request.Position = {"Zoom": zoom}
 
-        logger.debug(f"Absolute zoom: {zoom}")
+        logger.debug(f"{camera_name}: Absolute zoom: {zoom}")
 
         onvif.get_service("ptz").AbsoluteMove(move_request)
 
@@ -528,7 +547,7 @@ class OnvifController:
                 self.ptz_metrics[camera_name]["ptz_stopped"].set()
 
                 logger.debug(
-                    f"PTZ stop time: {self.ptz_metrics[camera_name]['ptz_frame_time'].value}"
+                    f"{camera_name}: PTZ stop time: {self.ptz_metrics[camera_name]['ptz_frame_time'].value}"
                 )
 
                 self.ptz_metrics[camera_name]["ptz_stop_time"].value = self.ptz_metrics[
@@ -540,7 +559,7 @@ class OnvifController:
                 self.ptz_metrics[camera_name]["ptz_stopped"].clear()
 
                 logger.debug(
-                    f"PTZ start time: {self.ptz_metrics[camera_name]['ptz_frame_time'].value}"
+                    f"{camera_name}: PTZ start time: {self.ptz_metrics[camera_name]['ptz_frame_time'].value}"
                 )
 
                 self.ptz_metrics[camera_name][
@@ -550,7 +569,7 @@ class OnvifController:
 
         if (
             self.config.cameras[camera_name].onvif.autotracking.zooming
-            == ZoomingModeEnum.absolute
+            != ZoomingModeEnum.disabled
         ):
             # store absolute zoom level as 0 to 1 interpolated from the values of the camera
             self.ptz_metrics[camera_name]["ptz_zoom_level"].value = numpy.interp(
@@ -562,5 +581,23 @@ class OnvifController:
                 ],
             )
             logger.debug(
-                f'Camera zoom level: {self.ptz_metrics[camera_name]["ptz_zoom_level"].value}'
+                f'{camera_name}: Camera zoom level: {self.ptz_metrics[camera_name]["ptz_zoom_level"].value}'
             )
+
+        # some hikvision cams won't update MoveStatus, so warn if it hasn't changed
+        if (
+            not self.ptz_metrics[camera_name]["ptz_stopped"].is_set()
+            and not self.ptz_metrics[camera_name]["ptz_reset"].is_set()
+            and self.ptz_metrics[camera_name]["ptz_start_time"].value != 0
+            and self.ptz_metrics[camera_name]["ptz_frame_time"].value
+            > (self.ptz_metrics[camera_name]["ptz_start_time"].value + 10)
+            and self.ptz_metrics[camera_name]["ptz_stop_time"].value == 0
+        ):
+            logger.debug(
+                f'Start time: {self.ptz_metrics[camera_name]["ptz_start_time"].value}, Stop time: {self.ptz_metrics[camera_name]["ptz_stop_time"].value}, Frame time: {self.ptz_metrics[camera_name]["ptz_frame_time"].value}'
+            )
+            # set the stop time so we don't come back into this again and spam the logs
+            self.ptz_metrics[camera_name]["ptz_stop_time"].value = self.ptz_metrics[
+                camera_name
+            ]["ptz_frame_time"].value
+            logger.warning(f"Camera {camera_name} is still in ONVIF 'MOVING' status.")

frigate/test/test_config.py

@@ -1641,7 +1641,9 @@ class TestConfig(unittest.TestCase):
                         "width": 1920,
                         "fps": 5,
                     },
-                    "onvif": {"autotracking": {"movement_weights": "1.23, 2.34, 0.50"}},
+                    "onvif": {
+                        "autotracking": {"movement_weights": "0, 1, 1.23, 2.34, 0.50"}
+                    },
                 }
             },
         }
@@ -1649,6 +1651,8 @@ class TestConfig(unittest.TestCase):
 
         runtime_config = frigate_config.runtime_config()
         assert runtime_config.cameras["back"].onvif.autotracking.movement_weights == [
+            0,
+            1,
             1.23,
             2.34,
             0.50,

frigate/track/norfair_tracker.py

@@ -278,11 +278,10 @@ class NorfairTracker(ObjectTracker):
                 min(self.detect_config.width - 1, estimate[2]),
                 min(self.detect_config.height - 1, estimate[3]),
             )
-            estimate_velocity = tuple(t.estimate_velocity.flatten().astype(int))
             obj = {
                 **t.last_detection.data,
                 "estimate": estimate,
-                "estimate_velocity": estimate_velocity,
+                "estimate_velocity": t.estimate_velocity,
             }
             active_ids.append(t.global_id)
             if t.global_id not in self.track_id_map:

frigate/types.py

@@ -35,6 +35,8 @@ class PTZMetricsTypes(TypedDict):
     ptz_stop_time: Synchronized
     ptz_frame_time: Synchronized
     ptz_zoom_level: Synchronized
+    ptz_max_zoom: Synchronized
+    ptz_min_zoom: Synchronized
 
 
 class FeatureMetricsTypes(TypedDict):