Face recognition improvements (#16034)

* Start working on bird processor

* Initial setup for bird processing

* Improvements to handling

* Get classification working

* Cleanup classification

* Add classification config

* Update sort

.cspell/frigate-dictionary.txt

@@ -2,6 +2,7 @@ aarch
 absdiff
 airockchip
 Alloc
+alpr
 Amcrest
 amdgpu
 analyzeduration
@@ -61,6 +62,7 @@ dsize
 dtype
 ECONNRESET
 edgetpu
+facenet
 fastapi
 faststart
 fflags
@@ -114,6 +116,8 @@ itemsize
 Jellyfin
 jetson
 jetsons
+jina
+jinaai
 joserfc
 jsmpeg
 jsonify
@@ -187,6 +191,7 @@ openai
 opencv
 openvino
 OWASP
+paddleocr
 paho
 passwordless
 popleft
@@ -308,4 +313,4 @@ yolo
 yolonas
 yolox
 zeep
-zerolatency
+zerolatency

.github/workflows/pull_request.yml

@@ -6,7 +6,7 @@ on:
       - "docs/**"
 
 env:
-  DEFAULT_PYTHON: 3.9
+  DEFAULT_PYTHON: 3.11
 
 jobs:
   build_devcontainer:

Makefile

@@ -1,7 +1,7 @@
 default_target: local
 
 COMMIT_HASH := $(shell git log -1 --pretty=format:"%h"|tail -1)
-VERSION = 0.15.0
+VERSION = 0.16.0
 IMAGE_REPO ?= ghcr.io/blakeblackshear/frigate
 GITHUB_REF_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
 BOARDS= #Initialized empty

docker/hailo8l/Dockerfile

@@ -5,6 +5,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 # Build Python wheels
 FROM wheels AS h8l-wheels
 
+RUN python3 -m pip config set global.break-system-packages true
 COPY docker/main/requirements-wheels.txt /requirements-wheels.txt
 COPY docker/hailo8l/requirements-wheels-h8l.txt /requirements-wheels-h8l.txt
 
@@ -30,6 +31,7 @@ COPY --from=hailort /hailo-wheels /deps/hailo-wheels
 COPY --from=hailort /rootfs/ /
 
 # Install the wheels
+RUN python3 -m pip config set global.break-system-packages true
 RUN pip3 install -U /deps/h8l-wheels/*.whl
 RUN pip3 install -U /deps/hailo-wheels/*.whl
 

docker/hailo8l/install_hailort.sh

@@ -2,7 +2,7 @@
 
 set -euxo pipefail
 
-hailo_version="4.19.0"
+hailo_version="4.20.0"
 
 if [[ "${TARGETARCH}" == "amd64" ]]; then
     arch="x86_64"
@@ -15,5 +15,5 @@ wget -qO- "https://github.com/frigate-nvr/hailort/releases/download/v${hailo_ver
 
 mkdir -p /hailo-wheels
 
-wget -P /hailo-wheels/ "https://github.com/frigate-nvr/hailort/releases/download/v${hailo_version}/hailort-${hailo_version}-cp39-cp39-linux_${arch}.whl"
+wget -P /hailo-wheels/ "https://github.com/frigate-nvr/hailort/releases/download/v${hailo_version}/hailort-${hailo_version}-cp311-cp311-linux_${arch}.whl"
 

docker/hailo8l/user_installation.sh

@@ -4,6 +4,7 @@
 sudo apt-get update
 sudo apt-get install -y build-essential cmake git wget
 
+hailo_version="4.20.0"
 arch=$(uname -m)
 
 if [[ $arch == "x86_64" ]]; then
@@ -13,7 +14,7 @@ else
 fi
 
 # Clone the HailoRT driver repository
-git clone --depth 1 --branch v4.19.0 https://github.com/hailo-ai/hailort-drivers.git
+git clone --depth 1 --branch v${hailo_version} https://github.com/hailo-ai/hailort-drivers.git
 
 # Build and install the HailoRT driver
 cd hailort-drivers/linux/pcie

docker/main/Dockerfile

@@ -3,12 +3,12 @@
 # https://askubuntu.com/questions/972516/debian-frontend-environment-variable
 ARG DEBIAN_FRONTEND=noninteractive
 
-ARG BASE_IMAGE=debian:11
-ARG SLIM_BASE=debian:11-slim
+ARG BASE_IMAGE=debian:12
+ARG SLIM_BASE=debian:12-slim
 
 FROM ${BASE_IMAGE} AS base
 
-FROM --platform=${BUILDPLATFORM} debian:11 AS base_host
+FROM --platform=${BUILDPLATFORM} debian:12 AS base_host
 
 FROM ${SLIM_BASE} AS slim-base
 
@@ -66,8 +66,8 @@ COPY docker/main/requirements-ov.txt /requirements-ov.txt
 RUN apt-get -qq update \
     && apt-get -qq install -y wget python3 python3-dev python3-distutils gcc pkg-config libhdf5-dev \
     && wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
-    && python3 get-pip.py "pip" \
-    && pip install -r /requirements-ov.txt
+    && python3 get-pip.py "pip" --break-system-packages \
+    && pip install --break-system-packages -r /requirements-ov.txt
 
 # Get OpenVino Model
 RUN --mount=type=bind,source=docker/main/build_ov_model.py,target=/build_ov_model.py \
@@ -139,24 +139,17 @@ ARG TARGETARCH
 # Use a separate container to build wheels to prevent build dependencies in final image
 RUN apt-get -qq update \
     && apt-get -qq install -y \
-    apt-transport-https \
-    gnupg \
-    wget \
-    # the key fingerprint can be obtained from https://ftp-master.debian.org/keys.html
-    && wget -qO- "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA4285295FC7B1A81600062A9605C66F00D6C9793" | \
-    gpg --dearmor > /usr/share/keyrings/debian-archive-bullseye-stable.gpg \
-    && echo "deb [signed-by=/usr/share/keyrings/debian-archive-bullseye-stable.gpg] http://deb.debian.org/debian bullseye main contrib non-free" | \
-    tee /etc/apt/sources.list.d/debian-bullseye-nonfree.list \
+    apt-transport-https wget \
     && apt-get -qq update \
     && apt-get -qq install -y \
-    python3.9 \
-    python3.9-dev \
+    python3 \
+    python3-dev \
     # opencv dependencies
     build-essential cmake git pkg-config libgtk-3-dev \
     libavcodec-dev libavformat-dev libswscale-dev libv4l-dev \
     libxvidcore-dev libx264-dev libjpeg-dev libpng-dev libtiff-dev \
     gfortran openexr libatlas-base-dev libssl-dev\
-    libtbb2 libtbb-dev libdc1394-22-dev libopenexr-dev \
+    libtbbmalloc2 libtbb-dev libdc1394-dev libopenexr-dev \
     libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev \
     # sqlite3 dependencies
     tclsh \
@@ -164,14 +157,11 @@ RUN apt-get -qq update \
     gcc gfortran libopenblas-dev liblapack-dev && \
     rm -rf /var/lib/apt/lists/*
 
-# Ensure python3 defaults to python3.9
-RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1
-
 RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
-    && python3 get-pip.py "pip"
+    && python3 get-pip.py "pip" --break-system-packages
 
 COPY docker/main/requirements.txt /requirements.txt
-RUN pip3 install -r /requirements.txt
+RUN pip3 install -r /requirements.txt --break-system-packages
 
 # Build pysqlite3 from source
 COPY docker/main/build_pysqlite3.sh /build_pysqlite3.sh
@@ -222,8 +212,8 @@ RUN --mount=type=bind,source=docker/main/install_deps.sh,target=/deps/install_de
     /deps/install_deps.sh
 
 RUN --mount=type=bind,from=wheels,source=/wheels,target=/deps/wheels \
-    python3 -m pip install --upgrade pip && \
-    pip3 install -U /deps/wheels/*.whl
+    python3 -m pip install --upgrade pip --break-system-packages && \
+    pip3 install -U /deps/wheels/*.whl --break-system-packages
 
 COPY --from=deps-rootfs / /
 
@@ -270,7 +260,7 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/*
 
 RUN --mount=type=bind,source=./docker/main/requirements-dev.txt,target=/workspace/frigate/requirements-dev.txt \
-    pip3 install -r requirements-dev.txt
+    pip3 install -r requirements-dev.txt --break-system-packages
 
 HEALTHCHECK NONE
 

docker/main/build_nginx.sh

@@ -8,8 +8,7 @@ SECURE_TOKEN_MODULE_VERSION="1.5"
 SET_MISC_MODULE_VERSION="v0.33"
 NGX_DEVEL_KIT_VERSION="v0.3.3"
 
-cp /etc/apt/sources.list /etc/apt/sources.list.d/sources-src.list
-sed -i 's|deb http|deb-src http|g' /etc/apt/sources.list.d/sources-src.list
+sed -i '/^Types:/s/deb/& deb-src/' /etc/apt/sources.list.d/debian.sources
 apt-get update
 
 apt-get -yqq build-dep nginx

docker/main/build_ov_model.py

@@ -4,7 +4,7 @@ from openvino.tools import mo
 ov_model = mo.convert_model(
     "/models/ssdlite_mobilenet_v2_coco_2018_05_09/frozen_inference_graph.pb",
     compress_to_fp16=True,
-    transformations_config="/usr/local/lib/python3.9/dist-packages/openvino/tools/mo/front/tf/ssd_v2_support.json",
+    transformations_config="/usr/local/lib/python3.11/dist-packages/openvino/tools/mo/front/tf/ssd_v2_support.json",
     tensorflow_object_detection_api_pipeline_config="/models/ssdlite_mobilenet_v2_coco_2018_05_09/pipeline.config",
     reverse_input_channels=True,
 )

docker/main/build_sqlite_vec.sh

@@ -4,8 +4,7 @@ set -euxo pipefail
 
 SQLITE_VEC_VERSION="0.1.3"
 
-cp /etc/apt/sources.list /etc/apt/sources.list.d/sources-src.list
-sed -i 's|deb http|deb-src http|g' /etc/apt/sources.list.d/sources-src.list
+sed -i '/^Types:/s/deb/& deb-src/' /etc/apt/sources.list.d/debian.sources
 apt-get update
 apt-get -yqq build-dep sqlite3 gettext git
 

docker/main/install_deps.sh

@@ -11,33 +11,34 @@ apt-get -qq install --no-install-recommends -y \
     lbzip2 \
     procps vainfo \
     unzip locales tzdata libxml2 xz-utils \
-    python3.9 \
+    python3 \
     python3-pip \
     curl \
     lsof \
     jq \
-    nethogs
-
-# ensure python3 defaults to python3.9
-update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1
+    nethogs \
+    libgl1 \
+    libglib2.0-0 \
+    libusb-1.0.0
 
 mkdir -p -m 600 /root/.gnupg
 
-# add coral repo
-curl -fsSLo - https://packages.cloud.google.com/apt/doc/apt-key.gpg | \
-    gpg --dearmor -o /etc/apt/trusted.gpg.d/google-cloud-packages-archive-keyring.gpg
-echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" | tee /etc/apt/sources.list.d/coral-edgetpu.list
-echo "libedgetpu1-max libedgetpu/accepted-eula select true" | debconf-set-selections
+# install coral runtime
+wget -q -O /tmp/libedgetpu1-max.deb "https://github.com/feranick/libedgetpu/releases/download/16.0TF2.17.0-1/libedgetpu1-max_16.0tf2.17.0-1.bookworm_${TARGETARCH}.deb"
+unset DEBIAN_FRONTEND
+yes | dpkg -i /tmp/libedgetpu1-max.deb && export DEBIAN_FRONTEND=noninteractive
+rm /tmp/libedgetpu1-max.deb
 
-# enable non-free repo in Debian
-if grep -q "Debian" /etc/issue; then
-    sed -i -e's/ main/ main contrib non-free/g' /etc/apt/sources.list
+# install python3 & tflite runtime
+if [[ "${TARGETARCH}" == "amd64" ]]; then
+    pip3 install --break-system-packages https://github.com/feranick/TFlite-builds/releases/download/v2.17.0/tflite_runtime-2.17.0-cp311-cp311-linux_x86_64.whl
+    pip3 install --break-system-packages https://github.com/feranick/pycoral/releases/download/2.0.2TF2.17.0/pycoral-2.0.2-cp311-cp311-linux_x86_64.whl
 fi
 
-# coral drivers
-apt-get -qq update
-apt-get -qq install --no-install-recommends --no-install-suggests -y \
-    libedgetpu1-max python3-tflite-runtime python3-pycoral
+if [[ "${TARGETARCH}" == "arm64" ]]; then
+    pip3 install --break-system-packages https://github.com/feranick/TFlite-builds/releases/download/v2.17.0/tflite_runtime-2.17.0-cp311-cp311-linux_aarch64.whl
+    pip3 install --break-system-packages https://github.com/feranick/pycoral/releases/download/2.0.2TF2.17.0/pycoral-2.0.2-cp311-cp311-linux_aarch64.whl
+fi
 
 # btbn-ffmpeg -> amd64
 if [[ "${TARGETARCH}" == "amd64" ]]; then
@@ -65,23 +66,15 @@ fi
 
 # arch specific packages
 if [[ "${TARGETARCH}" == "amd64" ]]; then
-    # use debian bookworm for amd / intel-i965 driver packages
-    echo 'deb https://deb.debian.org/debian bookworm main contrib non-free' >/etc/apt/sources.list.d/debian-bookworm.list
-    apt-get -qq update
+    # install amd / intel-i965 driver packages
     apt-get -qq install --no-install-recommends --no-install-suggests -y \
         i965-va-driver intel-gpu-tools onevpl-tools \
         libva-drm2 \
         mesa-va-drivers radeontop
 
-    # something about this dependency requires it to be installed in a separate call rather than in the line above
-    apt-get -qq install --no-install-recommends --no-install-suggests -y \
-        i965-va-driver-shaders
-
     # intel packages use zst compression so we need to update dpkg
     apt-get install -y dpkg
 
-    rm -f /etc/apt/sources.list.d/debian-bookworm.list
-
     # use intel apt intel packages
     wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | gpg --yes --dearmor --output /usr/share/keyrings/intel-graphics.gpg
     echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu jammy client" | tee /etc/apt/sources.list.d/intel-gpu-jammy.list

docker/main/requirements-wheels.txt

@@ -10,10 +10,10 @@ imutils == 0.5.*
 joserfc == 1.0.*
 pathvalidate == 3.2.*
 markupsafe == 2.1.*
+python-multipart == 0.0.12
+# General
 mypy == 1.6.1
-numpy == 1.26.*
-onvif_zeep == 0.2.12
-opencv-python-headless == 4.9.0.*
+onvif-zeep-async == 3.1.*
 paho-mqtt == 2.1.*
 pandas == 2.2.*
 peewee == 3.17.*
@@ -27,17 +27,21 @@ ruamel.yaml == 0.18.*
 tzlocal == 5.2
 requests == 2.32.*
 types-requests == 2.32.*
-scipy == 1.13.*
 norfair == 2.2.*
 setproctitle == 1.3.*
 ws4py == 0.5.*
 unidecode == 1.3.*
+# Image Manipulation
+numpy == 1.26.*
+opencv-python-headless == 4.10.0.*
+opencv-contrib-python == 4.9.0.*
+scipy == 1.14.*
 # OpenVino & ONNX
-openvino == 2024.3.*
-onnxruntime-openvino == 1.19.* ; platform_machine == 'x86_64'
-onnxruntime == 1.19.* ; platform_machine == 'aarch64'
+openvino == 2024.4.*
+onnxruntime-openvino == 1.20.* ; platform_machine == 'x86_64'
+onnxruntime == 1.20.* ; platform_machine == 'aarch64'
 # Embeddings
-transformers == 4.48.*
+transformers == 4.45.*
 # Generative AI
 google-generativeai == 0.8.*
 ollama == 0.3.*
@@ -45,3 +49,6 @@ openai == 1.51.*
 # push notifications
 py-vapid == 1.9.*
 pywebpush == 2.0.*
+# alpr
+pyclipper == 1.3.*
+shapely == 2.0.*

docker/main/requirements.txt

@@ -1,2 +1,2 @@
-scikit-build == 0.17.*
+scikit-build == 0.18.*
 nvidia-pyindex

docker/main/rootfs/usr/local/nginx/conf/nginx.conf

@@ -81,6 +81,9 @@ http {
         open_file_cache_errors on;
         aio on;
 
+        # file upload size
+        client_max_body_size 10M;
+
         # https://github.com/kaltura/nginx-vod-module#vod_open_file_thread_pool
         vod_open_file_thread_pool default;
 

docker/rockchip/COCO/coco_subset_20.txt

@@ -0,0 +1,20 @@
+./subset/000000005001.jpg
+./subset/000000038829.jpg
+./subset/000000052891.jpg
+./subset/000000075612.jpg
+./subset/000000098261.jpg
+./subset/000000181542.jpg
+./subset/000000215245.jpg
+./subset/000000277005.jpg
+./subset/000000288685.jpg
+./subset/000000301421.jpg
+./subset/000000334371.jpg
+./subset/000000348481.jpg
+./subset/000000373353.jpg
+./subset/000000397681.jpg
+./subset/000000414673.jpg
+./subset/000000419312.jpg
+./subset/000000465822.jpg
+./subset/000000475732.jpg
+./subset/000000559707.jpg
+./subset/000000574315.jpg

docker/rockchip/Dockerfile

@@ -7,21 +7,26 @@ FROM wheels as rk-wheels
 COPY docker/main/requirements-wheels.txt /requirements-wheels.txt
 COPY docker/rockchip/requirements-wheels-rk.txt /requirements-wheels-rk.txt
 RUN sed -i "/https:\/\//d" /requirements-wheels.txt
+RUN sed -i "/onnxruntime/d" /requirements-wheels.txt
+RUN python3 -m pip config set global.break-system-packages true
 RUN pip3 wheel --wheel-dir=/rk-wheels -c /requirements-wheels.txt -r /requirements-wheels-rk.txt
+RUN rm -rf /rk-wheels/opencv_python-*
 
 FROM deps AS rk-frigate
 ARG TARGETARCH
 
 RUN --mount=type=bind,from=rk-wheels,source=/rk-wheels,target=/deps/rk-wheels \
-    pip3 install -U /deps/rk-wheels/*.whl
+    pip3 install --no-deps -U /deps/rk-wheels/*.whl --break-system-packages
 
 WORKDIR /opt/frigate/
 COPY --from=rootfs / /
+COPY docker/rockchip/COCO /COCO
+COPY docker/rockchip/conv2rknn.py /opt/conv2rknn.py
 
-ADD https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.0.0/librknnrt.so /usr/lib/
+ADD https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.3.0/librknnrt.so /usr/lib/
 
 RUN rm -rf /usr/lib/btbn-ffmpeg/bin/ffmpeg
 RUN rm -rf /usr/lib/btbn-ffmpeg/bin/ffprobe
-ADD --chmod=111 https://github.com/MarcA711/Rockchip-FFmpeg-Builds/releases/download/6.1-5/ffmpeg /usr/lib/ffmpeg/6.0/bin/
-ADD --chmod=111 https://github.com/MarcA711/Rockchip-FFmpeg-Builds/releases/download/6.1-5/ffprobe /usr/lib/ffmpeg/6.0/bin/
+ADD --chmod=111 https://github.com/MarcA711/Rockchip-FFmpeg-Builds/releases/download/6.1-6/ffmpeg /usr/lib/ffmpeg/6.0/bin/
+ADD --chmod=111 https://github.com/MarcA711/Rockchip-FFmpeg-Builds/releases/download/6.1-6/ffprobe /usr/lib/ffmpeg/6.0/bin/
 ENV PATH="/usr/lib/ffmpeg/6.0/bin/:${PATH}"

docker/rockchip/conv2rknn.py

@@ -0,0 +1,82 @@
+import os
+
+import rknn
+import yaml
+from rknn.api import RKNN
+
+try:
+    with open(rknn.__path__[0] + "/VERSION") as file:
+        tk_version = file.read().strip()
+except FileNotFoundError:
+    pass
+
+try:
+    with open("/config/conv2rknn.yaml", "r") as config_file:
+        configuration = yaml.safe_load(config_file)
+except FileNotFoundError:
+    raise Exception("Please place a config.yaml file in /config/conv2rknn.yaml")
+
+if configuration["config"] != None:
+    rknn_config = configuration["config"]
+else:
+    rknn_config = {}
+
+if not os.path.isdir("/config/model_cache/rknn_cache/onnx"):
+    raise Exception(
+        "Place the onnx models you want to convert to rknn format in /config/model_cache/rknn_cache/onnx"
+    )
+
+if "soc" not in configuration:
+    try:
+        with open("/proc/device-tree/compatible") as file:
+            soc = file.read().split(",")[-1].strip("\x00")
+    except FileNotFoundError:
+        raise Exception("Make sure to run docker in privileged mode.")
+
+    configuration["soc"] = [
+        soc,
+    ]
+
+if "quantization" not in configuration:
+    configuration["quantization"] = False
+
+if "output_name" not in configuration:
+    configuration["output_name"] = "{{input_basename}}"
+
+for input_filename in os.listdir("/config/model_cache/rknn_cache/onnx"):
+    for soc in configuration["soc"]:
+        quant = "i8" if configuration["quantization"] else "fp16"
+
+        input_path = "/config/model_cache/rknn_cache/onnx/" + input_filename
+        input_basename = input_filename[: input_filename.rfind(".")]
+
+        output_filename = (
+            configuration["output_name"].format(
+                quant=quant,
+                input_basename=input_basename,
+                soc=soc,
+                tk_version=tk_version,
+            )
+            + ".rknn"
+        )
+        output_path = "/config/model_cache/rknn_cache/" + output_filename
+
+        rknn_config["target_platform"] = soc
+
+        rknn = RKNN(verbose=True)
+        rknn.config(**rknn_config)
+
+        if rknn.load_onnx(model=input_path) != 0:
+            raise Exception("Error loading model.")
+
+        if (
+            rknn.build(
+                do_quantization=configuration["quantization"],
+                dataset="/COCO/coco_subset_20.txt",
+            )
+            != 0
+        ):
+            raise Exception("Error building model.")
+
+        if rknn.export_rknn(output_path) != 0:
+            raise Exception("Error exporting rknn model.")

docker/rockchip/requirements-wheels-rk.txt

@@ -1 +1,2 @@
-rknn-toolkit-lite2 @ https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.0.0/rknn_toolkit_lite2-2.0.0b0-cp39-cp39-linux_aarch64.whl
+rknn-toolkit2 == 2.3.0
+rknn-toolkit-lite2 == 2.3.0

docker/rocm/Dockerfile

@@ -34,7 +34,7 @@ RUN mkdir -p /opt/rocm-dist/etc/ld.so.conf.d/
 RUN echo /opt/rocm/lib|tee /opt/rocm-dist/etc/ld.so.conf.d/rocm.conf
 
 #######################################################################
-FROM --platform=linux/amd64 debian:11 as debian-base
+FROM --platform=linux/amd64 debian:12 as debian-base
 
 RUN apt-get update && apt-get -y upgrade
 RUN apt-get -y install --no-install-recommends libelf1 libdrm2 libdrm-amdgpu1 libnuma1 kmod
@@ -51,7 +51,7 @@ COPY --from=rocm /opt/rocm-$ROCM /opt/rocm-$ROCM
 RUN ln -s /opt/rocm-$ROCM /opt/rocm
 
 RUN apt-get -y install g++ cmake
-RUN apt-get -y install python3-pybind11 python3.9-distutils python3-dev
+RUN apt-get -y install python3-pybind11 python3-distutils python3-dev
 
 WORKDIR /opt/build
 
@@ -70,10 +70,11 @@ RUN apt-get -y install libnuma1
 WORKDIR /opt/frigate/
 COPY --from=rootfs / /
 
-COPY docker/rocm/requirements-wheels-rocm.txt /requirements.txt
-RUN python3 -m pip install --upgrade pip \
-    && pip3 uninstall -y onnxruntime-openvino \
-    && pip3 install -r /requirements.txt
+# Temporarily disabled to see if a new wheel can be built to support py3.11
+#COPY docker/rocm/requirements-wheels-rocm.txt /requirements.txt
+#RUN python3 -m pip install --upgrade pip \
+#    && pip3 uninstall -y onnxruntime-openvino \
+#    && pip3 install -r /requirements.txt
 
 #######################################################################
 FROM scratch AS rocm-dist
@@ -86,12 +87,12 @@ COPY --from=rocm /opt/rocm-$ROCM/share/miopen/db/*$AMDGPU* /opt/rocm-$ROCM/share
 COPY --from=rocm /opt/rocm-$ROCM/share/miopen/db/*gfx908* /opt/rocm-$ROCM/share/miopen/db/
 COPY --from=rocm /opt/rocm-$ROCM/lib/rocblas/library/*$AMDGPU* /opt/rocm-$ROCM/lib/rocblas/library/
 COPY --from=rocm /opt/rocm-dist/ /
-COPY --from=debian-build /opt/rocm/lib/migraphx.cpython-39-x86_64-linux-gnu.so /opt/rocm-$ROCM/lib/
+COPY --from=debian-build /opt/rocm/lib/migraphx.cpython-311-x86_64-linux-gnu.so /opt/rocm-$ROCM/lib/
 
 #######################################################################
 FROM deps-prelim AS rocm-prelim-hsa-override0
-
-ENV HSA_ENABLE_SDMA=0
+\
+    ENV HSA_ENABLE_SDMA=0
 
 COPY --from=rocm-dist / /
 

docker/rpi/install_deps.sh

@@ -24,7 +24,7 @@ sed -i -e's/ main/ main contrib non-free/g' /etc/apt/sources.list
 if [[ "${TARGETARCH}" == "arm64" ]]; then
     # add raspberry pi repo
     gpg --no-default-keyring --keyring /usr/share/keyrings/raspbian.gpg --keyserver keyserver.ubuntu.com --recv-keys 82B129927FA3303E
-    echo "deb [signed-by=/usr/share/keyrings/raspbian.gpg] https://archive.raspberrypi.org/debian/ bullseye main" | tee /etc/apt/sources.list.d/raspi.list
+    echo "deb [signed-by=/usr/share/keyrings/raspbian.gpg] https://archive.raspberrypi.org/debian/ bookworm main" | tee /etc/apt/sources.list.d/raspi.list
     apt-get -qq update
     apt-get -qq install --no-install-recommends --no-install-suggests -y ffmpeg
 fi

docker/tensorrt/Dockerfile.amd64

@@ -7,18 +7,19 @@ ARG DEBIAN_FRONTEND=noninteractive
 FROM wheels as trt-wheels
 ARG DEBIAN_FRONTEND
 ARG TARGETARCH
+RUN python3 -m pip config set global.break-system-packages true
 
 # Add TensorRT wheels to another folder
 COPY docker/tensorrt/requirements-amd64.txt /requirements-tensorrt.txt
 RUN mkdir -p /trt-wheels && pip3 wheel --wheel-dir=/trt-wheels -r /requirements-tensorrt.txt
 
 FROM tensorrt-base AS frigate-tensorrt
-ENV TRT_VER=8.5.3
+ENV TRT_VER=8.6.1
+RUN python3 -m pip config set global.break-system-packages true
 RUN --mount=type=bind,from=trt-wheels,source=/trt-wheels,target=/deps/trt-wheels \
-    pip3 install -U /deps/trt-wheels/*.whl && \
+    pip3 install -U /deps/trt-wheels/*.whl --break-system-packages && \
     ldconfig
 
-ENV LD_LIBRARY_PATH=/usr/local/lib/python3.9/dist-packages/tensorrt:/usr/local/cuda/lib64:/usr/local/lib/python3.9/dist-packages/nvidia/cufft/lib
 WORKDIR /opt/frigate/
 COPY --from=rootfs / /
 
@@ -31,4 +32,4 @@ COPY --from=trt-deps /usr/local/cuda-12.1 /usr/local/cuda
 COPY docker/tensorrt/detector/rootfs/ /
 COPY --from=trt-deps /usr/local/lib/libyolo_layer.so /usr/local/lib/libyolo_layer.so
 RUN --mount=type=bind,from=trt-wheels,source=/trt-wheels,target=/deps/trt-wheels \
-    pip3 install -U /deps/trt-wheels/*.whl
+    pip3 install -U /deps/trt-wheels/*.whl --break-system-packages

docker/tensorrt/Dockerfile.arm64

@@ -41,11 +41,11 @@ RUN --mount=type=bind,source=docker/tensorrt/detector/build_python_tensorrt.sh,t
     && TENSORRT_VER=$(cat /etc/TENSORRT_VER) /deps/build_python_tensorrt.sh
 
 COPY docker/tensorrt/requirements-arm64.txt /requirements-tensorrt.txt
-ADD https://nvidia.box.com/shared/static/9aemm4grzbbkfaesg5l7fplgjtmswhj8.whl /tmp/onnxruntime_gpu-1.15.1-cp39-cp39-linux_aarch64.whl
+ADD https://nvidia.box.com/shared/static/psl23iw3bh7hlgku0mjo1xekxpego3e3.whl /tmp/onnxruntime_gpu-1.15.1-cp311-cp311-linux_aarch64.whl
 
 RUN pip3 uninstall -y onnxruntime-openvino \
     && pip3 wheel --wheel-dir=/trt-wheels -r /requirements-tensorrt.txt \
-    && pip3 install --no-deps /tmp/onnxruntime_gpu-1.15.1-cp39-cp39-linux_aarch64.whl
+    && pip3 install --no-deps /tmp/onnxruntime_gpu-1.15.1-cp311-cp311-linux_aarch64.whl
 
 FROM build-wheels AS trt-model-wheels
 ARG DEBIAN_FRONTEND

docker/tensorrt/Dockerfile.base

@@ -3,7 +3,7 @@
 # https://askubuntu.com/questions/972516/debian-frontend-environment-variable
 ARG DEBIAN_FRONTEND=noninteractive
 
-ARG TRT_BASE=nvcr.io/nvidia/tensorrt:23.03-py3
+ARG TRT_BASE=nvcr.io/nvidia/tensorrt:23.12-py3
 
 # Build TensorRT-specific library
 FROM ${TRT_BASE} AS trt-deps

docker/tensorrt/detector/rootfs/etc/ld.so.conf.d/cuda_tensorrt.conf

@@ -1,6 +1,8 @@
 /usr/local/lib
-/usr/local/lib/python3.9/dist-packages/nvidia/cudnn/lib
-/usr/local/lib/python3.9/dist-packages/nvidia/cuda_runtime/lib
-/usr/local/lib/python3.9/dist-packages/nvidia/cublas/lib
-/usr/local/lib/python3.9/dist-packages/nvidia/cuda_nvrtc/lib
-/usr/local/lib/python3.9/dist-packages/tensorrt
+/usr/local/cuda/lib64
+/usr/local/lib/python3.11/dist-packages/nvidia/cudnn/lib
+/usr/local/lib/python3.11/dist-packages/nvidia/cuda_runtime/lib
+/usr/local/lib/python3.11/dist-packages/nvidia/cublas/lib
+/usr/local/lib/python3.11/dist-packages/nvidia/cuda_nvrtc/lib
+/usr/local/lib/python3.11/dist-packages/tensorrt
+/usr/local/lib/python3.11/dist-packages/nvidia/cufft/lib

docker/tensorrt/requirements-amd64.txt

@@ -1,9 +1,9 @@
 # NVidia TensorRT Support (amd64 only)
 --extra-index-url 'https://pypi.nvidia.com'
 numpy < 1.24; platform_machine == 'x86_64'
-tensorrt == 8.5.3.*; platform_machine == 'x86_64'
-cuda-python == 11.8; platform_machine == 'x86_64'
-cython == 0.29.*; platform_machine == 'x86_64'
+tensorrt == 8.6.1.*; platform_machine == 'x86_64'
+cuda-python == 11.8.*; platform_machine == 'x86_64'
+cython == 3.0.*; platform_machine == 'x86_64'
 nvidia-cuda-runtime-cu12 == 12.1.*; platform_machine == 'x86_64'
 nvidia-cuda-runtime-cu11 == 11.8.*; platform_machine == 'x86_64'
 nvidia-cublas-cu11 == 11.11.3.6; platform_machine == 'x86_64'

docs/docs/configuration/camera_specific.md

@@ -67,14 +67,15 @@ ffmpeg:
 
 ### Annke C800
 
-This camera is H.265 only. To be able to play clips on some devices (like MacOs or iPhone) the H.265 stream has to be repackaged and the audio stream has to be converted to aac. Unfortunately direct playback of in the browser is not working (yet), but the downloaded clip can be played locally.
+This camera is H.265 only. To be able to play clips on some devices (like MacOs or iPhone) the H.265 stream has to be adjusted using the `apple_compatibility` config.
 
 ```yaml
 cameras:
   annkec800: # <------ Name the camera
     ffmpeg:
+      apple_compatibility: true  # <- Adds compatibility with MacOS and iPhone
       output_args:
-        record: -f segment -segment_time 10 -segment_format mp4 -reset_timestamps 1 -strftime 1 -c:v copy -tag:v hvc1 -bsf:v hevc_mp4toannexb -c:a aac
+        record: preset-record-generic-audio-aac
 
       inputs:
         - path: rtsp://user:password@camera-ip:554/H264/ch1/main/av_stream # <----- Update for your camera

docs/docs/configuration/face_recognition.md

@@ -0,0 +1,35 @@
+---
+id: face_recognition
+title: Face Recognition
+---
+
+Face recognition allows people to be assigned names and when their face is recognized Frigate will assign the person's name as a sub label. This information is included in the UI, filters, as well as in notifications.
+
+Frigate has support for FaceNet to create face embeddings, which runs locally. Embeddings are then saved to Frigate's database.
+
+## Minimum System Requirements
+
+Face recognition works by running a large AI model locally on your system. Systems without a GPU will not run Face Recognition reliably or at all.
+
+## Configuration
+
+Face recognition is disabled by default and requires semantic search to be enabled, face recognition must be enabled in your config file before it can be used. Semantic Search and face recognition are global configuration settings.
+
+```yaml
+face_recognition:
+  enabled: true
+```
+
+## Dataset
+
+The number of images needed for a sufficient training set for face recognition varies depending on several factors:
+
+- Complexity of the task: A simple task like recognizing faces of known individuals may require fewer images than a complex task like identifying unknown individuals in a large crowd.
+- Diversity of the dataset: A dataset with diverse images, including variations in lighting, pose, and facial expressions, will require fewer images per person than a less diverse dataset.
+- Desired accuracy: The higher the desired accuracy, the more images are typically needed.
+
+However, here are some general guidelines:
+
+- Minimum: For basic face recognition tasks, a minimum of 10-20 images per person is often recommended.
+- Recommended: For more robust and accurate systems, 30-50 images per person is a good starting point.
+- Ideal: For optimal performance, especially in challenging conditions, 100 or more images per person can be beneficial.

docs/docs/configuration/hardware_acceleration.md

@@ -175,6 +175,16 @@ For more information on the various values across different distributions, see h
 
 Depending on your OS and kernel configuration, you may need to change the `/proc/sys/kernel/perf_event_paranoid` kernel tunable. You can test the change by running `sudo sh -c 'echo 2 >/proc/sys/kernel/perf_event_paranoid'` which will persist until a reboot. Make it permanent by running `sudo sh -c 'echo kernel.perf_event_paranoid=2 >> /etc/sysctl.d/local.conf'`
 
+#### Stats for SR-IOV devices
+
+When using virtualized GPUs via SR-IOV, additional args are needed for GPU stats to function. This can be enabled with the following config:
+
+```yaml
+telemetry:
+  stats:
+    sriov: True
+```
+
 ## AMD/ATI GPUs (Radeon HD 2000 and newer GPUs) via libva-mesa-driver
 
 VAAPI supports automatic profile selection so it will work automatically with both H.264 and H.265 streams.

docs/docs/configuration/license_plate_recognition.md

@@ -0,0 +1,45 @@
+---
+id: license_plate_recognition
+title: License Plate Recognition (LPR)
+---
+
+Frigate can recognize license plates on vehicles and automatically add the detected characters as a `sub_label` to objects that are of type `car`. A common use case may be to read the license plates of cars pulling into a driveway or cars passing by on a street with a dedicated LPR camera.
+
+Users running a Frigate+ model should ensure that `license_plate` is added to the [list of objects to track](https://docs.frigate.video/plus/#available-label-types) either globally or for a specific camera. This will improve the accuracy and performance of the LPR model.
+
+LPR is most effective when the vehicle’s license plate is fully visible to the camera. For moving vehicles, Frigate will attempt to read the plate continuously, refining its detection and keeping the most confident result. LPR will not run on stationary vehicles.
+
+## Minimum System Requirements
+
+License plate recognition works by running AI models locally on your system. The models are relatively lightweight and run on your CPU. At least 4GB of RAM is required.
+
+## Configuration
+
+License plate recognition is disabled by default. Enable it in your config file:
+
+```yaml
+lpr:
+  enabled: true
+```
+
+## Advanced Configuration
+
+Several options are available to fine-tune the LPR feature. For example, you can adjust the `min_area` setting, which defines the minimum size in pixels a license plate must be before LPR runs. The default is 500 pixels.
+
+Additionally, you can define `known_plates` as strings or regular expressions, allowing Frigate to label tracked vehicles with custom sub_labels when a recognized plate is detected. This information is then accessible in the UI, filters, and notifications.
+
+```yaml
+lpr:
+  enabled: true
+  min_area: 500
+  known_plates:
+    Wife's Car:
+      - "ABC-1234"
+      - "ABC-I234"
+    Johnny:
+      - "J*N-*234" # Using wildcards for H/M and 1/I
+    Sally:
+      - "[S5]LL-1234" # Matches SLL-1234 and 5LL-1234
+```
+
+In this example, "Wife's Car" will appear as the label for any vehicle matching the plate "ABC-1234." The model might occasionally interpret the digit 1 as a capital I (e.g., "ABC-I234"), so both variations are listed. Similarly, multiple possible variations are specified for Johnny and Sally.

docs/docs/configuration/object_detectors.md

@@ -550,7 +550,7 @@ Hardware accelerated object detection is supported on the following SoCs:
 - RK3576
 - RK3588
 
-This implementation uses the [Rockchip's RKNN-Toolkit2](https://github.com/airockchip/rknn-toolkit2/), version v2.0.0.beta0. Currently, only [Yolo-NAS](https://github.com/Deci-AI/super-gradients/blob/master/YOLONAS.md) is supported as object detection model.
+This implementation uses the [Rockchip's RKNN-Toolkit2](https://github.com/airockchip/rknn-toolkit2/), version v2.3.0. Currently, only [Yolo-NAS](https://github.com/Deci-AI/super-gradients/blob/master/YOLONAS.md) is supported as object detection model.
 
 ### Prerequisites
 
@@ -623,7 +623,41 @@ $ cat /sys/kernel/debug/rknpu/load
 :::
 
 - All models are automatically downloaded and stored in the folder `config/model_cache/rknn_cache`. After upgrading Frigate, you should remove older models to free up space.
-- You can also provide your own `.rknn` model. You should not save your own models in the `rknn_cache` folder, store them directly in the `model_cache` folder or another subfolder. To convert a model to `.rknn` format see the `rknn-toolkit2` (requires a x86 machine). Note, that there is only post-processing for the supported models.
+- You can also provide your own `.rknn` model. You should not save your own models in the `rknn_cache` folder, store them directly in the `model_cache` folder or another subfolder. To convert a model to `.rknn` format see the `rknn-toolkit2`. Note, that there is only post-processing for the supported models.
+
+### Converting your own onnx model to rknn format
+
+To convert a onnx model to the rknn format using the [rknn-toolkit2](https://github.com/airockchip/rknn-toolkit2/) you have to:
+
+- Place one ore more models in onnx format in the directory `config/model_cache/rknn_cache/onnx` on your docker host (this might require `sudo` privileges).
+- Save the configuration file under `config/conv2rknn.yaml` (see below for details).
+- Run `docker exec <frigate_container_id> python3 /opt/conv2rknn.py`. If the conversion was successful, the rknn models will be placed in `config/model_cache/rknn_cache`.
+
+This is an example configuration file that you need to adjust to your specific onnx model:
+
+```yaml
+soc: ["rk3562","rk3566", "rk3568", "rk3576", "rk3588"]
+quantization: false
+
+output_name: "{input_basename}"
+
+config:
+  mean_values: [[0, 0, 0]]
+  std_values: [[255, 255, 255]]
+  quant_img_rgb2bgr: true
+```
+
+Explanation of the paramters:
+
+- `soc`: A list of all SoCs you want to build the rknn model for. If you don't specify this parameter, the script tries to find out your SoC and builds the rknn model for this one.
+- `quantization`: true: 8 bit integer (i8) quantization, false: 16 bit float (fp16). Default: false.
+- `output_name`: The output name of the model. The following variables are available:
+  - `quant`: "i8" or "fp16" depending on the config
+  - `input_basename`: the basename of the input model (e.g. "my_model" if the input model is calles "my_model.onnx")
+  - `soc`: the SoC this model was build for (e.g. "rk3588")
+  - `tk_version`: Version of `rknn-toolkit2` (e.g. "2.3.0")
+  - **example**: Specifying `output_name = "frigate-{quant}-{input_basename}-{soc}-v{tk_version}"` could result in a model called `frigate-i8-my_model-rk3588-v2.3.0.rknn`.
+- `config`: Configuration passed to `rknn-toolkit2` for model conversion. For an explanation of all available parameters have a look at section "2.2. Model configuration" of [this manual](https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.3.0/03_Rockchip_RKNPU_API_Reference_RKNN_Toolkit2_V2.3.0_EN.pdf).
 
 ## Hailo-8l
 

docs/docs/configuration/reference.md

@@ -244,6 +244,8 @@ ffmpeg:
   # If set too high, then if a ffmpeg crash or camera stream timeout occurs, you could potentially lose up to a maximum of retry_interval second(s) of footage
   # NOTE: this can be a useful setting for Wireless / Battery cameras to reduce how much footage is potentially lost during a connection timeout.
   retry_interval: 10
+  # Optional: Set tag on HEVC (H.265) recording stream to improve compatibility with Apple players. (default: shown below)
+  apple_compatibility: false
 
 # Optional: Detect configuration
 # NOTE: Can be overridden at the camera level
@@ -524,6 +526,14 @@ semantic_search:
   # NOTE: small model runs on CPU and large model runs on GPU
   model_size: "small"
 
+# Optional: Configuration for face recognition capability
+face_recognition:
+  # Optional: Enable semantic search (default: shown below)
+  enabled: False
+  # Optional: Set the model size used for embeddings. (default: shown below)
+  # NOTE: small model runs on CPU and large model runs on GPU
+  model_size: "small"
+
 # Optional: Configuration for AI generated tracked object descriptions
 # NOTE: Semantic Search must be enabled for this to do anything.
 # WARNING: Depending on the provider, this will send thumbnails over the internet
@@ -805,11 +815,13 @@ telemetry:
     - lo
   # Optional: Configure system stats
   stats:
-    # Enable AMD GPU stats (default: shown below)
+    # Optional: Enable AMD GPU stats (default: shown below)
     amd_gpu_stats: True
-    # Enable Intel GPU stats (default: shown below)
+    # Optional: Enable Intel GPU stats (default: shown below)
     intel_gpu_stats: True
-    # Enable network bandwidth stats monitoring for camera ffmpeg processes, go2rtc, and object detectors. (default: shown below)
+    # Optional: Treat GPU as SR-IOV to fix GPU stats (default: shown below)
+    sriov: False
+    # Optional: Enable network bandwidth stats monitoring for camera ffmpeg processes, go2rtc, and object detectors. (default: shown below)
     # NOTE: The container must either be privileged or have cap_net_admin, cap_net_raw capabilities enabled.
     network_bandwidth: False
   # Optional: Enable the latest version outbound check (default: shown below)

docs/sidebars.ts

@@ -36,6 +36,8 @@ const sidebars: SidebarsConfig = {
       'Semantic Search': [
         'configuration/semantic_search',
         'configuration/genai',
+        'configuration/face_recognition',
+        'configuration/license_plate_recognition',
       ],
       Cameras: [
         'configuration/cameras',

frigate/__main__.py

@@ -3,12 +3,15 @@ import faulthandler
 import signal
 import sys
 import threading
+from typing import Union
 
+import ruamel.yaml
 from pydantic import ValidationError
 
 from frigate.app import FrigateApp
 from frigate.config import FrigateConfig
 from frigate.log import setup_logging
+from frigate.util.config import find_config_file
 
 
 def main() -> None:
@@ -42,10 +45,51 @@ def main() -> None:
         print("*************************************************************")
         print("*************************************************************")
         print("***    Config Validation Errors                           ***")
-        print("*************************************************************")
+        print("*************************************************************\n")
+        # Attempt to get the original config file for line number tracking
+        config_path = find_config_file()
+        with open(config_path, "r") as f:
+            yaml_config = ruamel.yaml.YAML()
+            yaml_config.preserve_quotes = True
+            full_config = yaml_config.load(f)
+
         for error in e.errors():
-            location = ".".join(str(item) for item in error["loc"])
-            print(f"{location}: {error['msg']}")
+            error_path = error["loc"]
+
+            current = full_config
+            line_number = "Unknown"
+            last_line_number = "Unknown"
+
+            try:
+                for i, part in enumerate(error_path):
+                    key: Union[int, str] = (
+                        int(part) if isinstance(part, str) and part.isdigit() else part
+                    )
+
+                    if isinstance(current, ruamel.yaml.comments.CommentedMap):
+                        current = current[key]
+                    elif isinstance(current, list):
+                        if isinstance(key, int):
+                            current = current[key]
+
+                    if hasattr(current, "lc"):
+                        last_line_number = current.lc.line
+
+                    if i == len(error_path) - 1:
+                        if hasattr(current, "lc"):
+                            line_number = current.lc.line
+                        else:
+                            line_number = last_line_number
+
+            except Exception as traverse_error:
+                print(f"Could not determine exact line number: {traverse_error}")
+
+            if current != full_config:
+                print(f"Line #  : {line_number}")
+                print(f"Key     : {' -> '.join(map(str, error_path))}")
+                print(f"Value   : {error.get('input', '-')}")
+            print(f"Message : {error.get('msg', error.get('type', 'Unknown'))}\n")
+
         print("*************************************************************")
         print("***    End Config Validation Errors                       ***")
         print("*************************************************************")

frigate/api/app.py

@@ -7,15 +7,18 @@ import os
 import traceback
 from datetime import datetime, timedelta
 from functools import reduce
+from io import StringIO
 from typing import Any, Optional
 
 import requests
+import ruamel.yaml
 from fastapi import APIRouter, Body, Path, Request, Response
 from fastapi.encoders import jsonable_encoder
 from fastapi.params import Depends
 from fastapi.responses import JSONResponse, PlainTextResponse
 from markupsafe import escape
 from peewee import operator
+from pydantic import ValidationError
 
 from frigate.api.defs.query.app_query_parameters import AppTimelineHourlyQueryParameters
 from frigate.api.defs.request.app_body import AppConfigSetBody
@@ -185,7 +188,6 @@ def config_raw():
 @router.post("/config/save")
 def config_save(save_option: str, body: Any = Body(media_type="text/plain")):
     new_config = body.decode()
-
     if not new_config:
         return JSONResponse(
             content=(
@@ -196,13 +198,64 @@ def config_save(save_option: str, body: Any = Body(media_type="text/plain")):
 
     # Validate the config schema
     try:
+        # Use ruamel to parse and preserve line numbers
+        yaml_config = ruamel.yaml.YAML()
+        yaml_config.preserve_quotes = True
+        full_config = yaml_config.load(StringIO(new_config))
+
         FrigateConfig.parse_yaml(new_config)
+
+    except ValidationError as e:
+        error_message = []
+
+        for error in e.errors():
+            error_path = error["loc"]
+            current = full_config
+            line_number = "Unknown"
+            last_line_number = "Unknown"
+
+            try:
+                for i, part in enumerate(error_path):
+                    key = int(part) if part.isdigit() else part
+
+                    if isinstance(current, ruamel.yaml.comments.CommentedMap):
+                        current = current[key]
+                    elif isinstance(current, list):
+                        current = current[key]
+
+                    if hasattr(current, "lc"):
+                        last_line_number = current.lc.line
+
+                    if i == len(error_path) - 1:
+                        if hasattr(current, "lc"):
+                            line_number = current.lc.line
+                        else:
+                            line_number = last_line_number
+
+            except Exception:
+                line_number = "Unable to determine"
+
+            error_message.append(
+                f"Line {line_number}: {' -> '.join(map(str, error_path))} - {error.get('msg', error.get('type', 'Unknown'))}"
+            )
+
+        return JSONResponse(
+            content=(
+                {
+                    "success": False,
+                    "message": "Your configuration is invalid.\nSee the official documentation at docs.frigate.video.\n\n"
+                    + "\n".join(error_message),
+                }
+            ),
+            status_code=400,
+        )
+
     except Exception:
         return JSONResponse(
             content=(
                 {
                     "success": False,
-                    "message": f"\nConfig Error:\n\n{escape(str(traceback.format_exc()))}",
+                    "message": f"\nYour configuration is invalid.\nSee the official documentation at docs.frigate.video.\n\n{escape(str(traceback.format_exc()))}",
                 }
             ),
             status_code=400,

frigate/api/classification.py

@@ -0,0 +1,127 @@
+"""Object classification APIs."""
+
+import logging
+import os
+import random
+import shutil
+import string
+
+from fastapi import APIRouter, Request, UploadFile
+from fastapi.responses import JSONResponse
+from pathvalidate import sanitize_filename
+
+from frigate.api.defs.tags import Tags
+from frigate.const import FACE_DIR
+from frigate.embeddings import EmbeddingsContext
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=[Tags.events])
+
+
+@router.get("/faces")
+def get_faces():
+    face_dict: dict[str, list[str]] = {}
+
+    for name in os.listdir(FACE_DIR):
+        face_dir = os.path.join(FACE_DIR, name)
+
+        if not os.path.isdir(face_dir):
+            continue
+
+        face_dict[name] = []
+
+        for file in sorted(
+            os.listdir(face_dir),
+            key=lambda f: os.path.getctime(os.path.join(face_dir, f)),
+            reverse=True,
+        ):
+            face_dict[name].append(file)
+
+    return JSONResponse(status_code=200, content=face_dict)
+
+
+@router.post("/faces/{name}")
+async def register_face(request: Request, name: str, file: UploadFile):
+    if not request.app.frigate_config.face_recognition.enabled:
+        return JSONResponse(
+            status_code=400,
+            content={"message": "Face recognition is not enabled.", "success": False},
+        )
+
+    context: EmbeddingsContext = request.app.embeddings
+    result = context.register_face(name, await file.read())
+    return JSONResponse(
+        status_code=200 if result.get("success", True) else 400,
+        content=result,
+    )
+
+
+@router.post("/faces/train/{name}/classify")
+def train_face(request: Request, name: str, body: dict = None):
+    if not request.app.frigate_config.face_recognition.enabled:
+        return JSONResponse(
+            status_code=400,
+            content={"message": "Face recognition is not enabled.", "success": False},
+        )
+
+    json: dict[str, any] = body or {}
+    training_file = os.path.join(
+        FACE_DIR, f"train/{sanitize_filename(json.get('training_file', ''))}"
+    )
+
+    if not training_file or not os.path.isfile(training_file):
+        return JSONResponse(
+            content=(
+                {
+                    "success": False,
+                    "message": f"Invalid filename or no file exists: {training_file}",
+                }
+            ),
+            status_code=404,
+        )
+
+    rand_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=6))
+    new_name = f"{name}-{rand_id}.webp"
+    new_file = os.path.join(FACE_DIR, f"{name}/{new_name}")
+    shutil.move(training_file, new_file)
+
+    context: EmbeddingsContext = request.app.embeddings
+    context.clear_face_classifier()
+
+    return JSONResponse(
+        content=(
+            {
+                "success": True,
+                "message": f"Successfully saved {training_file} as {new_name}.",
+            }
+        ),
+        status_code=200,
+    )
+
+
+@router.post("/faces/{name}/delete")
+def deregister_faces(request: Request, name: str, body: dict = None):
+    if not request.app.frigate_config.face_recognition.enabled:
+        return JSONResponse(
+            status_code=400,
+            content={"message": "Face recognition is not enabled.", "success": False},
+        )
+
+    json: dict[str, any] = body or {}
+    list_of_ids = json.get("ids", "")
+
+    if not list_of_ids or len(list_of_ids) == 0:
+        return JSONResponse(
+            content=({"success": False, "message": "Not a valid list of ids"}),
+            status_code=404,
+        )
+
+    context: EmbeddingsContext = request.app.embeddings
+    context.delete_face_ids(
+        name, map(lambda file: sanitize_filename(file), list_of_ids)
+    )
+    return JSONResponse(
+        content=({"success": True, "message": "Successfully deleted faces."}),
+        status_code=200,
+    )

frigate/api/defs/query/media_query_parameters.py

@@ -20,6 +20,7 @@ class MediaLatestFrameQueryParams(BaseModel):
     regions: Optional[int] = None
     quality: Optional[int] = 70
     height: Optional[int] = None
+    store: Optional[int] = None
 
 
 class MediaEventsSnapshotQueryParams(BaseModel):

frigate/api/defs/request/events_body.py

@@ -8,6 +8,9 @@ class EventsSubLabelBody(BaseModel):
     subLabelScore: Optional[float] = Field(
         title="Score for sub label", default=None, gt=0.0, le=1.0
     )
+    camera: Optional[str] = Field(
+        title="Camera this object is detected on.", default=None
+    )
 
 
 class EventsDescriptionBody(BaseModel):

frigate/api/defs/tags.py

@@ -10,4 +10,5 @@ class Tags(Enum):
     review = "Review"
     export = "Export"
     events = "Events"
+    classification = "classification"
     auth = "Auth"

frigate/api/event.py

@@ -909,38 +909,59 @@ def set_sub_label(
     try:
         event: Event = Event.get(Event.id == event_id)
     except DoesNotExist:
+        if not body.camera:
+            return JSONResponse(
+                content=(
+                    {
+                        "success": False,
+                        "message": "Event "
+                        + event_id
+                        + " not found and camera is not provided.",
+                    }
+                ),
+                status_code=404,
+            )
+
+        event = None
+
+    if request.app.detected_frames_processor:
+        tracked_obj: TrackedObject = (
+            request.app.detected_frames_processor.camera_states[
+                event.camera if event else body.camera
+            ].tracked_objects.get(event_id)
+        )
+    else:
+        tracked_obj = None
+
+    if not event and not tracked_obj:
         return JSONResponse(
-            content=({"success": False, "message": "Event " + event_id + " not found"}),
+            content=(
+                {"success": False, "message": "Event " + event_id + " not found."}
+            ),
             status_code=404,
         )
 
     new_sub_label = body.subLabel
     new_score = body.subLabelScore
 
-    if not event.end_time:
-        # update tracked object
-        tracked_obj: TrackedObject = (
-            request.app.detected_frames_processor.camera_states[
-                event.camera
-            ].tracked_objects.get(event.id)
-        )
-
-        if tracked_obj:
-            tracked_obj.obj_data["sub_label"] = (new_sub_label, new_score)
+    if tracked_obj:
+        tracked_obj.obj_data["sub_label"] = (new_sub_label, new_score)
 
         # update timeline items
         Timeline.update(
             data=Timeline.data.update({"sub_label": (new_sub_label, new_score)})
         ).where(Timeline.source_id == event_id).execute()
 
-    event.sub_label = new_sub_label
+    if event:
+        event.sub_label = new_sub_label
 
-    if new_score:
-        data = event.data
-        data["sub_label_score"] = new_score
-        event.data = data
+        if new_score:
+            data = event.data
+            data["sub_label_score"] = new_score
+            event.data = data
+
+        event.save()
 
-    event.save()
     return JSONResponse(
         content=(
             {

frigate/api/fastapi_app.py

@@ -11,7 +11,16 @@ from starlette_context import middleware, plugins
 from starlette_context.plugins import Plugin
 
 from frigate.api import app as main_app
-from frigate.api import auth, event, export, media, notification, preview, review
+from frigate.api import (
+    auth,
+    classification,
+    event,
+    export,
+    media,
+    notification,
+    preview,
+    review,
+)
 from frigate.api.auth import get_jwt_secret, limiter
 from frigate.comms.event_metadata_updater import (
     EventMetadataPublisher,
@@ -99,6 +108,7 @@ def create_fastapi_app(
     # Routes
     # Order of include_router matters: https://fastapi.tiangolo.com/tutorial/path-params/#order-matters
     app.include_router(auth.router)
+    app.include_router(classification.router)
     app.include_router(review.router)
     app.include_router(main_app.router)
     app.include_router(preview.router)

frigate/api/media.py

@@ -179,7 +179,12 @@ def latest_frame(
         return Response(
             content=img.tobytes(),
             media_type=f"image/{extension}",
-            headers={"Content-Type": f"image/{extension}", "Cache-Control": "no-store"},
+            headers={
+                "Content-Type": f"image/{extension}",
+                "Cache-Control": "no-store"
+                if not params.store
+                else "private, max-age=60",
+            },
         )
     elif camera_name == "birdseye" and request.app.frigate_config.birdseye.restream:
         frame = cv2.cvtColor(
@@ -198,7 +203,12 @@ def latest_frame(
         return Response(
             content=img.tobytes(),
             media_type=f"image/{extension}",
-            headers={"Content-Type": f"image/{extension}", "Cache-Control": "no-store"},
+            headers={
+                "Content-Type": f"image/{extension}",
+                "Cache-Control": "no-store"
+                if not params.store
+                else "private, max-age=60",
+            },
         )
     else:
         return JSONResponse(

frigate/app.py

@@ -34,10 +34,12 @@ from frigate.const import (
     CLIPS_DIR,
     CONFIG_DIR,
     EXPORT_DIR,
+    FACE_DIR,
     MODEL_CACHE_DIR,
     RECORD_DIR,
     SHM_FRAMES_VAR,
 )
+from frigate.data_processing.types import DataProcessorMetrics
 from frigate.db.sqlitevecq import SqliteVecQueueDatabase
 from frigate.embeddings import EmbeddingsContext, manage_embeddings
 from frigate.events.audio import AudioProcessor
@@ -88,6 +90,9 @@ class FrigateApp:
         self.detection_shms: list[mp.shared_memory.SharedMemory] = []
         self.log_queue: Queue = mp.Queue()
         self.camera_metrics: dict[str, CameraMetrics] = {}
+        self.embeddings_metrics: DataProcessorMetrics | None = (
+            DataProcessorMetrics() if config.semantic_search.enabled else None
+        )
         self.ptz_metrics: dict[str, PTZMetrics] = {}
         self.processes: dict[str, int] = {}
         self.embeddings: Optional[EmbeddingsContext] = None
@@ -96,14 +101,19 @@ class FrigateApp:
         self.config = config
 
     def ensure_dirs(self) -> None:
-        for d in [
+        dirs = [
             CONFIG_DIR,
             RECORD_DIR,
             f"{CLIPS_DIR}/cache",
             CACHE_DIR,
             MODEL_CACHE_DIR,
             EXPORT_DIR,
-        ]:
+        ]
+
+        if self.config.face_recognition.enabled:
+            dirs.append(FACE_DIR)
+
+        for d in dirs:
             if not os.path.exists(d) and not os.path.islink(d):
                 logger.info(f"Creating directory: {d}")
                 os.makedirs(d)
@@ -229,7 +239,10 @@ class FrigateApp:
         embedding_process = util.Process(
             target=manage_embeddings,
             name="embeddings_manager",
-            args=(self.config,),
+            args=(
+                self.config,
+                self.embeddings_metrics,
+            ),
         )
         embedding_process.daemon = True
         self.embedding_process = embedding_process
@@ -491,7 +504,11 @@ class FrigateApp:
         self.stats_emitter = StatsEmitter(
             self.config,
             stats_init(
-                self.config, self.camera_metrics, self.detectors, self.processes
+                self.config,
+                self.camera_metrics,
+                self.embeddings_metrics,
+                self.detectors,
+                self.processes,
             ),
             self.stop_event,
         )

frigate/camera/activity_manager.py

@@ -0,0 +1,130 @@
+"""Manage camera activity and updating listeners."""
+
+from collections import Counter
+from typing import Callable
+
+from frigate.config.config import FrigateConfig
+
+
+class CameraActivityManager:
+    def __init__(
+        self, config: FrigateConfig, publish: Callable[[str, any], None]
+    ) -> None:
+        self.config = config
+        self.publish = publish
+        self.last_camera_activity: dict[str, dict[str, any]] = {}
+        self.camera_all_object_counts: dict[str, Counter] = {}
+        self.camera_active_object_counts: dict[str, Counter] = {}
+        self.zone_all_object_counts: dict[str, Counter] = {}
+        self.zone_active_object_counts: dict[str, Counter] = {}
+        self.all_zone_labels: dict[str, set[str]] = {}
+
+        for camera_config in config.cameras.values():
+            if not camera_config.enabled:
+                continue
+
+            self.last_camera_activity[camera_config.name] = {}
+            self.camera_all_object_counts[camera_config.name] = Counter()
+            self.camera_active_object_counts[camera_config.name] = Counter()
+
+            for zone, zone_config in camera_config.zones.items():
+                if zone not in self.all_zone_labels:
+                    self.zone_all_object_counts[zone] = Counter()
+                    self.zone_active_object_counts[zone] = Counter()
+                    self.all_zone_labels[zone] = set()
+
+                self.all_zone_labels[zone].update(zone_config.objects)
+
+    def update_activity(self, new_activity: dict[str, dict[str, any]]) -> None:
+        all_objects: list[dict[str, any]] = []
+
+        for camera in new_activity.keys():
+            new_objects = new_activity[camera].get("objects", [])
+            all_objects.extend(new_objects)
+
+            if self.last_camera_activity.get(camera, {}).get("objects") != new_objects:
+                self.compare_camera_activity(camera, new_objects)
+
+        # run through every zone, getting a count of objects in that zone right now
+        for zone, labels in self.all_zone_labels.items():
+            all_zone_objects = Counter(
+                obj["label"].replace("-verified", "")
+                for obj in all_objects
+                if zone in obj["current_zones"]
+            )
+            active_zone_objects = Counter(
+                obj["label"].replace("-verified", "")
+                for obj in all_objects
+                if zone in obj["current_zones"] and not obj["stationary"]
+            )
+            any_changed = False
+
+            # run through each object and check what topics need to be updated for this zone
+            for label in labels:
+                new_count = all_zone_objects[label]
+                new_active_count = active_zone_objects[label]
+
+                if (
+                    new_count != self.zone_all_object_counts[zone][label]
+                    or label not in self.zone_all_object_counts[zone]
+                ):
+                    any_changed = True
+                    self.publish(f"{zone}/{label}", new_count)
+                    self.zone_all_object_counts[zone][label] = new_count
+
+                if (
+                    new_active_count != self.zone_active_object_counts[zone][label]
+                    or label not in self.zone_active_object_counts[zone]
+                ):
+                    any_changed = True
+                    self.publish(f"{zone}/{label}/active", new_active_count)
+                    self.zone_active_object_counts[zone][label] = new_active_count
+
+            if any_changed:
+                self.publish(f"{zone}/all", sum(list(all_zone_objects.values())))
+                self.publish(
+                    f"{zone}/all/active", sum(list(active_zone_objects.values()))
+                )
+
+        self.last_camera_activity = new_activity
+
+    def compare_camera_activity(
+        self, camera: str, new_activity: dict[str, any]
+    ) -> None:
+        all_objects = Counter(
+            obj["label"].replace("-verified", "") for obj in new_activity
+        )
+        active_objects = Counter(
+            obj["label"].replace("-verified", "")
+            for obj in new_activity
+            if not obj["stationary"]
+        )
+        any_changed = False
+
+        # run through each object and check what topics need to be updated
+        for label in self.config.cameras[camera].objects.track:
+            if label in self.config.model.all_attributes:
+                continue
+
+            new_count = all_objects[label]
+            new_active_count = active_objects[label]
+
+            if (
+                new_count != self.camera_all_object_counts[camera][label]
+                or label not in self.camera_all_object_counts[camera]
+            ):
+                any_changed = True
+                self.publish(f"{camera}/{label}", new_count)
+                self.camera_all_object_counts[camera][label] = new_count
+
+            if (
+                new_active_count != self.camera_active_object_counts[camera][label]
+                or label not in self.camera_active_object_counts[camera]
+            ):
+                any_changed = True
+                self.publish(f"{camera}/{label}/active", new_active_count)
+                self.camera_active_object_counts[camera][label] = new_active_count
+
+        if any_changed:
+            self.publish(f"{camera}/all", sum(list(all_objects.values())))
+            self.publish(f"{camera}/all/active", sum(list(active_objects.values())))

frigate/comms/dispatcher.py

@@ -7,6 +7,7 @@ from abc import ABC, abstractmethod
 from typing import Any, Callable, Optional
 
 from frigate.camera import PTZMetrics
+from frigate.camera.activity_manager import CameraActivityManager
 from frigate.comms.config_updater import ConfigPublisher
 from frigate.config import BirdseyeModeEnum, FrigateConfig
 from frigate.const import (
@@ -64,7 +65,7 @@ class Dispatcher:
         self.onvif = onvif
         self.ptz_metrics = ptz_metrics
         self.comms = communicators
-        self.camera_activity = {}
+        self.camera_activity = CameraActivityManager(config, self.publish)
         self.model_state = {}
         self.embeddings_reindex = {}
 
@@ -130,7 +131,7 @@ class Dispatcher:
             ).execute()
 
         def handle_update_camera_activity():
-            self.camera_activity = payload
+            self.camera_activity.update_activity(payload)
 
         def handle_update_event_description():
             event: Event = Event.get(Event.id == payload["id"])
@@ -171,7 +172,7 @@ class Dispatcher:
             )
 
         def handle_on_connect():
-            camera_status = self.camera_activity.copy()
+            camera_status = self.camera_activity.last_camera_activity.copy()
 
             for camera in camera_status.keys():
                 camera_status[camera]["config"] = {

frigate/comms/embeddings_updater.py

@@ -9,9 +9,11 @@ SOCKET_REP_REQ = "ipc:///tmp/cache/embeddings"
 
 
 class EmbeddingsRequestEnum(Enum):
+    clear_face_classifier = "clear_face_classifier"
     embed_description = "embed_description"
     embed_thumbnail = "embed_thumbnail"
     generate_search = "generate_search"
+    register_face = "register_face"
 
 
 class EmbeddingsResponder:
@@ -22,7 +24,7 @@ class EmbeddingsResponder:
 
     def check_for_request(self, process: Callable) -> None:
         while True:  # load all messages that are queued
-            has_message, _, _ = zmq.select([self.socket], [], [], 0.1)
+            has_message, _, _ = zmq.select([self.socket], [], [], 0.01)
 
             if not has_message:
                 break

frigate/config/__init__.py

@@ -3,13 +3,13 @@ from frigate.detectors import DetectorConfig, ModelConfig  # noqa: F401
 from .auth import *  # noqa: F403
 from .camera import *  # noqa: F403
 from .camera_group import *  # noqa: F403
+from .classification import *  # noqa: F403
 from .config import *  # noqa: F403
 from .database import *  # noqa: F403
 from .logger import *  # noqa: F403
 from .mqtt import *  # noqa: F403
 from .notification import *  # noqa: F403
 from .proxy import *  # noqa: F403
-from .semantic_search import *  # noqa: F403
 from .telemetry import *  # noqa: F403
 from .tls import *  # noqa: F403
 from .ui import *  # noqa: F403

frigate/config/camera/camera.py

@@ -167,7 +167,7 @@ class CameraConfig(FrigateBaseModel):
             record_args = get_ffmpeg_arg_list(
                 parse_preset_output_record(
                     self.ffmpeg.output_args.record,
-                    self.ffmpeg.output_args._force_record_hvc1,
+                    self.ffmpeg.apple_compatibility,
                 )
                 or self.ffmpeg.output_args.record
             )

frigate/config/camera/ffmpeg.py

@@ -2,7 +2,7 @@ import shutil
 from enum import Enum
 from typing import Union
 
-from pydantic import Field, PrivateAttr, field_validator
+from pydantic import Field, field_validator
 
 from frigate.const import DEFAULT_FFMPEG_VERSION, INCLUDED_FFMPEG_VERSIONS
 
@@ -42,7 +42,6 @@ class FfmpegOutputArgsConfig(FrigateBaseModel):
         default=RECORD_FFMPEG_OUTPUT_ARGS_DEFAULT,
         title="Record role FFmpeg output arguments.",
     )
-    _force_record_hvc1: bool = PrivateAttr(default=False)
 
 
 class FfmpegConfig(FrigateBaseModel):
@@ -64,6 +63,10 @@ class FfmpegConfig(FrigateBaseModel):
         default=10.0,
         title="Time in seconds to wait before FFmpeg retries connecting to the camera.",
     )
+    apple_compatibility: bool = Field(
+        default=False,
+        title="Set tag on HEVC (H.265) recording stream to improve compatibility with Apple players.",
+    )
 
     @property
     def ffmpeg_path(self) -> str:

frigate/config/camera/objects.py

@@ -1,6 +1,6 @@
 from typing import Any, Optional, Union
 
-from pydantic import Field, field_serializer
+from pydantic import Field, PrivateAttr, field_serializer
 
 from ..base import FrigateBaseModel
 
@@ -53,3 +53,20 @@ class ObjectConfig(FrigateBaseModel):
         default_factory=dict, title="Object filters."
     )
     mask: Union[str, list[str]] = Field(default="", title="Object mask.")
+    _all_objects: list[str] = PrivateAttr()
+
+    @property
+    def all_objects(self) -> list[str]:
+        return self._all_objects
+
+    def parse_all_objects(self, cameras):
+        if "_all_objects" in self:
+            return
+
+        # get list of unique enabled labels for tracking
+        enabled_labels = set(self.track)
+
+        for camera in cameras.values():
+            enabled_labels.update(camera.objects.track)
+
+        self._all_objects = list(enabled_labels)

frigate/config/classification.py

@@ -0,0 +1,74 @@
+from typing import Dict, List, Optional
+
+from pydantic import Field
+
+from .base import FrigateBaseModel
+
+__all__ = [
+    "FaceRecognitionConfig",
+    "SemanticSearchConfig",
+    "LicensePlateRecognitionConfig",
+]
+
+
+class BirdClassificationConfig(FrigateBaseModel):
+    enabled: bool = Field(default=False, title="Enable bird classification.")
+    threshold: float = Field(
+        default=0.9,
+        title="Minimum classification score required to be considered a match.",
+        gt=0.0,
+        le=1.0,
+    )
+
+
+class ClassificationConfig(FrigateBaseModel):
+    bird: BirdClassificationConfig = Field(
+        default_factory=BirdClassificationConfig, title="Bird classification config."
+    )
+
+
+class SemanticSearchConfig(FrigateBaseModel):
+    enabled: bool = Field(default=False, title="Enable semantic search.")
+    reindex: Optional[bool] = Field(
+        default=False, title="Reindex all detections on startup."
+    )
+    model_size: str = Field(
+        default="small", title="The size of the embeddings model used."
+    )
+
+
+class FaceRecognitionConfig(FrigateBaseModel):
+    enabled: bool = Field(default=False, title="Enable face recognition.")
+    min_score: float = Field(
+        title="Minimum face distance score required to save the attempt.",
+        default=0.8,
+        gt=0.0,
+        le=1.0,
+    )
+    threshold: float = Field(
+        default=0.9,
+        title="Minimum face distance score required to be considered a match.",
+        gt=0.0,
+        le=1.0,
+    )
+    min_area: int = Field(
+        default=500, title="Min area of face box to consider running face recognition."
+    )
+    save_attempts: bool = Field(
+        default=True, title="Save images of face detections for training."
+    )
+
+
+class LicensePlateRecognitionConfig(FrigateBaseModel):
+    enabled: bool = Field(default=False, title="Enable license plate recognition.")
+    threshold: float = Field(
+        default=0.9,
+        title="License plate confidence score required to be added to the object as a sub label.",
+    )
+    min_area: int = Field(
+        default=500,
+        title="Min area of license plate to consider running license plate recognition.",
+    )
+    known_plates: Optional[Dict[str, List[str]]] = Field(
+        default={}, title="Known plates to track."
+    )

frigate/config/config.py

@@ -51,13 +51,18 @@ from .camera.review import ReviewConfig
 from .camera.snapshots import SnapshotsConfig
 from .camera.timestamp import TimestampStyleConfig
 from .camera_group import CameraGroupConfig
+from .classification import (
+    ClassificationConfig,
+    FaceRecognitionConfig,
+    LicensePlateRecognitionConfig,
+    SemanticSearchConfig,
+)
 from .database import DatabaseConfig
 from .env import EnvVars
 from .logger import LoggerConfig
 from .mqtt import MqttConfig
 from .notification import NotificationConfig
 from .proxy import ProxyConfig
-from .semantic_search import SemanticSearchConfig
 from .telemetry import TelemetryConfig
 from .tls import TlsConfig
 from .ui import UIConfig
@@ -159,6 +164,16 @@ class RestreamConfig(BaseModel):
     model_config = ConfigDict(extra="allow")
 
 
+def verify_semantic_search_dependent_configs(config: FrigateConfig) -> None:
+    """Verify that semantic search is enabled if required features are enabled."""
+    if not config.semantic_search.enabled:
+        if config.genai.enabled:
+            raise ValueError("Genai requires semantic search to be enabled.")
+
+        if config.face_recognition.enabled:
+            raise ValueError("Face recognition requires semantic to be enabled.")
+
+
 def verify_config_roles(camera_config: CameraConfig) -> None:
     """Verify that roles are setup in the config correctly."""
     assigned_roles = list(
@@ -317,9 +332,19 @@ class FrigateConfig(FrigateBaseModel):
         default_factory=TelemetryConfig, title="Telemetry configuration."
     )
     tls: TlsConfig = Field(default_factory=TlsConfig, title="TLS configuration.")
+    classification: ClassificationConfig = Field(
+        default_factory=ClassificationConfig, title="Object classification config."
+    )
     semantic_search: SemanticSearchConfig = Field(
         default_factory=SemanticSearchConfig, title="Semantic search configuration."
     )
+    face_recognition: FaceRecognitionConfig = Field(
+        default_factory=FaceRecognitionConfig, title="Face recognition config."
+    )
+    lpr: LicensePlateRecognitionConfig = Field(
+        default_factory=LicensePlateRecognitionConfig,
+        title="License Plate recognition config.",
+    )
     ui: UIConfig = Field(default_factory=UIConfig, title="UI configuration.")
 
     # Detector config
@@ -437,13 +462,12 @@ class FrigateConfig(FrigateBaseModel):
                 camera_config.ffmpeg.hwaccel_args = self.ffmpeg.hwaccel_args
 
             for input in camera_config.ffmpeg.inputs:
-                need_record_fourcc = False and "record" in input.roles
                 need_detect_dimensions = "detect" in input.roles and (
                     camera_config.detect.height is None
                     or camera_config.detect.width is None
                 )
 
-                if need_detect_dimensions or need_record_fourcc:
+                if need_detect_dimensions:
                     stream_info = {"width": 0, "height": 0, "fourcc": None}
                     try:
                         stream_info = stream_info_retriever.get_stream_info(
@@ -467,14 +491,6 @@ class FrigateConfig(FrigateBaseModel):
                         else DEFAULT_DETECT_DIMENSIONS["height"]
                     )
 
-                if need_record_fourcc:
-                    # Apple only supports HEVC if it is hvc1 (vs. hev1)
-                    camera_config.ffmpeg.output_args._force_record_hvc1 = (
-                        stream_info["fourcc"] == "hevc"
-                        if stream_info.get("hevc")
-                        else False
-                    )
-
             # Warn if detect fps > 10
             if camera_config.detect.fps > 10:
                 logger.warning(
@@ -578,13 +594,8 @@ class FrigateConfig(FrigateBaseModel):
             verify_autotrack_zones(camera_config)
             verify_motion_and_detect(camera_config)
 
-        # get list of unique enabled labels for tracking
-        enabled_labels = set(self.objects.track)
-
-        for camera in self.cameras.values():
-            enabled_labels.update(camera.objects.track)
-
-        self.model.create_colormap(sorted(enabled_labels))
+        self.objects.parse_all_objects(self.cameras)
+        self.model.create_colormap(sorted(self.objects.all_objects))
         self.model.check_and_load_plus_model(self.plus_api)
 
         for key, detector in self.detectors.items():
@@ -617,6 +628,7 @@ class FrigateConfig(FrigateBaseModel):
             detector_config.model = model
             self.detectors[key] = detector_config
 
+        verify_semantic_search_dependent_configs(self)
         return self
 
     @field_validator("cameras")

frigate/config/logger.py

@@ -29,6 +29,7 @@ class LoggerConfig(FrigateBaseModel):
             logging.getLogger().setLevel(self.default.value.upper())
 
             log_levels = {
+                "httpx": LogLevel.error,
                 "werkzeug": LogLevel.error,
                 "ws4py": LogLevel.error,
                 **self.logs,

frigate/config/semantic_search.py

@@ -1,17 +0,0 @@
-from typing import Optional
-
-from pydantic import Field
-
-from .base import FrigateBaseModel
-
-__all__ = ["SemanticSearchConfig"]
-
-
-class SemanticSearchConfig(FrigateBaseModel):
-    enabled: bool = Field(default=False, title="Enable semantic search.")
-    reindex: Optional[bool] = Field(
-        default=False, title="Reindex all detections on startup."
-    )
-    model_size: str = Field(
-        default="small", title="The size of the embeddings model used."
-    )

frigate/config/telemetry.py

@@ -11,6 +11,9 @@ class StatsConfig(FrigateBaseModel):
     network_bandwidth: bool = Field(
         default=False, title="Enable network bandwidth for ffmpeg processes."
     )
+    sriov: bool = Field(
+        default=False, title="Treat device as SR-IOV to support GPU stats."
+    )
 
 
 class TelemetryConfig(FrigateBaseModel):

frigate/const.py

@@ -5,8 +5,9 @@ DEFAULT_DB_PATH = f"{CONFIG_DIR}/frigate.db"
 MODEL_CACHE_DIR = f"{CONFIG_DIR}/model_cache"
 BASE_DIR = "/media/frigate"
 CLIPS_DIR = f"{BASE_DIR}/clips"
-RECORD_DIR = f"{BASE_DIR}/recordings"
 EXPORT_DIR = f"{BASE_DIR}/exports"
+FACE_DIR = f"{CLIPS_DIR}/faces"
+RECORD_DIR = f"{BASE_DIR}/recordings"
 BIRDSEYE_PIPE = "/tmp/cache/birdseye"
 CACHE_DIR = "/tmp/cache"
 FRIGATE_LOCALHOST = "http://127.0.0.1:5000"
@@ -64,6 +65,7 @@ INCLUDED_FFMPEG_VERSIONS = ["7.0", "5.0"]
 FFMPEG_HWACCEL_NVIDIA = "preset-nvidia"
 FFMPEG_HWACCEL_VAAPI = "preset-vaapi"
 FFMPEG_HWACCEL_VULKAN = "preset-vulkan"
+FFMPEG_HVC1_ARGS = ["-tag:v", "hvc1"]
 
 # Regex constants
 

frigate/data_processing/post/api.py

@@ -0,0 +1,43 @@
+"""Local or remote processors to handle post processing."""
+
+import logging
+from abc import ABC, abstractmethod
+
+from frigate.config import FrigateConfig
+
+from ..types import DataProcessorMetrics, PostProcessDataEnum
+
+logger = logging.getLogger(__name__)
+
+
+class PostProcessorApi(ABC):
+    @abstractmethod
+    def __init__(self, config: FrigateConfig, metrics: DataProcessorMetrics) -> None:
+        self.config = config
+        self.metrics = metrics
+        pass
+
+    @abstractmethod
+    def process_data(
+        self, data: dict[str, any], data_type: PostProcessDataEnum
+    ) -> None:
+        """Processes the data of data type.
+        Args:
+            data (dict): containing data about the input.
+            data_type (enum): Describing the data that is being processed.
+
+        Returns:
+            None.
+        """
+        pass
+
+    @abstractmethod
+    def handle_request(self, request_data: dict[str, any]) -> dict[str, any] | None:
+        """Handle metadata requests.
+        Args:
+            request_data (dict): containing data about requested change to process.
+
+        Returns:
+            None if request was not handled, otherwise return response.
+        """
+        pass

frigate/data_processing/real_time/api.py

@@ -0,0 +1,57 @@
+"""Local only processors for handling real time object processing."""
+
+import logging
+from abc import ABC, abstractmethod
+
+import numpy as np
+
+from frigate.config import FrigateConfig
+
+from ..types import DataProcessorMetrics
+
+logger = logging.getLogger(__name__)
+
+
+class RealTimeProcessorApi(ABC):
+    @abstractmethod
+    def __init__(self, config: FrigateConfig, metrics: DataProcessorMetrics) -> None:
+        self.config = config
+        self.metrics = metrics
+        pass
+
+    @abstractmethod
+    def process_frame(self, obj_data: dict[str, any], frame: np.ndarray) -> None:
+        """Processes the frame with object data.
+        Args:
+            obj_data (dict): containing data about focused object in frame.
+            frame (ndarray): full yuv frame.
+
+        Returns:
+            None.
+        """
+        pass
+
+    @abstractmethod
+    def handle_request(
+        self, topic: str, request_data: dict[str, any]
+    ) -> dict[str, any] | None:
+        """Handle metadata requests.
+        Args:
+            topic (str): topic that dictates what work is requested.
+            request_data (dict): containing data about requested change to process.
+
+        Returns:
+            None if request was not handled, otherwise return response.
+        """
+        pass
+
+    @abstractmethod
+    def expire_object(self, object_id: str) -> None:
+        """Handle objects that are no longer detected.
+        Args:
+            object_id (str): id of object that is no longer detected.
+
+        Returns:
+            None.
+        """
+        pass

frigate/data_processing/real_time/bird_processor.py

@@ -0,0 +1,154 @@
+"""Handle processing images to classify birds."""
+
+import logging
+import os
+
+import cv2
+import numpy as np
+import requests
+
+from frigate.config import FrigateConfig
+from frigate.const import FRIGATE_LOCALHOST, MODEL_CACHE_DIR
+from frigate.util.object import calculate_region
+
+from ..types import DataProcessorMetrics
+from .api import RealTimeProcessorApi
+
+try:
+    from tflite_runtime.interpreter import Interpreter
+except ModuleNotFoundError:
+    from tensorflow.lite.python.interpreter import Interpreter
+
+logger = logging.getLogger(__name__)
+
+
+class BirdProcessor(RealTimeProcessorApi):
+    def __init__(self, config: FrigateConfig, metrics: DataProcessorMetrics):
+        super().__init__(config, metrics)
+        self.interpreter: Interpreter = None
+        self.tensor_input_details: dict[str, any] = None
+        self.tensor_output_details: dict[str, any] = None
+        self.detected_birds: dict[str, float] = {}
+        self.labelmap: dict[int, str] = {}
+
+        download_path = os.path.join(MODEL_CACHE_DIR, "bird")
+        self.model_files = {
+            "bird.tflite": "https://raw.githubusercontent.com/google-coral/test_data/master/mobilenet_v2_1.0_224_inat_bird_quant.tflite",
+            "birdmap.txt": "https://raw.githubusercontent.com/google-coral/test_data/master/inat_bird_labels.txt",
+        }
+
+        if not all(
+            os.path.exists(os.path.join(download_path, n))
+            for n in self.model_files.keys()
+        ):
+            # conditionally import ModelDownloader
+            from frigate.util.downloader import ModelDownloader
+
+            self.downloader = ModelDownloader(
+                model_name="bird",
+                download_path=download_path,
+                file_names=self.model_files.keys(),
+                download_func=self.__download_models,
+                complete_func=self.__build_detector,
+            )
+            self.downloader.ensure_model_files()
+        else:
+            self.__build_detector()
+
+    def __download_models(self, path: str) -> None:
+        try:
+            file_name = os.path.basename(path)
+
+            # conditionally import ModelDownloader
+            from frigate.util.downloader import ModelDownloader
+
+            ModelDownloader.download_from_url(self.model_files[file_name], path)
+        except Exception as e:
+            logger.error(f"Failed to download {path}: {e}")
+
+    def __build_detector(self) -> None:
+        self.interpreter = Interpreter(
+            model_path=os.path.join(MODEL_CACHE_DIR, "bird/bird.tflite"),
+            num_threads=2,
+        )
+        self.interpreter.allocate_tensors()
+        self.tensor_input_details = self.interpreter.get_input_details()
+        self.tensor_output_details = self.interpreter.get_output_details()
+
+        i = 0
+
+        with open(os.path.join(MODEL_CACHE_DIR, "bird/birdmap.txt")) as f:
+            line = f.readline()
+            while line:
+                start = line.find("(")
+                end = line.find(")")
+                self.labelmap[i] = line[start + 1 : end]
+                i += 1
+                line = f.readline()
+
+    def process_frame(self, obj_data, frame):
+        if obj_data["label"] != "bird":
+            return
+
+        x, y, x2, y2 = calculate_region(
+            frame.shape,
+            obj_data["box"][0],
+            obj_data["box"][1],
+            obj_data["box"][2],
+            obj_data["box"][3],
+            224,
+            1.0,
+        )
+
+        rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420)
+        input = rgb[
+            y:y2,
+            x:x2,
+        ]
+
+        cv2.imwrite("/media/frigate/test_class.png", input)
+
+        input = np.expand_dims(input, axis=0)
+
+        self.interpreter.set_tensor(self.tensor_input_details[0]["index"], input)
+        self.interpreter.invoke()
+        res: np.ndarray = self.interpreter.get_tensor(
+            self.tensor_output_details[0]["index"]
+        )[0]
+        probs = res / res.sum(axis=0)
+        best_id = np.argmax(probs)
+
+        if best_id == 964:
+            logger.debug("No bird classification was detected.")
+            return
+
+        score = round(probs[best_id], 2)
+
+        if score < self.config.classification.bird.threshold:
+            logger.debug(f"Score {score} is not above required threshold")
+            return
+
+        previous_score = self.detected_birds.get(obj_data["id"], 0.0)
+
+        if score <= previous_score:
+            logger.debug(f"Score {score} is worse than previous score {previous_score}")
+            return
+
+        resp = requests.post(
+            f"{FRIGATE_LOCALHOST}/api/events/{obj_data['id']}/sub_label",
+            json={
+                "camera": obj_data.get("camera"),
+                "subLabel": self.labelmap[best_id],
+                "subLabelScore": score,
+            },
+        )
+
+        if resp.status_code == 200:
+            self.detected_birds[obj_data["id"]] = score
+
+    def handle_request(self, topic, request_data):
+        return None
+
+    def expire_object(self, object_id):
+        if object_id in self.detected_birds:
+            self.detected_birds.pop(object_id)

frigate/data_processing/real_time/face_processor.py

@@ -0,0 +1,406 @@
+"""Handle processing images for face detection and recognition."""
+
+import base64
+import datetime
+import logging
+import os
+import random
+import string
+from typing import Optional
+
+import cv2
+import numpy as np
+import requests
+
+from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
+from frigate.config import FrigateConfig
+from frigate.const import FACE_DIR, FRIGATE_LOCALHOST, MODEL_CACHE_DIR
+from frigate.util.image import area
+
+from ..types import DataProcessorMetrics
+from .api import RealTimeProcessorApi
+
+logger = logging.getLogger(__name__)
+
+
+MIN_MATCHING_FACES = 2
+
+
+class FaceProcessor(RealTimeProcessorApi):
+    def __init__(self, config: FrigateConfig, metrics: DataProcessorMetrics):
+        super().__init__(config, metrics)
+        self.face_config = config.face_recognition
+        self.face_detector: cv2.FaceDetectorYN = None
+        self.landmark_detector: cv2.face.FacemarkLBF = None
+        self.face_recognizer: cv2.face.LBPHFaceRecognizer = None
+        self.requires_face_detection = "face" not in self.config.objects.all_objects
+        self.detected_faces: dict[str, float] = {}
+
+        download_path = os.path.join(MODEL_CACHE_DIR, "facedet")
+        self.model_files = {
+            "facedet.onnx": "https://github.com/NickM-27/facenet-onnx/releases/download/v1.0/facedet.onnx",
+            "landmarkdet.yaml": "https://github.com/NickM-27/facenet-onnx/releases/download/v1.0/landmarkdet.yaml",
+        }
+
+        if not all(
+            os.path.exists(os.path.join(download_path, n))
+            for n in self.model_files.keys()
+        ):
+            # conditionally import ModelDownloader
+            from frigate.util.downloader import ModelDownloader
+
+            self.downloader = ModelDownloader(
+                model_name="facedet",
+                download_path=download_path,
+                file_names=self.model_files.keys(),
+                download_func=self.__download_models,
+                complete_func=self.__build_detector,
+            )
+            self.downloader.ensure_model_files()
+        else:
+            self.__build_detector()
+
+        self.label_map: dict[int, str] = {}
+        self.__build_classifier()
+
+    def __download_models(self, path: str) -> None:
+        try:
+            file_name = os.path.basename(path)
+            # conditionally import ModelDownloader
+            from frigate.util.downloader import ModelDownloader
+
+            ModelDownloader.download_from_url(self.model_files[file_name], path)
+        except Exception as e:
+            logger.error(f"Failed to download {path}: {e}")
+
+    def __build_detector(self) -> None:
+        self.face_detector = cv2.FaceDetectorYN.create(
+            "/config/model_cache/facedet/facedet.onnx",
+            config="",
+            input_size=(320, 320),
+            score_threshold=0.8,
+            nms_threshold=0.3,
+        )
+        self.landmark_detector = cv2.face.createFacemarkLBF()
+        self.landmark_detector.loadModel("/config/model_cache/facedet/landmarkdet.yaml")
+
+    def __build_classifier(self) -> None:
+        if not self.landmark_detector:
+            return None
+
+        labels = []
+        faces = []
+
+        dir = "/media/frigate/clips/faces"
+        for idx, name in enumerate(os.listdir(dir)):
+            if name == "train":
+                continue
+
+            face_folder = os.path.join(dir, name)
+
+            if not os.path.isdir(face_folder):
+                continue
+
+            self.label_map[idx] = name
+            for image in os.listdir(face_folder):
+                img = cv2.imread(os.path.join(face_folder, image))
+
+                if img is None:
+                    continue
+
+                img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+                img = self.__align_face(img, img.shape[1], img.shape[0])
+                faces.append(img)
+                labels.append(idx)
+
+        self.recognizer: cv2.face.LBPHFaceRecognizer = (
+            cv2.face.LBPHFaceRecognizer_create(
+                radius=2, threshold=(1 - self.face_config.min_score) * 1000
+            )
+        )
+        self.recognizer.train(faces, np.array(labels))
+
+    def __align_face(
+        self,
+        image: np.ndarray,
+        output_width: int,
+        output_height: int,
+    ) -> np.ndarray:
+        _, lands = self.landmark_detector.fit(
+            image, np.array([(0, 0, image.shape[1], image.shape[0])])
+        )
+        landmarks: np.ndarray = lands[0][0]
+
+        # get landmarks for eyes
+        leftEyePts = landmarks[42:48]
+        rightEyePts = landmarks[36:42]
+
+        # compute the center of mass for each eye
+        leftEyeCenter = leftEyePts.mean(axis=0).astype("int")
+        rightEyeCenter = rightEyePts.mean(axis=0).astype("int")
+
+        # compute the angle between the eye centroids
+        dY = rightEyeCenter[1] - leftEyeCenter[1]
+        dX = rightEyeCenter[0] - leftEyeCenter[0]
+        angle = np.degrees(np.arctan2(dY, dX)) - 180
+
+        # compute the desired right eye x-coordinate based on the
+        # desired x-coordinate of the left eye
+        desiredRightEyeX = 1.0 - 0.35
+
+        # determine the scale of the new resulting image by taking
+        # the ratio of the distance between eyes in the *current*
+        # image to the ratio of distance between eyes in the
+        # *desired* image
+        dist = np.sqrt((dX**2) + (dY**2))
+        desiredDist = desiredRightEyeX - 0.35
+        desiredDist *= output_width
+        scale = desiredDist / dist
+
+        # compute center (x, y)-coordinates (i.e., the median point)
+        # between the two eyes in the input image
+        # grab the rotation matrix for rotating and scaling the face
+        eyesCenter = (
+            int((leftEyeCenter[0] + rightEyeCenter[0]) // 2),
+            int((leftEyeCenter[1] + rightEyeCenter[1]) // 2),
+        )
+        M = cv2.getRotationMatrix2D(eyesCenter, angle, scale)
+
+        # update the translation component of the matrix
+        tX = output_width * 0.5
+        tY = output_height * 0.35
+        M[0, 2] += tX - eyesCenter[0]
+        M[1, 2] += tY - eyesCenter[1]
+
+        # apply the affine transformation
+        return cv2.warpAffine(
+            image, M, (output_width, output_height), flags=cv2.INTER_CUBIC
+        )
+
+    def __clear_classifier(self) -> None:
+        self.face_recognizer = None
+        self.label_map = {}
+
+    def __detect_face(self, input: np.ndarray) -> tuple[int, int, int, int]:
+        """Detect faces in input image."""
+        if not self.face_detector:
+            return None
+
+        self.face_detector.setInputSize((input.shape[1], input.shape[0]))
+        faces = self.face_detector.detect(input)
+
+        if faces is None or faces[1] is None:
+            return None
+
+        face = None
+
+        for _, potential_face in enumerate(faces[1]):
+            raw_bbox = potential_face[0:4].astype(np.uint16)
+            x: int = max(raw_bbox[0], 0)
+            y: int = max(raw_bbox[1], 0)
+            w: int = raw_bbox[2]
+            h: int = raw_bbox[3]
+            bbox = (x, y, x + w, y + h)
+
+            if face is None or area(bbox) > area(face):
+                face = bbox
+
+        return face
+
+    def __classify_face(self, face_image: np.ndarray) -> tuple[str, float] | None:
+        if not self.landmark_detector:
+            return None
+
+        if not self.label_map:
+            self.__build_classifier()
+
+        img = cv2.cvtColor(face_image, cv2.COLOR_BGR2GRAY)
+        img = self.__align_face(img, img.shape[1], img.shape[0])
+        index, distance = self.recognizer.predict(img)
+
+        if index == -1:
+            return None
+
+        score = 1.0 - (distance / 1000)
+        return self.label_map[index], round(score, 2)
+
+    def __update_metrics(self, duration: float) -> None:
+        self.metrics.face_rec_fps.value = (
+            self.metrics.face_rec_fps.value * 9 + duration
+        ) / 10
+
+    def process_frame(self, obj_data: dict[str, any], frame: np.ndarray):
+        """Look for faces in image."""
+        start = datetime.datetime.now().timestamp()
+        id = obj_data["id"]
+
+        # don't run for non person objects
+        if obj_data.get("label") != "person":
+            logger.debug("Not a processing face for non person object.")
+            return
+
+        # don't overwrite sub label for objects that have a sub label
+        # that is not a face
+        if obj_data.get("sub_label") and id not in self.detected_faces:
+            logger.debug(
+                f"Not processing face due to existing sub label: {obj_data.get('sub_label')}."
+            )
+            return
+
+        face: Optional[dict[str, any]] = None
+
+        if self.requires_face_detection:
+            logger.debug("Running manual face detection.")
+            person_box = obj_data.get("box")
+
+            if not person_box:
+                return
+
+            rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420)
+            left, top, right, bottom = person_box
+            person = rgb[top:bottom, left:right]
+            face_box = self.__detect_face(person)
+
+            if not face_box:
+                logger.debug("Detected no faces for person object.")
+                return
+
+            face_frame = person[
+                max(0, face_box[1]) : min(frame.shape[0], face_box[3]),
+                max(0, face_box[0]) : min(frame.shape[1], face_box[2]),
+            ]
+            face_frame = cv2.cvtColor(face_frame, cv2.COLOR_RGB2BGR)
+        else:
+            # don't run for object without attributes
+            if not obj_data.get("current_attributes"):
+                logger.debug("No attributes to parse.")
+                return
+
+            attributes: list[dict[str, any]] = obj_data.get("current_attributes", [])
+            for attr in attributes:
+                if attr.get("label") != "face":
+                    continue
+
+                if face is None or attr.get("score", 0.0) > face.get("score", 0.0):
+                    face = attr
+
+            # no faces detected in this frame
+            if not face:
+                return
+
+            face_box = face.get("box")
+
+            # check that face is valid
+            if not face_box or area(face_box) < self.config.face_recognition.min_area:
+                logger.debug(f"Invalid face box {face}")
+                return
+
+            face_frame = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
+
+            face_frame = face_frame[
+                max(0, face_box[1]) : min(frame.shape[0], face_box[3]),
+                max(0, face_box[0]) : min(frame.shape[1], face_box[2]),
+            ]
+
+        res = self.__classify_face(face_frame)
+
+        if not res:
+            return
+
+        sub_label, score = res
+
+        # calculate the overall face score as the probability * area of face
+        # this will help to reduce false positives from small side-angle faces
+        # if a large front-on face image may have scored slightly lower but
+        # is more likely to be accurate due to the larger face area
+        face_score = round(score * face_frame.shape[0] * face_frame.shape[1], 2)
+
+        logger.debug(
+            f"Detected best face for person as: {sub_label} with probability {score} and overall face score {face_score}"
+        )
+
+        if self.config.face_recognition.save_attempts:
+            # write face to library
+            folder = os.path.join(FACE_DIR, "train")
+            file = os.path.join(folder, f"{id}-{sub_label}-{score}-{face_score}.webp")
+            os.makedirs(folder, exist_ok=True)
+            cv2.imwrite(file, face_frame)
+
+        if score < self.config.face_recognition.threshold:
+            logger.debug(
+                f"Recognized face distance {score} is less than threshold {self.config.face_recognition.threshold}"
+            )
+            self.__update_metrics(datetime.datetime.now().timestamp() - start)
+            return
+
+        if id in self.detected_faces and face_score <= self.detected_faces[id]:
+            logger.debug(
+                f"Recognized face distance {score} and overall score {face_score} is less than previous overall face score ({self.detected_faces.get(id)})."
+            )
+            self.__update_metrics(datetime.datetime.now().timestamp() - start)
+            return
+
+        resp = requests.post(
+            f"{FRIGATE_LOCALHOST}/api/events/{id}/sub_label",
+            json={
+                "camera": obj_data.get("camera"),
+                "subLabel": sub_label,
+                "subLabelScore": score,
+            },
+        )
+
+        if resp.status_code == 200:
+            self.detected_faces[id] = face_score
+
+        self.__update_metrics(datetime.datetime.now().timestamp() - start)
+
+    def handle_request(self, topic, request_data) -> dict[str, any] | None:
+        if topic == EmbeddingsRequestEnum.clear_face_classifier.value:
+            self.__clear_classifier()
+        elif topic == EmbeddingsRequestEnum.register_face.value:
+            rand_id = "".join(
+                random.choices(string.ascii_lowercase + string.digits, k=6)
+            )
+            label = request_data["face_name"]
+            id = f"{label}-{rand_id}"
+
+            if request_data.get("cropped"):
+                thumbnail = request_data["image"]
+            else:
+                img = cv2.imdecode(
+                    np.frombuffer(
+                        base64.b64decode(request_data["image"]), dtype=np.uint8
+                    ),
+                    cv2.IMREAD_COLOR,
+                )
+                face_box = self.__detect_face(img)
+
+                if not face_box:
+                    return {
+                        "message": "No face was detected.",
+                        "success": False,
+                    }
+
+                face = img[face_box[1] : face_box[3], face_box[0] : face_box[2]]
+                _, thumbnail = cv2.imencode(
+                    ".webp", face, [int(cv2.IMWRITE_WEBP_QUALITY), 100]
+                )
+
+            # write face to library
+            folder = os.path.join(FACE_DIR, label)
+            file = os.path.join(folder, f"{id}.webp")
+            os.makedirs(folder, exist_ok=True)
+
+            # save face image
+            with open(file, "wb") as output:
+                output.write(thumbnail.tobytes())
+
+            self.__clear_classifier()
+            return {
+                "message": "Successfully registered face.",
+                "success": True,
+            }
+
+    def expire_object(self, object_id: str):
+        if object_id in self.detected_faces:
+            self.detected_faces.pop(object_id)

frigate/data_processing/types.py

@@ -0,0 +1,24 @@
+"""Embeddings types."""
+
+import multiprocessing as mp
+from enum import Enum
+from multiprocessing.sharedctypes import Synchronized
+
+
+class DataProcessorMetrics:
+    image_embeddings_fps: Synchronized
+    text_embeddings_sps: Synchronized
+    face_rec_fps: Synchronized
+    alpr_pps: Synchronized
+
+    def __init__(self):
+        self.image_embeddings_fps = mp.Value("d", 0.01)
+        self.text_embeddings_sps = mp.Value("d", 0.01)
+        self.face_rec_fps = mp.Value("d", 0.01)
+        self.alpr_pps = mp.Value("d", 0.01)
+
+
+class PostProcessDataEnum(str, Enum):
+    recording = "recording"
+    review = "review"
+    tracked_object = "tracked_object"

frigate/detectors/plugins/rknn.py

@@ -108,7 +108,7 @@ class Rknn(DetectionApi):
                     model_props["model_type"] = model_type
 
             if model_matched:
-                model_props["filename"] = model_path + f"-{soc}-v2.0.0-1.rknn"
+                model_props["filename"] = model_path + f"-{soc}-v2.3.0-1.rknn"
 
                 model_props["path"] = model_cache_dir + model_props["filename"]
 
@@ -129,7 +129,7 @@ class Rknn(DetectionApi):
             os.mkdir(model_cache_dir)
 
         urllib.request.urlretrieve(
-            f"https://github.com/MarcA711/rknn-models/releases/download/v2.0.0/{filename}",
+            f"https://github.com/MarcA711/rknn-models/releases/download/v2.3.0/{filename}",
             model_cache_dir + filename,
         )
 

frigate/embeddings/__init__.py

@@ -1,5 +1,6 @@
 """SQLite-vec embeddings database."""
 
+import base64
 import json
 import logging
 import multiprocessing as mp
@@ -13,7 +14,8 @@ from setproctitle import setproctitle
 
 from frigate.comms.embeddings_updater import EmbeddingsRequestEnum, EmbeddingsRequestor
 from frigate.config import FrigateConfig
-from frigate.const import CONFIG_DIR
+from frigate.const import CONFIG_DIR, FACE_DIR
+from frigate.data_processing.types import DataProcessorMetrics
 from frigate.db.sqlitevecq import SqliteVecQueueDatabase
 from frigate.models import Event
 from frigate.util.builtin import serialize
@@ -25,7 +27,7 @@ from .util import ZScoreNormalization
 logger = logging.getLogger(__name__)
 
 
-def manage_embeddings(config: FrigateConfig) -> None:
+def manage_embeddings(config: FrigateConfig, metrics: DataProcessorMetrics) -> None:
     # Only initialize embeddings if semantic search is enabled
     if not config.semantic_search.enabled:
         return
@@ -59,6 +61,7 @@ def manage_embeddings(config: FrigateConfig) -> None:
     maintainer = EmbeddingMaintainer(
         db,
         config,
+        metrics,
         stop_event,
     )
     maintainer.start()
@@ -189,6 +192,38 @@ class EmbeddingsContext:
 
         return results
 
+    def register_face(self, face_name: str, image_data: bytes) -> dict[str, any]:
+        return self.requestor.send_data(
+            EmbeddingsRequestEnum.register_face.value,
+            {
+                "face_name": face_name,
+                "image": base64.b64encode(image_data).decode("ASCII"),
+            },
+        )
+
+    def get_face_ids(self, name: str) -> list[str]:
+        sql_query = f"""
+            SELECT
+                id
+            FROM vec_descriptions
+            WHERE id LIKE '%{name}%'
+        """
+
+        return self.db.execute_sql(sql_query).fetchall()
+
+    def clear_face_classifier(self) -> None:
+        self.requestor.send_data(
+            EmbeddingsRequestEnum.clear_face_classifier.value, None
+        )
+
+    def delete_face_ids(self, face: str, ids: list[str]) -> None:
+        folder = os.path.join(FACE_DIR, face)
+        for id in ids:
+            file_path = os.path.join(folder, id)
+
+            if os.path.isfile(file_path):
+                os.unlink(file_path)
+
     def update_description(self, event_id: str, description: str) -> None:
         self.requestor.send_data(
             EmbeddingsRequestEnum.embed_description.value,

frigate/embeddings/embeddings.py

@@ -1,6 +1,7 @@
 """SQLite-vec embeddings database."""
 
 import base64
+import datetime
 import logging
 import os
 import time
@@ -9,12 +10,13 @@ from numpy import ndarray
 from playhouse.shortcuts import model_to_dict
 
 from frigate.comms.inter_process import InterProcessRequestor
-from frigate.config.semantic_search import SemanticSearchConfig
+from frigate.config import FrigateConfig
 from frigate.const import (
     CONFIG_DIR,
     UPDATE_EMBEDDINGS_REINDEX_PROGRESS,
     UPDATE_MODEL_STATE,
 )
+from frigate.data_processing.types import DataProcessorMetrics
 from frigate.db.sqlitevecq import SqliteVecQueueDatabase
 from frigate.models import Event
 from frigate.types import ModelStatusTypesEnum
@@ -60,10 +62,14 @@ class Embeddings:
     """SQLite-vec embeddings database."""
 
     def __init__(
-        self, config: SemanticSearchConfig, db: SqliteVecQueueDatabase
+        self,
+        config: FrigateConfig,
+        db: SqliteVecQueueDatabase,
+        metrics: DataProcessorMetrics,
     ) -> None:
         self.config = config
         self.db = db
+        self.metrics = metrics
         self.requestor = InterProcessRequestor()
 
         # Create tables if they don't exist
@@ -73,9 +79,13 @@ class Embeddings:
             "jinaai/jina-clip-v1-text_model_fp16.onnx",
             "jinaai/jina-clip-v1-tokenizer",
             "jinaai/jina-clip-v1-vision_model_fp16.onnx"
-            if config.model_size == "large"
+            if config.semantic_search.model_size == "large"
             else "jinaai/jina-clip-v1-vision_model_quantized.onnx",
             "jinaai/jina-clip-v1-preprocessor_config.json",
+            "facenet-facenet.onnx",
+            "paddleocr-onnx-detection.onnx",
+            "paddleocr-onnx-classification.onnx",
+            "paddleocr-onnx-recognition.onnx",
         ]
 
         for model in models:
@@ -94,7 +104,7 @@ class Embeddings:
             download_urls={
                 "text_model_fp16.onnx": "https://huggingface.co/jinaai/jina-clip-v1/resolve/main/onnx/text_model_fp16.onnx",
             },
-            model_size=config.model_size,
+            model_size=config.semantic_search.model_size,
             model_type=ModelTypeEnum.text,
             requestor=self.requestor,
             device="CPU",
@@ -102,7 +112,7 @@ class Embeddings:
 
         model_file = (
             "vision_model_fp16.onnx"
-            if self.config.model_size == "large"
+            if self.config.semantic_search.model_size == "large"
             else "vision_model_quantized.onnx"
         )
 
@@ -115,12 +125,53 @@ class Embeddings:
             model_name="jinaai/jina-clip-v1",
             model_file=model_file,
             download_urls=download_urls,
-            model_size=config.model_size,
+            model_size=config.semantic_search.model_size,
             model_type=ModelTypeEnum.vision,
             requestor=self.requestor,
-            device="GPU" if config.model_size == "large" else "CPU",
+            device="GPU" if config.semantic_search.model_size == "large" else "CPU",
         )
 
+        self.lpr_detection_model = None
+        self.lpr_classification_model = None
+        self.lpr_recognition_model = None
+
+        if self.config.lpr.enabled:
+            self.lpr_detection_model = GenericONNXEmbedding(
+                model_name="paddleocr-onnx",
+                model_file="detection.onnx",
+                download_urls={
+                    "detection.onnx": "https://github.com/hawkeye217/paddleocr-onnx/raw/refs/heads/master/models/detection.onnx"
+                },
+                model_size="large",
+                model_type=ModelTypeEnum.lpr_detect,
+                requestor=self.requestor,
+                device="CPU",
+            )
+
+            self.lpr_classification_model = GenericONNXEmbedding(
+                model_name="paddleocr-onnx",
+                model_file="classification.onnx",
+                download_urls={
+                    "classification.onnx": "https://github.com/hawkeye217/paddleocr-onnx/raw/refs/heads/master/models/classification.onnx"
+                },
+                model_size="large",
+                model_type=ModelTypeEnum.lpr_classify,
+                requestor=self.requestor,
+                device="CPU",
+            )
+
+            self.lpr_recognition_model = GenericONNXEmbedding(
+                model_name="paddleocr-onnx",
+                model_file="recognition.onnx",
+                download_urls={
+                    "recognition.onnx": "https://github.com/hawkeye217/paddleocr-onnx/raw/refs/heads/master/models/recognition.onnx"
+                },
+                model_size="large",
+                model_type=ModelTypeEnum.lpr_recognize,
+                requestor=self.requestor,
+                device="CPU",
+            )
+
     def embed_thumbnail(
         self, event_id: str, thumbnail: bytes, upsert: bool = True
     ) -> ndarray:
@@ -130,6 +181,7 @@ class Embeddings:
         @param: thumbnail bytes in jpg format
         @param: upsert If embedding should be upserted into vec DB
         """
+        start = datetime.datetime.now().timestamp()
         # Convert thumbnail bytes to PIL Image
         embedding = self.vision_embedding([thumbnail])[0]
 
@@ -142,6 +194,11 @@ class Embeddings:
                 (event_id, serialize(embedding)),
             )
 
+        duration = datetime.datetime.now().timestamp() - start
+        self.metrics.image_embeddings_fps.value = (
+            self.metrics.image_embeddings_fps.value * 9 + duration
+        ) / 10
+
         return embedding
 
     def batch_embed_thumbnail(
@@ -152,6 +209,7 @@ class Embeddings:
         @param: event_thumbs Map of Event IDs in DB to thumbnail bytes in jpg format
         @param: upsert If embedding should be upserted into vec DB
         """
+        start = datetime.datetime.now().timestamp()
         ids = list(event_thumbs.keys())
         embeddings = self.vision_embedding(list(event_thumbs.values()))
 
@@ -170,11 +228,17 @@ class Embeddings:
                 items,
             )
 
+        duration = datetime.datetime.now().timestamp() - start
+        self.metrics.text_embeddings_sps.value = (
+            self.metrics.text_embeddings_sps.value * 9 + (duration / len(ids))
+        ) / 10
+
         return embeddings
 
     def embed_description(
         self, event_id: str, description: str, upsert: bool = True
     ) -> ndarray:
+        start = datetime.datetime.now().timestamp()
         embedding = self.text_embedding([description])[0]
 
         if upsert:
@@ -186,11 +250,17 @@ class Embeddings:
                 (event_id, serialize(embedding)),
             )
 
+        duration = datetime.datetime.now().timestamp() - start
+        self.metrics.text_embeddings_sps.value = (
+            self.metrics.text_embeddings_sps.value * 9 + duration
+        ) / 10
+
         return embedding
 
     def batch_embed_description(
         self, event_descriptions: dict[str, str], upsert: bool = True
     ) -> ndarray:
+        start = datetime.datetime.now().timestamp()
         # upsert embeddings one by one to avoid token limit
         embeddings = []
 
@@ -213,6 +283,11 @@ class Embeddings:
                 items,
             )
 
+        duration = datetime.datetime.now().timestamp() - start
+        self.metrics.text_embeddings_sps.value = (
+            self.metrics.text_embeddings_sps.value * 9 + (duration / len(ids))
+        ) / 10
+
         return embeddings
 
     def reindex(self) -> None:

frigate/embeddings/functions/onnx.py

@@ -31,11 +31,16 @@ warnings.filterwarnings(
 disable_progress_bar()
 logger = logging.getLogger(__name__)
 
+FACE_EMBEDDING_SIZE = 160
+
 
 class ModelTypeEnum(str, Enum):
     face = "face"
     vision = "vision"
     text = "text"
+    lpr_detect = "lpr_detect"
+    lpr_classify = "lpr_classify"
+    lpr_recognize = "lpr_recognize"
 
 
 class GenericONNXEmbedding:
@@ -47,7 +52,7 @@ class GenericONNXEmbedding:
         model_file: str,
         download_urls: Dict[str, str],
         model_size: str,
-        model_type: str,
+        model_type: ModelTypeEnum,
         requestor: InterProcessRequestor,
         tokenizer_file: Optional[str] = None,
         device: str = "AUTO",
@@ -57,7 +62,7 @@ class GenericONNXEmbedding:
         self.tokenizer_file = tokenizer_file
         self.requestor = requestor
         self.download_urls = download_urls
-        self.model_type = model_type  # 'text' or 'vision'
+        self.model_type = model_type
         self.model_size = model_size
         self.device = device
         self.download_path = os.path.join(MODEL_CACHE_DIR, self.model_name)
@@ -87,12 +92,13 @@ class GenericONNXEmbedding:
                 files_names,
                 ModelStatusTypesEnum.downloaded,
             )
-            self._load_model_and_tokenizer()
+            self._load_model_and_utils()
             logger.debug(f"models are already downloaded for {self.model_name}")
 
     def _download_model(self, path: str):
         try:
             file_name = os.path.basename(path)
+
             if file_name in self.download_urls:
                 ModelDownloader.download_from_url(self.download_urls[file_name], path)
             elif (
@@ -101,6 +107,7 @@ class GenericONNXEmbedding:
             ):
                 if not os.path.exists(path + "/" + self.model_name):
                     logger.info(f"Downloading {self.model_name} tokenizer")
+
                 tokenizer = AutoTokenizer.from_pretrained(
                     self.model_name,
                     trust_remote_code=True,
@@ -125,14 +132,23 @@ class GenericONNXEmbedding:
                 },
             )
 
-    def _load_model_and_tokenizer(self):
+    def _load_model_and_utils(self):
         if self.runner is None:
             if self.downloader:
                 self.downloader.wait_for_download()
             if self.model_type == ModelTypeEnum.text:
                 self.tokenizer = self._load_tokenizer()
-            else:
+            elif self.model_type == ModelTypeEnum.vision:
                 self.feature_extractor = self._load_feature_extractor()
+            elif self.model_type == ModelTypeEnum.face:
+                self.feature_extractor = []
+            elif self.model_type == ModelTypeEnum.lpr_detect:
+                self.feature_extractor = []
+            elif self.model_type == ModelTypeEnum.lpr_classify:
+                self.feature_extractor = []
+            elif self.model_type == ModelTypeEnum.lpr_recognize:
+                self.feature_extractor = []
+
             self.runner = ONNXModelRunner(
                 os.path.join(self.download_path, self.model_file),
                 self.device,
@@ -172,23 +188,72 @@ class GenericONNXEmbedding:
                 self.feature_extractor(images=image, return_tensors="np")
                 for image in processed_images
             ]
+        elif self.model_type == ModelTypeEnum.face:
+            if isinstance(raw_inputs, list):
+                raise ValueError("Face embedding does not support batch inputs.")
+
+            pil = self._process_image(raw_inputs)
+
+            # handle images larger than input size
+            width, height = pil.size
+            if width != FACE_EMBEDDING_SIZE or height != FACE_EMBEDDING_SIZE:
+                if width > height:
+                    new_height = int(((height / width) * FACE_EMBEDDING_SIZE) // 4 * 4)
+                    pil = pil.resize((FACE_EMBEDDING_SIZE, new_height))
+                else:
+                    new_width = int(((width / height) * FACE_EMBEDDING_SIZE) // 4 * 4)
+                    pil = pil.resize((new_width, FACE_EMBEDDING_SIZE))
+
+            og = np.array(pil).astype(np.float32)
+
+            # Image must be FACE_EMBEDDING_SIZExFACE_EMBEDDING_SIZE
+            og_h, og_w, channels = og.shape
+            frame = np.full(
+                (FACE_EMBEDDING_SIZE, FACE_EMBEDDING_SIZE, channels),
+                (0, 0, 0),
+                dtype=np.float32,
+            )
+
+            # compute center offset
+            x_center = (FACE_EMBEDDING_SIZE - og_w) // 2
+            y_center = (FACE_EMBEDDING_SIZE - og_h) // 2
+
+            # copy img image into center of result image
+            frame[y_center : y_center + og_h, x_center : x_center + og_w] = og
+            frame = np.expand_dims(frame, axis=0)
+            return [{"input_2": frame}]
+        elif self.model_type == ModelTypeEnum.lpr_detect:
+            preprocessed = []
+            for x in raw_inputs:
+                preprocessed.append(x)
+            return [{"x": preprocessed[0]}]
+        elif self.model_type == ModelTypeEnum.lpr_classify:
+            processed = []
+            for img in raw_inputs:
+                processed.append({"x": img})
+            return processed
+        elif self.model_type == ModelTypeEnum.lpr_recognize:
+            processed = []
+            for img in raw_inputs:
+                processed.append({"x": img})
+            return processed
         else:
             raise ValueError(f"Unable to preprocess inputs for {self.model_type}")
 
-    def _process_image(self, image):
+    def _process_image(self, image, output: str = "RGB") -> Image.Image:
         if isinstance(image, str):
             if image.startswith("http"):
                 response = requests.get(image)
-                image = Image.open(BytesIO(response.content)).convert("RGB")
+                image = Image.open(BytesIO(response.content)).convert(output)
         elif isinstance(image, bytes):
-            image = Image.open(BytesIO(image)).convert("RGB")
+            image = Image.open(BytesIO(image)).convert(output)
 
         return image
 
     def __call__(
         self, inputs: Union[List[str], List[Image.Image], List[str]]
     ) -> List[np.ndarray]:
-        self._load_model_and_tokenizer()
+        self._load_model_and_utils()
         if self.runner is None or (
             self.tokenizer is None and self.feature_extractor is None
         ):

frigate/embeddings/lpr/lpr.py

@@ -0,0 +1,808 @@
+import logging
+import math
+from typing import List, Tuple
+
+import cv2
+import numpy as np
+from pyclipper import ET_CLOSEDPOLYGON, JT_ROUND, PyclipperOffset
+from shapely.geometry import Polygon
+
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config.classification import LicensePlateRecognitionConfig
+from frigate.embeddings.embeddings import Embeddings
+
+logger = logging.getLogger(__name__)
+
+MIN_PLATE_LENGTH = 3
+
+
+class LicensePlateRecognition:
+    def __init__(
+        self,
+        config: LicensePlateRecognitionConfig,
+        requestor: InterProcessRequestor,
+        embeddings: Embeddings,
+    ):
+        self.lpr_config = config
+        self.requestor = requestor
+        self.embeddings = embeddings
+        self.detection_model = self.embeddings.lpr_detection_model
+        self.classification_model = self.embeddings.lpr_classification_model
+        self.recognition_model = self.embeddings.lpr_recognition_model
+        self.ctc_decoder = CTCDecoder()
+
+        self.batch_size = 6
+
+        # Detection specific parameters
+        self.min_size = 3
+        self.max_size = 960
+        self.box_thresh = 0.8
+        self.mask_thresh = 0.8
+
+        if self.lpr_config.enabled:
+            # all models need to be loaded to run LPR
+            self.detection_model._load_model_and_utils()
+            self.classification_model._load_model_and_utils()
+            self.recognition_model._load_model_and_utils()
+
+    def detect(self, image: np.ndarray) -> List[np.ndarray]:
+        """
+        Detect possible license plates in the input image by first resizing and normalizing it,
+        running a detection model, and filtering out low-probability regions.
+
+        Args:
+            image (np.ndarray): The input image in which license plates will be detected.
+
+        Returns:
+            List[np.ndarray]: A list of bounding box coordinates representing detected license plates.
+        """
+        h, w = image.shape[:2]
+
+        if sum([h, w]) < 64:
+            image = self.zero_pad(image)
+
+        resized_image = self.resize_image(image)
+        normalized_image = self.normalize_image(resized_image)
+
+        outputs = self.detection_model([normalized_image])[0]
+        outputs = outputs[0, :, :]
+
+        boxes, _ = self.boxes_from_bitmap(outputs, outputs > self.mask_thresh, w, h)
+        return self.filter_polygon(boxes, (h, w))
+
+    def classify(
+        self, images: List[np.ndarray]
+    ) -> Tuple[List[np.ndarray], List[Tuple[str, float]]]:
+        """
+        Classify the orientation or category of each detected license plate.
+
+        Args:
+            images (List[np.ndarray]): A list of images of detected license plates.
+
+        Returns:
+            Tuple[List[np.ndarray], List[Tuple[str, float]]]: A tuple of rotated/normalized plate images
+                                                            and classification results with confidence scores.
+        """
+        num_images = len(images)
+        indices = np.argsort([x.shape[1] / x.shape[0] for x in images])
+
+        for i in range(0, num_images, self.batch_size):
+            norm_images = []
+            for j in range(i, min(num_images, i + self.batch_size)):
+                norm_img = self._preprocess_classification_image(images[indices[j]])
+                norm_img = norm_img[np.newaxis, :]
+                norm_images.append(norm_img)
+
+        outputs = self.classification_model(norm_images)
+
+        return self._process_classification_output(images, outputs)
+
+    def recognize(
+        self, images: List[np.ndarray]
+    ) -> Tuple[List[str], List[List[float]]]:
+        """
+        Recognize the characters on the detected license plates using the recognition model.
+
+        Args:
+            images (List[np.ndarray]): A list of images of license plates to recognize.
+
+        Returns:
+            Tuple[List[str], List[List[float]]]: A tuple of recognized license plate texts and confidence scores.
+        """
+        input_shape = [3, 48, 320]
+        num_images = len(images)
+
+        # sort images by aspect ratio for processing
+        indices = np.argsort(np.array([x.shape[1] / x.shape[0] for x in images]))
+
+        for index in range(0, num_images, self.batch_size):
+            input_h, input_w = input_shape[1], input_shape[2]
+            max_wh_ratio = input_w / input_h
+            norm_images = []
+
+            # calculate the maximum aspect ratio in the current batch
+            for i in range(index, min(num_images, index + self.batch_size)):
+                h, w = images[indices[i]].shape[0:2]
+                max_wh_ratio = max(max_wh_ratio, w * 1.0 / h)
+
+            # preprocess the images based on the max aspect ratio
+            for i in range(index, min(num_images, index + self.batch_size)):
+                norm_image = self._preprocess_recognition_image(
+                    images[indices[i]], max_wh_ratio
+                )
+                norm_image = norm_image[np.newaxis, :]
+                norm_images.append(norm_image)
+
+        outputs = self.recognition_model(norm_images)
+        return self.ctc_decoder(outputs)
+
+    def process_license_plate(
+        self, image: np.ndarray
+    ) -> Tuple[List[str], List[float], List[int]]:
+        """
+        Complete pipeline for detecting, classifying, and recognizing license plates in the input image.
+
+        Args:
+            image (np.ndarray): The input image in which to detect, classify, and recognize license plates.
+
+        Returns:
+            Tuple[List[str], List[float], List[int]]: Detected license plate texts, confidence scores, and areas of the plates.
+        """
+        if (
+            self.detection_model.runner is None
+            or self.classification_model.runner is None
+            or self.recognition_model.runner is None
+        ):
+            # we might still be downloading the models
+            logger.debug("Model runners not loaded")
+            return [], [], []
+
+        plate_points = self.detect(image)
+        if len(plate_points) == 0:
+            return [], [], []
+
+        plate_points = self.sort_polygon(list(plate_points))
+        plate_images = [self._crop_license_plate(image, x) for x in plate_points]
+        rotated_images, _ = self.classify(plate_images)
+
+        # keep track of the index of each image for correct area calc later
+        sorted_indices = np.argsort([x.shape[1] / x.shape[0] for x in rotated_images])
+        reverse_mapping = {
+            idx: original_idx for original_idx, idx in enumerate(sorted_indices)
+        }
+
+        results, confidences = self.recognize(rotated_images)
+
+        if results:
+            license_plates = [""] * len(rotated_images)
+            average_confidences = [[0.0]] * len(rotated_images)
+            areas = [0] * len(rotated_images)
+
+            # map results back to original image order
+            for i, (plate, conf) in enumerate(zip(results, confidences)):
+                original_idx = reverse_mapping[i]
+
+                height, width = rotated_images[original_idx].shape[:2]
+                area = height * width
+
+                average_confidence = conf
+
+                # set to True to write each cropped image for debugging
+                if False:
+                    save_image = cv2.cvtColor(
+                        rotated_images[original_idx], cv2.COLOR_RGB2BGR
+                    )
+                    filename = f"/config/plate_{original_idx}_{plate}_{area}.jpg"
+                    cv2.imwrite(filename, save_image)
+
+                license_plates[original_idx] = plate
+                average_confidences[original_idx] = average_confidence
+                areas[original_idx] = area
+
+            # Filter out plates that have a length of less than 3 characters
+            # Sort by area, then by plate length, then by confidence all desc
+            sorted_data = sorted(
+                [
+                    (plate, conf, area)
+                    for plate, conf, area in zip(
+                        license_plates, average_confidences, areas
+                    )
+                    if len(plate) >= MIN_PLATE_LENGTH
+                ],
+                key=lambda x: (x[2], len(x[0]), x[1]),
+                reverse=True,
+            )
+
+            if sorted_data:
+                return map(list, zip(*sorted_data))
+
+        return [], [], []
+
+    def resize_image(self, image: np.ndarray) -> np.ndarray:
+        """
+        Resize the input image while maintaining the aspect ratio, ensuring dimensions are multiples of 32.
+
+        Args:
+            image (np.ndarray): The input image to resize.
+
+        Returns:
+            np.ndarray: The resized image.
+        """
+        h, w = image.shape[:2]
+        ratio = min(self.max_size / max(h, w), 1.0)
+        resize_h = max(int(round(int(h * ratio) / 32) * 32), 32)
+        resize_w = max(int(round(int(w * ratio) / 32) * 32), 32)
+        return cv2.resize(image, (resize_w, resize_h))
+
+    def normalize_image(self, image: np.ndarray) -> np.ndarray:
+        """
+        Normalize the input image by subtracting the mean and multiplying by the standard deviation.
+
+        Args:
+            image (np.ndarray): The input image to normalize.
+
+        Returns:
+            np.ndarray: The normalized image, transposed to match the model's expected input format.
+        """
+        mean = np.array([123.675, 116.28, 103.53]).reshape(1, -1).astype("float64")
+        std = 1 / np.array([58.395, 57.12, 57.375]).reshape(1, -1).astype("float64")
+
+        image = image.astype("float32")
+        cv2.subtract(image, mean, image)
+        cv2.multiply(image, std, image)
+        return image.transpose((2, 0, 1))[np.newaxis, ...]
+
+    def boxes_from_bitmap(
+        self, output: np.ndarray, mask: np.ndarray, dest_width: int, dest_height: int
+    ) -> Tuple[np.ndarray, List[float]]:
+        """
+        Process the binary mask to extract bounding boxes and associated confidence scores.
+
+        Args:
+            output (np.ndarray): Output confidence map from the model.
+            mask (np.ndarray): Binary mask of detected regions.
+            dest_width (int): Target width for scaling the box coordinates.
+            dest_height (int): Target height for scaling the box coordinates.
+
+        Returns:
+            Tuple[np.ndarray, List[float]]: Array of bounding boxes and list of corresponding scores.
+        """
+
+        mask = (mask * 255).astype(np.uint8)
+        height, width = mask.shape
+        outs = cv2.findContours(mask, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
+
+        # handle different return values of findContours between OpenCV versions
+        contours = outs[0] if len(outs) == 2 else outs[1]
+
+        boxes = []
+        scores = []
+
+        for index in range(len(contours)):
+            contour = contours[index]
+
+            # get minimum bounding box (rotated rectangle) around the contour and the smallest side length.
+            points, min_side = self.get_min_boxes(contour)
+
+            if min_side < self.min_size:
+                continue
+
+            points = np.array(points)
+
+            score = self.box_score(output, contour)
+            if self.box_thresh > score:
+                continue
+
+            polygon = Polygon(points)
+            distance = polygon.area / polygon.length
+
+            # Use pyclipper to shrink the polygon slightly based on the computed distance.
+            offset = PyclipperOffset()
+            offset.AddPath(points, JT_ROUND, ET_CLOSEDPOLYGON)
+            points = np.array(offset.Execute(distance * 1.5)).reshape((-1, 1, 2))
+
+            # get the minimum bounding box around the shrunken polygon.
+            box, min_side = self.get_min_boxes(points)
+
+            if min_side < self.min_size + 2:
+                continue
+
+            box = np.array(box)
+
+            # normalize and clip box coordinates to fit within the destination image size.
+            box[:, 0] = np.clip(np.round(box[:, 0] / width * dest_width), 0, dest_width)
+            box[:, 1] = np.clip(
+                np.round(box[:, 1] / height * dest_height), 0, dest_height
+            )
+
+            boxes.append(box.astype("int32"))
+            scores.append(score)
+
+        return np.array(boxes, dtype="int32"), scores
+
+    @staticmethod
+    def get_min_boxes(contour: np.ndarray) -> Tuple[List[Tuple[float, float]], float]:
+        """
+        Calculate the minimum bounding box (rotated rectangle) for a given contour.
+
+        Args:
+            contour (np.ndarray): The contour points of the detected shape.
+
+        Returns:
+            Tuple[List[Tuple[float, float]], float]: A list of four points representing the
+            corners of the bounding box, and the length of the shortest side.
+        """
+        bounding_box = cv2.minAreaRect(contour)
+        points = sorted(cv2.boxPoints(bounding_box), key=lambda x: x[0])
+        index_1, index_4 = (0, 1) if points[1][1] > points[0][1] else (1, 0)
+        index_2, index_3 = (2, 3) if points[3][1] > points[2][1] else (3, 2)
+        box = [points[index_1], points[index_2], points[index_3], points[index_4]]
+        return box, min(bounding_box[1])
+
+    @staticmethod
+    def box_score(bitmap: np.ndarray, contour: np.ndarray) -> float:
+        """
+        Calculate the average score within the bounding box of a contour.
+
+        Args:
+            bitmap (np.ndarray): The output confidence map from the model.
+            contour (np.ndarray): The contour of the detected shape.
+
+        Returns:
+            float: The average score of the pixels inside the contour region.
+        """
+        h, w = bitmap.shape[:2]
+        contour = contour.reshape(-1, 2)
+        x1, y1 = np.clip(contour.min(axis=0), 0, [w - 1, h - 1])
+        x2, y2 = np.clip(contour.max(axis=0), 0, [w - 1, h - 1])
+        mask = np.zeros((y2 - y1 + 1, x2 - x1 + 1), dtype=np.uint8)
+        cv2.fillPoly(mask, [contour - [x1, y1]], 1)
+        return cv2.mean(bitmap[y1 : y2 + 1, x1 : x2 + 1], mask)[0]
+
+    @staticmethod
+    def expand_box(points: List[Tuple[float, float]]) -> np.ndarray:
+        """
+        Expand a polygonal shape slightly by a factor determined by the area-to-perimeter ratio.
+
+        Args:
+            points (List[Tuple[float, float]]): Points of the polygon to expand.
+
+        Returns:
+            np.ndarray: Expanded polygon points.
+        """
+        polygon = Polygon(points)
+        distance = polygon.area / polygon.length
+        offset = PyclipperOffset()
+        offset.AddPath(points, JT_ROUND, ET_CLOSEDPOLYGON)
+        expanded = np.array(offset.Execute(distance * 1.5)).reshape((-1, 2))
+        return expanded
+
+    def filter_polygon(
+        self, points: List[np.ndarray], shape: Tuple[int, int]
+    ) -> np.ndarray:
+        """
+        Filter a set of polygons to include only valid ones that fit within an image shape
+        and meet size constraints.
+
+        Args:
+            points (List[np.ndarray]): List of polygons to filter.
+            shape (Tuple[int, int]): Shape of the image (height, width).
+
+        Returns:
+            np.ndarray: List of filtered polygons.
+        """
+        height, width = shape
+        return np.array(
+            [
+                self.clockwise_order(point)
+                for point in points
+                if self.is_valid_polygon(point, width, height)
+            ]
+        )
+
+    @staticmethod
+    def is_valid_polygon(point: np.ndarray, width: int, height: int) -> bool:
+        """
+        Check if a polygon is valid, meaning it fits within the image bounds
+        and has sides of a minimum length.
+
+        Args:
+            point (np.ndarray): The polygon to validate.
+            width (int): Image width.
+            height (int): Image height.
+
+        Returns:
+            bool: Whether the polygon is valid or not.
+        """
+        return (
+            point[:, 0].min() >= 0
+            and point[:, 0].max() < width
+            and point[:, 1].min() >= 0
+            and point[:, 1].max() < height
+            and np.linalg.norm(point[0] - point[1]) > 3
+            and np.linalg.norm(point[0] - point[3]) > 3
+        )
+
+    @staticmethod
+    def clockwise_order(point: np.ndarray) -> np.ndarray:
+        """
+        Arrange the points of a polygon in clockwise order based on their angular positions
+        around the polygon's center.
+
+        Args:
+            point (np.ndarray): Array of points of the polygon.
+
+        Returns:
+            np.ndarray: Points ordered in clockwise direction.
+        """
+        center = point.mean(axis=0)
+        return point[
+            np.argsort(np.arctan2(point[:, 1] - center[1], point[:, 0] - center[0]))
+        ]
+
+    @staticmethod
+    def sort_polygon(points):
+        """
+        Sort polygons based on their position in the image. If polygons are close in vertical
+        position (within 10 pixels), sort them by horizontal position.
+
+        Args:
+            points: List of polygons to sort.
+
+        Returns:
+            List: Sorted list of polygons.
+        """
+        points.sort(key=lambda x: (x[0][1], x[0][0]))
+        for i in range(len(points) - 1):
+            for j in range(i, -1, -1):
+                if abs(points[j + 1][0][1] - points[j][0][1]) < 10 and (
+                    points[j + 1][0][0] < points[j][0][0]
+                ):
+                    temp = points[j]
+                    points[j] = points[j + 1]
+                    points[j + 1] = temp
+                else:
+                    break
+        return points
+
+    @staticmethod
+    def zero_pad(image: np.ndarray) -> np.ndarray:
+        """
+        Apply zero-padding to an image, ensuring its dimensions are at least 32x32.
+        The padding is added only if needed.
+
+        Args:
+            image (np.ndarray): Input image.
+
+        Returns:
+            np.ndarray: Zero-padded image.
+        """
+        h, w, c = image.shape
+        pad = np.zeros((max(32, h), max(32, w), c), np.uint8)
+        pad[:h, :w, :] = image
+        return pad
+
+    @staticmethod
+    def _preprocess_classification_image(image: np.ndarray) -> np.ndarray:
+        """
+        Preprocess a single image for classification by resizing, normalizing, and padding.
+
+        This method resizes the input image to a fixed height of 48 pixels while adjusting
+        the width dynamically up to a maximum of 192 pixels. The image is then normalized and
+        padded to fit the required input dimensions for classification.
+
+        Args:
+            image (np.ndarray): Input image to preprocess.
+
+        Returns:
+            np.ndarray: Preprocessed and padded image.
+        """
+        # fixed height of 48, dynamic width up to 192
+        input_shape = (3, 48, 192)
+        input_c, input_h, input_w = input_shape
+
+        h, w = image.shape[:2]
+        ratio = w / h
+        resized_w = min(input_w, math.ceil(input_h * ratio))
+
+        resized_image = cv2.resize(image, (resized_w, input_h))
+
+        # handle single-channel images (grayscale) if needed
+        if input_c == 1 and resized_image.ndim == 2:
+            resized_image = resized_image[np.newaxis, :, :]
+        else:
+            resized_image = resized_image.transpose((2, 0, 1))
+
+        # normalize
+        resized_image = (resized_image.astype("float32") / 255.0 - 0.5) / 0.5
+
+        padded_image = np.zeros((input_c, input_h, input_w), dtype=np.float32)
+        padded_image[:, :, :resized_w] = resized_image
+
+        return padded_image
+
+    def _process_classification_output(
+        self, images: List[np.ndarray], outputs: List[np.ndarray]
+    ) -> Tuple[List[np.ndarray], List[Tuple[str, float]]]:
+        """
+        Process the classification model output by matching labels with confidence scores.
+
+        This method processes the outputs from the classification model and rotates images
+        with high confidence of being labeled "180". It ensures that results are mapped to
+        the original image order.
+
+        Args:
+            images (List[np.ndarray]): List of input images.
+            outputs (List[np.ndarray]): Corresponding model outputs.
+
+        Returns:
+            Tuple[List[np.ndarray], List[Tuple[str, float]]]: A tuple of processed images and
+            classification results (label and confidence score).
+        """
+        labels = ["0", "180"]
+        results = [["", 0.0]] * len(images)
+        indices = np.argsort(np.array([x.shape[1] / x.shape[0] for x in images]))
+
+        outputs = np.stack(outputs)
+
+        outputs = [
+            (labels[idx], outputs[i, idx])
+            for i, idx in enumerate(outputs.argmax(axis=1))
+        ]
+
+        for i in range(0, len(images), self.batch_size):
+            for j in range(len(outputs)):
+                label, score = outputs[j]
+                results[indices[i + j]] = [label, score]
+                if "180" in label and score >= self.lpr_config.threshold:
+                    images[indices[i + j]] = cv2.rotate(images[indices[i + j]], 1)
+
+        return images, results
+
+    def _preprocess_recognition_image(
+        self, image: np.ndarray, max_wh_ratio: float
+    ) -> np.ndarray:
+        """
+        Preprocess an image for recognition by dynamically adjusting its width.
+
+        This method adjusts the width of the image based on the maximum width-to-height ratio
+        while keeping the height fixed at 48 pixels. The image is then normalized and padded
+        to fit the required input dimensions for recognition.
+
+        Args:
+            image (np.ndarray): Input image to preprocess.
+            max_wh_ratio (float): Maximum width-to-height ratio for resizing.
+
+        Returns:
+            np.ndarray: Preprocessed and padded image.
+        """
+        # fixed height of 48, dynamic width based on ratio
+        input_shape = [3, 48, 320]
+        input_h, input_w = input_shape[1], input_shape[2]
+
+        assert image.shape[2] == input_shape[0], "Unexpected number of image channels."
+
+        # dynamically adjust input width based on max_wh_ratio
+        input_w = int(input_h * max_wh_ratio)
+
+        # check for model-specific input width
+        model_input_w = self.recognition_model.runner.ort.get_inputs()[0].shape[3]
+        if isinstance(model_input_w, int) and model_input_w > 0:
+            input_w = model_input_w
+
+        h, w = image.shape[:2]
+        aspect_ratio = w / h
+        resized_w = min(input_w, math.ceil(input_h * aspect_ratio))
+
+        resized_image = cv2.resize(image, (resized_w, input_h))
+        resized_image = resized_image.transpose((2, 0, 1))
+        resized_image = (resized_image.astype("float32") / 255.0 - 0.5) / 0.5
+
+        padded_image = np.zeros((input_shape[0], input_h, input_w), dtype=np.float32)
+        padded_image[:, :, :resized_w] = resized_image
+
+        return padded_image
+
+    @staticmethod
+    def _crop_license_plate(image: np.ndarray, points: np.ndarray) -> np.ndarray:
+        """
+        Crop the license plate from the image using four corner points.
+
+        This method crops the region containing the license plate by using the perspective
+        transformation based on four corner points. If the resulting image is significantly
+        taller than wide, the image is rotated to the correct orientation.
+
+        Args:
+            image (np.ndarray): Input image containing the license plate.
+            points (np.ndarray): Four corner points defining the plate's position.
+
+        Returns:
+            np.ndarray: Cropped and potentially rotated license plate image.
+        """
+        assert len(points) == 4, "shape of points must be 4*2"
+        points = points.astype(np.float32)
+        crop_width = int(
+            max(
+                np.linalg.norm(points[0] - points[1]),
+                np.linalg.norm(points[2] - points[3]),
+            )
+        )
+        crop_height = int(
+            max(
+                np.linalg.norm(points[0] - points[3]),
+                np.linalg.norm(points[1] - points[2]),
+            )
+        )
+        pts_std = np.float32(
+            [[0, 0], [crop_width, 0], [crop_width, crop_height], [0, crop_height]]
+        )
+        matrix = cv2.getPerspectiveTransform(points, pts_std)
+        image = cv2.warpPerspective(
+            image,
+            matrix,
+            (crop_width, crop_height),
+            borderMode=cv2.BORDER_REPLICATE,
+            flags=cv2.INTER_CUBIC,
+        )
+        height, width = image.shape[0:2]
+        if height * 1.0 / width >= 1.5:
+            image = np.rot90(image, k=3)
+        return image
+
+
+class CTCDecoder:
+    """
+    A decoder for interpreting the output of a CTC (Connectionist Temporal Classification) model.
+
+    This decoder converts the model's output probabilities into readable sequences of characters
+    while removing duplicates and handling blank tokens. It also calculates the confidence scores
+    for each decoded character sequence.
+    """
+
+    def __init__(self):
+        """
+        Initialize the CTCDecoder with a list of characters and a character map.
+
+        The character set includes digits, letters, special characters, and a "blank" token
+        (used by the CTC model for decoding purposes). A character map is created to map
+        indices to characters.
+        """
+        self.characters = [
+            "blank",
+            "0",
+            "1",
+            "2",
+            "3",
+            "4",
+            "5",
+            "6",
+            "7",
+            "8",
+            "9",
+            ":",
+            ";",
+            "<",
+            "=",
+            ">",
+            "?",
+            "@",
+            "A",
+            "B",
+            "C",
+            "D",
+            "E",
+            "F",
+            "G",
+            "H",
+            "I",
+            "J",
+            "K",
+            "L",
+            "M",
+            "N",
+            "O",
+            "P",
+            "Q",
+            "R",
+            "S",
+            "T",
+            "U",
+            "V",
+            "W",
+            "X",
+            "Y",
+            "Z",
+            "[",
+            "\\",
+            "]",
+            "^",
+            "_",
+            "`",
+            "a",
+            "b",
+            "c",
+            "d",
+            "e",
+            "f",
+            "g",
+            "h",
+            "i",
+            "j",
+            "k",
+            "l",
+            "m",
+            "n",
+            "o",
+            "p",
+            "q",
+            "r",
+            "s",
+            "t",
+            "u",
+            "v",
+            "w",
+            "x",
+            "y",
+            "z",
+            "{",
+            "|",
+            "}",
+            "~",
+            "!",
+            '"',
+            "#",
+            "$",
+            "%",
+            "&",
+            "'",
+            "(",
+            ")",
+            "*",
+            "+",
+            ",",
+            "-",
+            ".",
+            "/",
+            " ",
+            " ",
+        ]
+        self.char_map = {i: char for i, char in enumerate(self.characters)}
+
+    def __call__(
+        self, outputs: List[np.ndarray]
+    ) -> Tuple[List[str], List[List[float]]]:
+        """
+        Decode a batch of model outputs into character sequences and their confidence scores.
+
+        The method takes the output probability distributions for each time step and uses
+        the best path decoding strategy. It then merges repeating characters and ignores
+        blank tokens. Confidence scores for each decoded character are also calculated.
+
+        Args:
+            outputs (List[np.ndarray]): A list of model outputs, where each element is
+                                        a probability distribution for each time step.
+
+        Returns:
+            Tuple[List[str], List[List[float]]]: A tuple of decoded character sequences
+                                                and confidence scores for each sequence.
+        """
+        results = []
+        confidences = []
+        for output in outputs:
+            seq_log_probs = np.log(output + 1e-8)
+            best_path = np.argmax(seq_log_probs, axis=1)
+
+            merged_path = []
+            merged_probs = []
+            for t, char_index in enumerate(best_path):
+                if char_index != 0 and (t == 0 or char_index != best_path[t - 1]):
+                    merged_path.append(char_index)
+                    merged_probs.append(seq_log_probs[t, char_index])
+
+            result = "".join(self.char_map[idx] for idx in merged_path)
+            results.append(result)
+
+            confidence = np.exp(merged_probs).tolist()
+            confidences.append(confidence)
+
+        return results, confidences

frigate/embeddings/maintainer.py

@@ -1,8 +1,10 @@
 """Maintain embeddings in SQLite-vec."""
 
 import base64
+import datetime
 import logging
 import os
+import re
 import threading
 from multiprocessing.synchronize import Event as MpEvent
 from pathlib import Path
@@ -10,6 +12,7 @@ from typing import Optional
 
 import cv2
 import numpy as np
+import requests
 from peewee import DoesNotExist
 from playhouse.sqliteq import SqliteQueueDatabase
 
@@ -21,13 +24,22 @@ from frigate.comms.event_metadata_updater import (
 from frigate.comms.events_updater import EventEndSubscriber, EventUpdateSubscriber
 from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config import FrigateConfig
-from frigate.const import CLIPS_DIR, UPDATE_EVENT_DESCRIPTION
+from frigate.const import (
+    CLIPS_DIR,
+    FRIGATE_LOCALHOST,
+    UPDATE_EVENT_DESCRIPTION,
+)
+from frigate.data_processing.real_time.api import RealTimeProcessorApi
+from frigate.data_processing.real_time.bird_processor import BirdProcessor
+from frigate.data_processing.real_time.face_processor import FaceProcessor
+from frigate.data_processing.types import DataProcessorMetrics
+from frigate.embeddings.lpr.lpr import LicensePlateRecognition
 from frigate.events.types import EventTypeEnum
 from frigate.genai import get_genai_client
 from frigate.models import Event
 from frigate.types import TrackedObjectUpdateTypesEnum
 from frigate.util.builtin import serialize
-from frigate.util.image import SharedMemoryFrameManager, calculate_region
+from frigate.util.image import SharedMemoryFrameManager, area, calculate_region
 
 from .embeddings import Embeddings
 
@@ -43,11 +55,13 @@ class EmbeddingMaintainer(threading.Thread):
         self,
         db: SqliteQueueDatabase,
         config: FrigateConfig,
+        metrics: DataProcessorMetrics,
         stop_event: MpEvent,
     ) -> None:
         super().__init__(name="embeddings_maintainer")
         self.config = config
-        self.embeddings = Embeddings(config.semantic_search, db)
+        self.metrics = metrics
+        self.embeddings = Embeddings(config, db, metrics)
 
         # Check if we need to re-index events
         if config.semantic_search.reindex:
@@ -60,12 +74,32 @@ class EmbeddingMaintainer(threading.Thread):
         )
         self.embeddings_responder = EmbeddingsResponder()
         self.frame_manager = SharedMemoryFrameManager()
+        self.processors: list[RealTimeProcessorApi] = []
+
+        if self.config.face_recognition.enabled:
+            self.processors.append(FaceProcessor(self.config, metrics))
+
+        if self.config.classification.bird.enabled:
+            self.processors.append(BirdProcessor(self.config, metrics))
+
         # create communication for updating event descriptions
         self.requestor = InterProcessRequestor()
         self.stop_event = stop_event
-        self.tracked_events = {}
+        self.tracked_events: dict[str, list[any]] = {}
         self.genai_client = get_genai_client(config)
 
+        # set license plate recognition conditions
+        self.lpr_config = self.config.lpr
+        self.requires_license_plate_detection = (
+            "license_plate" not in self.config.objects.all_objects
+        )
+        self.detected_license_plates: dict[str, dict[str, any]] = {}
+
+        if self.lpr_config.enabled:
+            self.license_plate_recognition = LicensePlateRecognition(
+                self.lpr_config, self.requestor, self.embeddings
+            )
+
     def run(self) -> None:
         """Maintain a SQLite-vec database for semantic search."""
         while not self.stop_event.is_set():
@@ -84,7 +118,7 @@ class EmbeddingMaintainer(threading.Thread):
     def _process_requests(self) -> None:
         """Process embeddings requests"""
 
-        def _handle_request(topic: str, data: str) -> str:
+        def _handle_request(topic: str, data: dict[str, any]) -> str:
             try:
                 if topic == EmbeddingsRequestEnum.embed_description.value:
                     return serialize(
@@ -101,8 +135,15 @@ class EmbeddingMaintainer(threading.Thread):
                     )
                 elif topic == EmbeddingsRequestEnum.generate_search.value:
                     return serialize(
-                        self.embeddings.text_embedding([data])[0], pack=False
+                        self.embeddings.embed_description("", data, upsert=False),
+                        pack=False,
                     )
+                else:
+                    for processor in self.processors:
+                        resp = processor.handle_request(topic, data)
+
+                        if resp is not None:
+                            return resp
             except Exception as e:
                 logger.error(f"Unable to handle embeddings request {e}")
 
@@ -110,7 +151,7 @@ class EmbeddingMaintainer(threading.Thread):
 
     def _process_updates(self) -> None:
         """Process event updates"""
-        update = self.event_subscriber.check_for_update(timeout=0.1)
+        update = self.event_subscriber.check_for_update(timeout=0.01)
 
         if update is None:
             return
@@ -121,42 +162,63 @@ class EmbeddingMaintainer(threading.Thread):
             return
 
         camera_config = self.config.cameras[camera]
-        # no need to save our own thumbnails if genai is not enabled
-        # or if the object has become stationary
+
+        # no need to process updated objects if face recognition, lpr, genai are disabled
         if (
             not camera_config.genai.enabled
-            or self.genai_client is None
-            or data["stationary"]
+            and not self.lpr_config.enabled
+            and len(self.processors) == 0
         ):
             return
 
-        if data["id"] not in self.tracked_events:
-            self.tracked_events[data["id"]] = []
-
         # Create our own thumbnail based on the bounding box and the frame time
         try:
             yuv_frame = self.frame_manager.get(
                 frame_name, camera_config.frame_shape_yuv
             )
-
-            if yuv_frame is not None:
-                data["thumbnail"] = self._create_thumbnail(yuv_frame, data["box"])
-
-                # Limit the number of thumbnails saved
-                if len(self.tracked_events[data["id"]]) >= MAX_THUMBNAILS:
-                    # Always keep the first thumbnail for the event
-                    self.tracked_events[data["id"]].pop(1)
-
-                self.tracked_events[data["id"]].append(data)
-
-                self.frame_manager.close(frame_name)
         except FileNotFoundError:
             pass
 
+        if yuv_frame is None:
+            logger.debug(
+                "Unable to process object update because frame is unavailable."
+            )
+            return
+
+        for processor in self.processors:
+            processor.process_frame(data, yuv_frame)
+
+        if self.lpr_config.enabled:
+            start = datetime.datetime.now().timestamp()
+            processed = self._process_license_plate(data, yuv_frame)
+
+            if processed:
+                duration = datetime.datetime.now().timestamp() - start
+                self.metrics.alpr_pps.value = (
+                    self.metrics.alpr_pps.value * 9 + duration
+                ) / 10
+
+        # no need to save our own thumbnails if genai is not enabled
+        # or if the object has become stationary
+        if self.genai_client is not None and not data["stationary"]:
+            if data["id"] not in self.tracked_events:
+                self.tracked_events[data["id"]] = []
+
+            data["thumbnail"] = self._create_thumbnail(yuv_frame, data["box"])
+
+            # Limit the number of thumbnails saved
+            if len(self.tracked_events[data["id"]]) >= MAX_THUMBNAILS:
+                # Always keep the first thumbnail for the event
+                self.tracked_events[data["id"]].pop(1)
+
+            self.tracked_events[data["id"]].append(data)
+
+        self.frame_manager.close(frame_name)
+
     def _process_finalized(self) -> None:
         """Process the end of an event."""
         while True:
-            ended = self.event_end_subscriber.check_for_update(timeout=0.1)
+            ended = self.event_end_subscriber.check_for_update(timeout=0.01)
 
             if ended == None:
                 break
@@ -164,6 +226,12 @@ class EmbeddingMaintainer(threading.Thread):
             event_id, camera, updated_db = ended
             camera_config = self.config.cameras[camera]
 
+            for processor in self.processors:
+                processor.expire_object(event_id)
+
+            if event_id in self.detected_license_plates:
+                self.detected_license_plates.pop(event_id)
+
             if updated_db:
                 try:
                     event: Event = Event.get(Event.id == event_id)
@@ -277,7 +345,7 @@ class EmbeddingMaintainer(threading.Thread):
     def _process_event_metadata(self):
         # Check for regenerate description requests
         (topic, event_id, source) = self.event_metadata_subscriber.check_for_update(
-            timeout=0.1
+            timeout=0.01
         )
 
         if topic is None:
@@ -286,6 +354,199 @@ class EmbeddingMaintainer(threading.Thread):
         if event_id:
             self.handle_regenerate_description(event_id, source)
 
+    def _detect_license_plate(self, input: np.ndarray) -> tuple[int, int, int, int]:
+        """Return the dimensions of the input image as [x, y, width, height]."""
+        height, width = input.shape[:2]
+        return (0, 0, width, height)
+
+    def _process_license_plate(
+        self, obj_data: dict[str, any], frame: np.ndarray
+    ) -> bool:
+        """Look for license plates in image."""
+        id = obj_data["id"]
+
+        # don't run for non car objects
+        if obj_data.get("label") != "car":
+            logger.debug("Not a processing license plate for non car object.")
+            return False
+
+        # don't run for stationary car objects
+        if obj_data.get("stationary") == True:
+            logger.debug("Not a processing license plate for a stationary car object.")
+            return False
+
+        # don't overwrite sub label for objects that have a sub label
+        # that is not a license plate
+        if obj_data.get("sub_label") and id not in self.detected_license_plates:
+            logger.debug(
+                f"Not processing license plate due to existing sub label: {obj_data.get('sub_label')}."
+            )
+            return False
+
+        license_plate: Optional[dict[str, any]] = None
+
+        if self.requires_license_plate_detection:
+            logger.debug("Running manual license_plate detection.")
+            car_box = obj_data.get("box")
+
+            if not car_box:
+                return False
+
+            rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420)
+            left, top, right, bottom = car_box
+            car = rgb[top:bottom, left:right]
+            license_plate = self._detect_license_plate(car)
+
+            if not license_plate:
+                logger.debug("Detected no license plates for car object.")
+                return False
+
+            license_plate_frame = car[
+                license_plate[1] : license_plate[3], license_plate[0] : license_plate[2]
+            ]
+            license_plate_frame = cv2.cvtColor(license_plate_frame, cv2.COLOR_RGB2BGR)
+        else:
+            # don't run for object without attributes
+            if not obj_data.get("current_attributes"):
+                logger.debug("No attributes to parse.")
+                return False
+
+            attributes: list[dict[str, any]] = obj_data.get("current_attributes", [])
+            for attr in attributes:
+                if attr.get("label") != "license_plate":
+                    continue
+
+                if license_plate is None or attr.get("score", 0.0) > license_plate.get(
+                    "score", 0.0
+                ):
+                    license_plate = attr
+
+            # no license plates detected in this frame
+            if not license_plate:
+                return False
+
+            license_plate_box = license_plate.get("box")
+
+            # check that license plate is valid
+            if (
+                not license_plate_box
+                or area(license_plate_box) < self.config.lpr.min_area
+            ):
+                logger.debug(f"Invalid license plate box {license_plate}")
+                return False
+
+            license_plate_frame = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
+            license_plate_frame = license_plate_frame[
+                license_plate_box[1] : license_plate_box[3],
+                license_plate_box[0] : license_plate_box[2],
+            ]
+
+        # run detection, returns results sorted by confidence, best first
+        license_plates, confidences, areas = (
+            self.license_plate_recognition.process_license_plate(license_plate_frame)
+        )
+
+        logger.debug(f"Text boxes: {license_plates}")
+        logger.debug(f"Confidences: {confidences}")
+        logger.debug(f"Areas: {areas}")
+
+        if license_plates:
+            for plate, confidence, text_area in zip(license_plates, confidences, areas):
+                avg_confidence = (
+                    (sum(confidence) / len(confidence)) if confidence else 0
+                )
+
+                logger.debug(
+                    f"Detected text: {plate} (average confidence: {avg_confidence:.2f}, area: {text_area} pixels)"
+                )
+        else:
+            # no plates found
+            logger.debug("No text detected")
+            return True
+
+        top_plate, top_char_confidences, top_area = (
+            license_plates[0],
+            confidences[0],
+            areas[0],
+        )
+        avg_confidence = (
+            (sum(top_char_confidences) / len(top_char_confidences))
+            if top_char_confidences
+            else 0
+        )
+
+        # Check if we have a previously detected plate for this ID
+        if id in self.detected_license_plates:
+            prev_plate = self.detected_license_plates[id]["plate"]
+            prev_char_confidences = self.detected_license_plates[id]["char_confidences"]
+            prev_area = self.detected_license_plates[id]["area"]
+            prev_avg_confidence = (
+                (sum(prev_char_confidences) / len(prev_char_confidences))
+                if prev_char_confidences
+                else 0
+            )
+
+            # Define conditions for keeping the previous plate
+            shorter_than_previous = len(top_plate) < len(prev_plate)
+            lower_avg_confidence = avg_confidence <= prev_avg_confidence
+            smaller_area = top_area < prev_area
+
+            # Compare character-by-character confidence where possible
+            min_length = min(len(top_plate), len(prev_plate))
+            char_confidence_comparison = sum(
+                1
+                for i in range(min_length)
+                if top_char_confidences[i] <= prev_char_confidences[i]
+            )
+            worse_char_confidences = char_confidence_comparison >= min_length / 2
+
+            if (shorter_than_previous or smaller_area) and (
+                lower_avg_confidence and worse_char_confidences
+            ):
+                logger.debug(
+                    f"Keeping previous plate. New plate stats: "
+                    f"length={len(top_plate)}, avg_conf={avg_confidence:.2f}, area={top_area} "
+                    f"vs Previous: length={len(prev_plate)}, avg_conf={prev_avg_confidence:.2f}, area={prev_area}"
+                )
+                return True
+
+        # Check against minimum confidence threshold
+        if avg_confidence < self.lpr_config.threshold:
+            logger.debug(
+                f"Average confidence {avg_confidence} is less than threshold ({self.lpr_config.threshold})"
+            )
+            return True
+
+        # Determine subLabel based on known plates, use regex matching
+        # Default to the detected plate, use label name if there's a match
+        sub_label = next(
+            (
+                label
+                for label, plates in self.lpr_config.known_plates.items()
+                if any(re.match(f"^{plate}$", top_plate) for plate in plates)
+            ),
+            top_plate,
+        )
+
+        # Send the result to the API
+        resp = requests.post(
+            f"{FRIGATE_LOCALHOST}/api/events/{id}/sub_label",
+            json={
+                "camera": obj_data.get("camera"),
+                "subLabel": sub_label,
+                "subLabelScore": avg_confidence,
+            },
+        )
+
+        if resp.status_code == 200:
+            self.detected_license_plates[id] = {
+                "plate": top_plate,
+                "char_confidences": top_char_confidences,
+                "area": top_area,
+            }
+
+        return True
+
     def _create_thumbnail(self, yuv_frame, box, height=500) -> Optional[bytes]:
         """Return jpg thumbnail of a region of the frame."""
         frame = cv2.cvtColor(yuv_frame, cv2.COLOR_YUV2BGR_I420)

frigate/ffmpeg_presets.py

@@ -6,6 +6,7 @@ from enum import Enum
 from typing import Any
 
 from frigate.const import (
+    FFMPEG_HVC1_ARGS,
     FFMPEG_HWACCEL_NVIDIA,
     FFMPEG_HWACCEL_VAAPI,
     FFMPEG_HWACCEL_VULKAN,
@@ -50,9 +51,16 @@ class LibvaGpuSelector:
         return ""
 
 
-LIBAV_VERSION = int(os.getenv("LIBAVFORMAT_VERSION_MAJOR", "59") or "59")
-FPS_VFR_PARAM = "-fps_mode vfr" if LIBAV_VERSION >= 59 else "-vsync 2"
-TIMEOUT_PARAM = "-timeout" if LIBAV_VERSION >= 59 else "-stimeout"
+FPS_VFR_PARAM = (
+    "-fps_mode vfr"
+    if int(os.getenv("LIBAVFORMAT_VERSION_MAJOR", "59") or "59") >= 59
+    else "-vsync 2"
+)
+TIMEOUT_PARAM = (
+    "-timeout"
+    if int(os.getenv("LIBAVFORMAT_VERSION_MAJOR", "59") or "59") >= 59
+    else "-stimeout"
+)
 
 _gpu_selector = LibvaGpuSelector()
 _user_agent_args = [
@@ -64,8 +72,8 @@ PRESETS_HW_ACCEL_DECODE = {
     "preset-rpi-64-h264": "-c:v:1 h264_v4l2m2m",
     "preset-rpi-64-h265": "-c:v:1 hevc_v4l2m2m",
     FFMPEG_HWACCEL_VAAPI: f"-hwaccel_flags allow_profile_mismatch -hwaccel vaapi -hwaccel_device {_gpu_selector.get_selected_gpu()} -hwaccel_output_format vaapi",
-    "preset-intel-qsv-h264": f"-hwaccel qsv -qsv_device {_gpu_selector.get_selected_gpu()} -hwaccel_output_format qsv -c:v h264_qsv{' -bsf:v dump_extra' if LIBAV_VERSION >= 61 else ''}",  # https://trac.ffmpeg.org/ticket/9766#comment:17
-    "preset-intel-qsv-h265": f"-load_plugin hevc_hw -hwaccel qsv -qsv_device {_gpu_selector.get_selected_gpu()} -hwaccel_output_format qsv{' -bsf:v dump_extra' if LIBAV_VERSION >= 61 else ''}",  # https://trac.ffmpeg.org/ticket/9766#comment:17
+    "preset-intel-qsv-h264": f"-hwaccel qsv -qsv_device {_gpu_selector.get_selected_gpu()} -hwaccel_output_format qsv -c:v h264_qsv -bsf:v dump_extra",  # https://trac.ffmpeg.org/ticket/9766#comment:17
+    "preset-intel-qsv-h265": f"-load_plugin hevc_hw -hwaccel qsv -qsv_device {_gpu_selector.get_selected_gpu()} -hwaccel_output_format qsv -c:v hevc_qsv -bsf:v dump_extra",  # https://trac.ffmpeg.org/ticket/9766#comment:17
     FFMPEG_HWACCEL_NVIDIA: "-hwaccel cuda -hwaccel_output_format cuda",
     "preset-jetson-h264": "-c:v h264_nvmpi -resize {1}x{2}",
     "preset-jetson-h265": "-c:v hevc_nvmpi -resize {1}x{2}",
@@ -490,6 +498,6 @@ def parse_preset_output_record(arg: Any, force_record_hvc1: bool) -> list[str]:
 
     if force_record_hvc1:
         # Apple only supports HEVC if it is hvc1 (vs. hev1)
-        preset += ["-tag:v", "hvc1"]
+        preset += FFMPEG_HVC1_ARGS
 
     return preset

frigate/log.py

@@ -18,12 +18,19 @@ LOG_HANDLER.setFormatter(
     )
 )
 
+# filter out norfair warning
 LOG_HANDLER.addFilter(
     lambda record: not record.getMessage().startswith(
         "You are using a scalar distance function"
     )
 )
 
+# filter out tflite logging
+LOG_HANDLER.addFilter(
+    lambda record: "Created TensorFlow Lite XNNPACK delegate for CPU."
+    not in record.getMessage()
+)
+
 log_listener: Optional[QueueListener] = None
 
 

frigate/mypy.ini

@@ -1,5 +1,5 @@
 [mypy]
-python_version = 3.9
+python_version = 3.11
 show_error_codes = true
 follow_imports = normal
 ignore_missing_imports = true

frigate/object_processing.py

@@ -4,7 +4,7 @@ import logging
 import os
 import queue
 import threading
-from collections import Counter, defaultdict
+from collections import defaultdict
 from multiprocessing.synchronize import Event as MpEvent
 from typing import Callable, Optional
 
@@ -51,8 +51,6 @@ class CameraState:
         self.camera_config = config.cameras[name]
         self.frame_manager = frame_manager
         self.best_objects: dict[str, TrackedObject] = {}
-        self.object_counts = defaultdict(int)
-        self.active_object_counts = defaultdict(int)
         self.tracked_objects: dict[str, TrackedObject] = {}
         self.frame_cache = {}
         self.zone_objects = defaultdict(list)
@@ -338,6 +336,7 @@ class CameraState:
                         "ratio": obj.obj_data["ratio"],
                         "score": obj.obj_data["score"],
                         "sub_label": sub_label,
+                        "current_zones": obj.current_zones,
                     }
                 )
 
@@ -377,78 +376,6 @@ class CameraState:
         for c in self.callbacks["camera_activity"]:
             c(self.name, camera_activity)
 
-        # update overall camera state for each object type
-        obj_counter = Counter(
-            obj.obj_data["label"]
-            for obj in tracked_objects.values()
-            if not obj.false_positive
-        )
-
-        active_obj_counter = Counter(
-            obj.obj_data["label"]
-            for obj in tracked_objects.values()
-            if not obj.false_positive and obj.active
-        )
-
-        # keep track of all labels detected for this camera
-        total_label_count = 0
-        total_active_label_count = 0
-
-        # report on all detected objects
-        for obj_name, count in obj_counter.items():
-            total_label_count += count
-
-            if count != self.object_counts[obj_name]:
-                self.object_counts[obj_name] = count
-                for c in self.callbacks["object_status"]:
-                    c(self.name, obj_name, count)
-
-        # update the active count on all detected objects
-        # To ensure we emit 0's if all objects are stationary, we need to loop
-        # over the set of all objects, not just active ones.
-        for obj_name in set(obj_counter):
-            count = active_obj_counter[obj_name]
-            total_active_label_count += count
-
-            if count != self.active_object_counts[obj_name]:
-                self.active_object_counts[obj_name] = count
-                for c in self.callbacks["active_object_status"]:
-                    c(self.name, obj_name, count)
-
-        # publish for all labels detected for this camera
-        if total_label_count != self.object_counts.get("all"):
-            self.object_counts["all"] = total_label_count
-            for c in self.callbacks["object_status"]:
-                c(self.name, "all", total_label_count)
-
-        # publish active label counts for this camera
-        if total_active_label_count != self.active_object_counts.get("all"):
-            self.active_object_counts["all"] = total_active_label_count
-            for c in self.callbacks["active_object_status"]:
-                c(self.name, "all", total_active_label_count)
-
-        # expire any objects that are >0 and no longer detected
-        expired_objects = [
-            obj_name
-            for obj_name, count in self.object_counts.items()
-            if count > 0 and obj_name not in obj_counter
-        ]
-        for obj_name in expired_objects:
-            # Ignore the artificial all label
-            if obj_name == "all":
-                continue
-
-            self.object_counts[obj_name] = 0
-            for c in self.callbacks["object_status"]:
-                c(self.name, obj_name, 0)
-            # Only publish if the object was previously active.
-            if self.active_object_counts[obj_name] > 0:
-                for c in self.callbacks["active_object_status"]:
-                    c(self.name, obj_name, 0)
-                self.active_object_counts[obj_name] = 0
-            for c in self.callbacks["snapshot"]:
-                c(self.name, self.best_objects[obj_name], frame_name)
-
         # cleanup thumbnail frame cache
         current_thumb_frames = {
             obj.thumbnail_data["frame_time"]
@@ -635,14 +562,6 @@ class TrackedObjectProcessor(threading.Thread):
                         retain=True,
                     )
 
-        def object_status(camera, object_name, status):
-            self.dispatcher.publish(f"{camera}/{object_name}", status, retain=False)
-
-        def active_object_status(camera, object_name, status):
-            self.dispatcher.publish(
-                f"{camera}/{object_name}/active", status, retain=False
-            )
-
         def camera_activity(camera, activity):
             last_activity = self.camera_activity.get(camera)
 
@@ -659,8 +578,6 @@ class TrackedObjectProcessor(threading.Thread):
             camera_state.on("update", update)
             camera_state.on("end", end)
             camera_state.on("snapshot", snapshot)
-            camera_state.on("object_status", object_status)
-            camera_state.on("active_object_status", active_object_status)
             camera_state.on("camera_activity", camera_activity)
             self.camera_states[camera] = camera_state
 
@@ -817,124 +734,6 @@ class TrackedObjectProcessor(threading.Thread):
                 )
             )
 
-            # update zone counts for each label
-            # for each zone in the current camera
-            for zone in self.config.cameras[camera].zones.keys():
-                # count labels for the camera in the zone
-                obj_counter = Counter(
-                    obj.obj_data["label"]
-                    for obj in camera_state.tracked_objects.values()
-                    if zone in obj.current_zones and not obj.false_positive
-                )
-                active_obj_counter = Counter(
-                    obj.obj_data["label"]
-                    for obj in camera_state.tracked_objects.values()
-                    if (
-                        zone in obj.current_zones
-                        and not obj.false_positive
-                        and obj.active
-                    )
-                )
-                total_label_count = 0
-                total_active_label_count = 0
-
-                # update counts and publish status
-                for label in set(self.zone_data[zone].keys()) | set(obj_counter.keys()):
-                    # Ignore the artificial all label
-                    if label == "all":
-                        continue
-
-                    # if we have previously published a count for this zone/label
-                    zone_label = self.zone_data[zone][label]
-                    active_zone_label = self.active_zone_data[zone][label]
-                    if camera in zone_label:
-                        current_count = sum(zone_label.values())
-                        current_active_count = sum(active_zone_label.values())
-                        zone_label[camera] = (
-                            obj_counter[label] if label in obj_counter else 0
-                        )
-                        active_zone_label[camera] = (
-                            active_obj_counter[label]
-                            if label in active_obj_counter
-                            else 0
-                        )
-                        new_count = sum(zone_label.values())
-                        new_active_count = sum(active_zone_label.values())
-                        if new_count != current_count:
-                            self.dispatcher.publish(
-                                f"{zone}/{label}",
-                                new_count,
-                                retain=False,
-                            )
-                        if new_active_count != current_active_count:
-                            self.dispatcher.publish(
-                                f"{zone}/{label}/active",
-                                new_active_count,
-                                retain=False,
-                            )
-
-                        # Set the count for the /zone/all topic.
-                        total_label_count += new_count
-                        total_active_label_count += new_active_count
-
-                    # if this is a new zone/label combo for this camera
-                    else:
-                        if label in obj_counter:
-                            zone_label[camera] = obj_counter[label]
-                            active_zone_label[camera] = active_obj_counter[label]
-                            self.dispatcher.publish(
-                                f"{zone}/{label}",
-                                obj_counter[label],
-                                retain=False,
-                            )
-                            self.dispatcher.publish(
-                                f"{zone}/{label}/active",
-                                active_obj_counter[label],
-                                retain=False,
-                            )
-
-                            # Set the count for the /zone/all topic.
-                            total_label_count += obj_counter[label]
-                            total_active_label_count += active_obj_counter[label]
-
-                # if we have previously published a count for this zone all labels
-                zone_label = self.zone_data[zone]["all"]
-                active_zone_label = self.active_zone_data[zone]["all"]
-                if camera in zone_label:
-                    current_count = sum(zone_label.values())
-                    current_active_count = sum(active_zone_label.values())
-                    zone_label[camera] = total_label_count
-                    active_zone_label[camera] = total_active_label_count
-                    new_count = sum(zone_label.values())
-                    new_active_count = sum(active_zone_label.values())
-
-                    if new_count != current_count:
-                        self.dispatcher.publish(
-                            f"{zone}/all",
-                            new_count,
-                            retain=False,
-                        )
-                    if new_active_count != current_active_count:
-                        self.dispatcher.publish(
-                            f"{zone}/all/active",
-                            new_active_count,
-                            retain=False,
-                        )
-                # if this is a new zone all label for this camera
-                else:
-                    zone_label[camera] = total_label_count
-                    active_zone_label[camera] = total_active_label_count
-                    self.dispatcher.publish(
-                        f"{zone}/all",
-                        total_label_count,
-                        retain=False,
-                    )
-                    self.dispatcher.publish(
-                        f"{zone}/all/active",
-                        total_active_label_count,
-                        retain=False,
-                    )
-
             # cleanup event finished queue
             while not self.stop_event.is_set():
                 update = self.event_end_subscriber.check_for_update(timeout=0.01)

frigate/ptz/onvif.py

@@ -1,15 +1,14 @@
 """Configure and control camera via onvif."""
 
+import asyncio
 import logging
 from enum import Enum
 from importlib.util import find_spec
 from pathlib import Path
 
 import numpy
-import requests
-from onvif import ONVIFCamera, ONVIFError
+from onvif import ONVIFCamera, ONVIFError, ONVIFService
 from zeep.exceptions import Fault, TransportError
-from zeep.transports import Transport
 
 from frigate.camera import PTZMetrics
 from frigate.config import FrigateConfig, ZoomingModeEnum
@@ -49,11 +48,6 @@ class OnvifController:
 
             if cam.onvif.host:
                 try:
-                    session = requests.Session()
-                    session.verify = not cam.onvif.tls_insecure
-                    transport = Transport(
-                        timeout=10, operation_timeout=10, session=session
-                    )
                     self.cams[cam_name] = {
                         "onvif": ONVIFCamera(
                             cam.onvif.host,
@@ -62,9 +56,9 @@ class OnvifController:
                             cam.onvif.password,
                             wsdl_dir=str(
                                 Path(find_spec("onvif").origin).parent / "wsdl"
-                            ).replace("dist-packages/onvif", "site-packages"),
+                            ),
                             adjust_time=cam.onvif.ignore_time_mismatch,
-                            transport=transport,
+                            encrypt=not cam.onvif.tls_insecure,
                         ),
                         "init": False,
                         "active": False,
@@ -74,11 +68,12 @@ class OnvifController:
                 except ONVIFError as e:
                     logger.error(f"Onvif connection to {cam.name} failed: {e}")
 
-    def _init_onvif(self, camera_name: str) -> bool:
+    async def _init_onvif(self, camera_name: str) -> bool:
         onvif: ONVIFCamera = self.cams[camera_name]["onvif"]
+        await onvif.update_xaddrs()
 
         # create init services
-        media = onvif.create_media_service()
+        media: ONVIFService = await onvif.create_media_service()
         logger.debug(f"Onvif media xaddr for {camera_name}: {media.xaddr}")
 
         try:
@@ -92,7 +87,7 @@ class OnvifController:
             return False
 
         try:
-            profiles = media.GetProfiles()
+            profiles = await media.GetProfiles()
             logger.debug(f"Onvif profiles for {camera_name}: {profiles}")
         except (ONVIFError, Fault, TransportError) as e:
             logger.error(
@@ -101,7 +96,7 @@ class OnvifController:
             return False
 
         profile = None
-        for key, onvif_profile in enumerate(profiles):
+        for _, onvif_profile in enumerate(profiles):
             if (
                 onvif_profile.VideoEncoderConfiguration
                 and onvif_profile.PTZConfiguration
@@ -135,7 +130,8 @@ class OnvifController:
             )
             return False
 
-        ptz = onvif.create_ptz_service()
+        ptz: ONVIFService = await onvif.create_ptz_service()
+        self.cams[camera_name]["ptz"] = ptz
 
         # setup continuous moving request
         move_request = ptz.create_type("ContinuousMove")
@@ -246,7 +242,7 @@ class OnvifController:
 
         # setup existing presets
         try:
-            presets: list[dict] = ptz.GetPresets({"ProfileToken": profile.token})
+            presets: list[dict] = await ptz.GetPresets({"ProfileToken": profile.token})
         except ONVIFError as e:
             logger.warning(f"Unable to get presets from camera: {camera_name}: {e}")
             presets = []
@@ -325,19 +321,19 @@ class OnvifController:
             )
 
         self.cams[camera_name]["features"] = supported_features
-
         self.cams[camera_name]["init"] = True
         return True
 
     def _stop(self, camera_name: str) -> None:
-        onvif: ONVIFCamera = self.cams[camera_name]["onvif"]
         move_request = self.cams[camera_name]["move_request"]
-        onvif.get_service("ptz").Stop(
-            {
-                "ProfileToken": move_request.ProfileToken,
-                "PanTilt": True,
-                "Zoom": True,
-            }
+        asyncio.run(
+            self.cams[camera_name]["ptz"].Stop(
+                {
+                    "ProfileToken": move_request.ProfileToken,
+                    "PanTilt": True,
+                    "Zoom": True,
+                }
+            )
         )
         self.cams[camera_name]["active"] = False
 
@@ -353,7 +349,6 @@ class OnvifController:
             return
 
         self.cams[camera_name]["active"] = True
-        onvif: ONVIFCamera = self.cams[camera_name]["onvif"]
         move_request = self.cams[camera_name]["move_request"]
 
         if command == OnvifCommandEnum.move_left:
@@ -376,7 +371,7 @@ class OnvifController:
             }
 
         try:
-            onvif.get_service("ptz").ContinuousMove(move_request)
+            asyncio.run(self.cams[camera_name]["ptz"].ContinuousMove(move_request))
         except ONVIFError as e:
             logger.warning(f"Onvif sending move request to {camera_name} failed: {e}")
 
@@ -404,7 +399,6 @@ class OnvifController:
             camera_name
         ].frame_time.value
         self.ptz_metrics[camera_name].stop_time.value = 0
-        onvif: ONVIFCamera = self.cams[camera_name]["onvif"]
         move_request = self.cams[camera_name]["relative_move_request"]
 
         # function takes in -1 to 1 for pan and tilt, interpolate to the values of the camera.
@@ -450,7 +444,7 @@ class OnvifController:
             }
             move_request.Translation.Zoom.x = zoom
 
-        onvif.get_service("ptz").RelativeMove(move_request)
+        asyncio.run(self.cams[camera_name]["ptz"].RelativeMove(move_request))
 
         # reset after the move request
         move_request.Translation.PanTilt.x = 0
@@ -475,13 +469,14 @@ class OnvifController:
         self.ptz_metrics[camera_name].start_time.value = 0
         self.ptz_metrics[camera_name].stop_time.value = 0
         move_request = self.cams[camera_name]["move_request"]
-        onvif: ONVIFCamera = self.cams[camera_name]["onvif"]
         preset_token = self.cams[camera_name]["presets"][preset]
-        onvif.get_service("ptz").GotoPreset(
-            {
-                "ProfileToken": move_request.ProfileToken,
-                "PresetToken": preset_token,
-            }
+        asyncio.run(
+            self.cams[camera_name]["ptz"].GotoPreset(
+                {
+                    "ProfileToken": move_request.ProfileToken,
+                    "PresetToken": preset_token,
+                }
+            )
         )
 
         self.cams[camera_name]["active"] = False
@@ -498,7 +493,6 @@ class OnvifController:
             return
 
         self.cams[camera_name]["active"] = True
-        onvif: ONVIFCamera = self.cams[camera_name]["onvif"]
         move_request = self.cams[camera_name]["move_request"]
 
         if command == OnvifCommandEnum.zoom_in:
@@ -506,7 +500,7 @@ class OnvifController:
         elif command == OnvifCommandEnum.zoom_out:
             move_request.Velocity = {"Zoom": {"x": -0.5}}
 
-        onvif.get_service("ptz").ContinuousMove(move_request)
+        asyncio.run(self.cams[camera_name]["ptz"].ContinuousMove(move_request))
 
     def _zoom_absolute(self, camera_name: str, zoom, speed) -> None:
         if "zoom-a" not in self.cams[camera_name]["features"]:
@@ -530,7 +524,6 @@ class OnvifController:
             camera_name
         ].frame_time.value
         self.ptz_metrics[camera_name].stop_time.value = 0
-        onvif: ONVIFCamera = self.cams[camera_name]["onvif"]
         move_request = self.cams[camera_name]["absolute_move_request"]
 
         # function takes in 0 to 1 for zoom, interpolate to the values of the camera.
@@ -548,7 +541,7 @@ class OnvifController:
 
         logger.debug(f"{camera_name}: Absolute zoom: {zoom}")
 
-        onvif.get_service("ptz").AbsoluteMove(move_request)
+        asyncio.run(self.cams[camera_name]["ptz"].AbsoluteMove(move_request))
 
         self.cams[camera_name]["active"] = False
 
@@ -560,7 +553,7 @@ class OnvifController:
             return
 
         if not self.cams[camera_name]["init"]:
-            if not self._init_onvif(camera_name):
+            if not asyncio.run(self._init_onvif(camera_name)):
                 return
 
         try:
@@ -590,7 +583,7 @@ class OnvifController:
             return {}
 
         if not self.cams[camera_name]["init"]:
-            self._init_onvif(camera_name)
+            asyncio.run(self._init_onvif(camera_name))
 
         return {
             "name": camera_name,
@@ -604,15 +597,16 @@ class OnvifController:
             return {}
 
         if not self.cams[camera_name]["init"]:
-            self._init_onvif(camera_name)
+            asyncio.run(self._init_onvif(camera_name))
 
-        onvif: ONVIFCamera = self.cams[camera_name]["onvif"]
         service_capabilities_request = self.cams[camera_name][
             "service_capabilities_request"
         ]
         try:
-            service_capabilities = onvif.get_service("ptz").GetServiceCapabilities(
-                service_capabilities_request
+            service_capabilities = asyncio.run(
+                self.cams[camera_name]["ptz"].GetServiceCapabilities(
+                    service_capabilities_request
+                )
             )
 
             logger.debug(
@@ -633,12 +627,13 @@ class OnvifController:
             return {}
 
         if not self.cams[camera_name]["init"]:
-            self._init_onvif(camera_name)
+            asyncio.run(self._init_onvif(camera_name))
 
-        onvif: ONVIFCamera = self.cams[camera_name]["onvif"]
         status_request = self.cams[camera_name]["status_request"]
         try:
-            status = onvif.get_service("ptz").GetStatus(status_request)
+            status = asyncio.run(
+                self.cams[camera_name]["ptz"].GetStatus(status_request)
+            )
         except Exception:
             pass  # We're unsupported, that'll be reported in the next check.
 

frigate/record/export.py

@@ -19,6 +19,7 @@ from frigate.const import (
     CACHE_DIR,
     CLIPS_DIR,
     EXPORT_DIR,
+    FFMPEG_HVC1_ARGS,
     MAX_PLAYLIST_SECONDS,
     PREVIEW_FRAME_TYPE,
 )
@@ -219,7 +220,7 @@ class RecordingExporter(threading.Thread):
 
         if self.playback_factor == PlaybackFactorEnum.realtime:
             ffmpeg_cmd = (
-                f"{self.config.ffmpeg.ffmpeg_path} -hide_banner {ffmpeg_input} -c copy -movflags +faststart {video_path}"
+                f"{self.config.ffmpeg.ffmpeg_path} -hide_banner {ffmpeg_input} -c copy -movflags +faststart"
             ).split(" ")
         elif self.playback_factor == PlaybackFactorEnum.timelapse_25x:
             ffmpeg_cmd = (
@@ -227,11 +228,16 @@ class RecordingExporter(threading.Thread):
                     self.config.ffmpeg.ffmpeg_path,
                     self.config.ffmpeg.hwaccel_args,
                     f"-an {ffmpeg_input}",
-                    f"{self.config.cameras[self.camera].record.export.timelapse_args} -movflags +faststart {video_path}",
+                    f"{self.config.cameras[self.camera].record.export.timelapse_args} -movflags +faststart",
                     EncodeTypeEnum.timelapse,
                 )
             ).split(" ")
 
+        if self.config.ffmpeg.apple_compatibility:
+            ffmpeg_cmd += FFMPEG_HVC1_ARGS
+
+        ffmpeg_cmd.append(video_path)
+
         return ffmpeg_cmd, playlist_lines
 
     def get_preview_export_command(self, video_path: str) -> list[str]:

frigate/service_manager/service.py

@@ -26,7 +26,7 @@ class Service(ABC):
             self.__dict__["name"] = name
 
         self.__manager = manager or ServiceManager.current()
-        self.__lock = asyncio.Lock(loop=self.__manager._event_loop)
+        self.__lock = asyncio.Lock(loop=self.__manager._event_loop)  # type: ignore[call-arg]
         self.__manager._register(self)
 
     @property

frigate/stats/util.py

@@ -14,6 +14,7 @@ from requests.exceptions import RequestException
 from frigate.camera import CameraMetrics
 from frigate.config import FrigateConfig
 from frigate.const import CACHE_DIR, CLIPS_DIR, RECORD_DIR
+from frigate.data_processing.types import DataProcessorMetrics
 from frigate.object_detection import ObjectDetectProcess
 from frigate.types import StatsTrackingTypes
 from frigate.util.services import (
@@ -51,11 +52,13 @@ def get_latest_version(config: FrigateConfig) -> str:
 def stats_init(
     config: FrigateConfig,
     camera_metrics: dict[str, CameraMetrics],
+    embeddings_metrics: DataProcessorMetrics | None,
     detectors: dict[str, ObjectDetectProcess],
     processes: dict[str, int],
 ) -> StatsTrackingTypes:
     stats_tracking: StatsTrackingTypes = {
         "camera_metrics": camera_metrics,
+        "embeddings_metrics": embeddings_metrics,
         "detectors": detectors,
         "started": int(time.time()),
         "latest_frigate_version": get_latest_version(config),
@@ -195,7 +198,7 @@ async def set_gpu_stats(
                 continue
 
             # intel QSV GPU
-            intel_usage = get_intel_gpu_stats()
+            intel_usage = get_intel_gpu_stats(config.telemetry.stats.sriov)
 
             if intel_usage is not None:
                 stats["intel-qsv"] = intel_usage or {"gpu": "", "mem": ""}
@@ -220,7 +223,7 @@ async def set_gpu_stats(
                     continue
 
                 # intel VAAPI GPU
-                intel_usage = get_intel_gpu_stats()
+                intel_usage = get_intel_gpu_stats(config.telemetry.stats.sriov)
 
                 if intel_usage is not None:
                     stats["intel-vaapi"] = intel_usage or {"gpu": "", "mem": ""}
@@ -279,6 +282,27 @@ def stats_snapshot(
         }
     stats["detection_fps"] = round(total_detection_fps, 2)
 
+    if config.semantic_search.enabled:
+        embeddings_metrics = stats_tracking["embeddings_metrics"]
+        stats["embeddings"] = {
+            "image_embedding_speed": round(
+                embeddings_metrics.image_embeddings_fps.value * 1000, 2
+            ),
+            "text_embedding_speed": round(
+                embeddings_metrics.text_embeddings_sps.value * 1000, 2
+            ),
+        }
+
+        if config.face_recognition.enabled:
+            stats["embeddings"]["face_recognition_speed"] = round(
+                embeddings_metrics.face_rec_fps.value * 1000, 2
+            )
+
+        if config.lpr.enabled:
+            stats["embeddings"]["plate_recognition_speed"] = round(
+                embeddings_metrics.alpr_pps.value * 1000, 2
+            )
+
     get_processing_stats(config, stats, hwaccel_errors)
 
     stats["service"] = {

frigate/test/test_gpu_stats.py

@@ -38,7 +38,7 @@ class TestGpuStats(unittest.TestCase):
         process.returncode = 124
         process.stdout = self.intel_results
         sp.return_value = process
-        intel_stats = get_intel_gpu_stats()
+        intel_stats = get_intel_gpu_stats(False)
         print(f"the intel stats are {intel_stats}")
         assert intel_stats == {
             "gpu": "1.13%",

frigate/types.py

@@ -2,11 +2,13 @@ from enum import Enum
 from typing import TypedDict
 
 from frigate.camera import CameraMetrics
+from frigate.data_processing.types import DataProcessorMetrics
 from frigate.object_detection import ObjectDetectProcess
 
 
 class StatsTrackingTypes(TypedDict):
     camera_metrics: dict[str, CameraMetrics]
+    embeddings_metrics: DataProcessorMetrics | None
     detectors: dict[str, ObjectDetectProcess]
     started: int
     latest_frigate_version: str

frigate/util/downloader.py

@@ -51,12 +51,14 @@ class ModelDownloader:
         download_path: str,
         file_names: List[str],
         download_func: Callable[[str], None],
+        complete_func: Callable[[], None] | None = None,
         silent: bool = False,
     ):
         self.model_name = model_name
         self.download_path = download_path
         self.file_names = file_names
         self.download_func = download_func
+        self.complete_func = complete_func
         self.silent = silent
         self.requestor = InterProcessRequestor()
         self.download_thread = None
@@ -97,11 +99,14 @@ class ModelDownloader:
                 },
             )
 
+        if self.complete_func:
+            self.complete_func()
+
         self.requestor.stop()
         self.download_complete.set()
 
     @staticmethod
-    def download_from_url(url: str, save_path: str, silent: bool = False):
+    def download_from_url(url: str, save_path: str, silent: bool = False) -> Path:
         temporary_filename = Path(save_path).with_name(
             os.path.basename(save_path) + ".part"
         )
@@ -125,6 +130,8 @@ class ModelDownloader:
         if not silent:
             logger.info(f"Downloading complete: {url}")
 
+        return Path(save_path)
+
     @staticmethod
     def mark_files_state(
         requestor: InterProcessRequestor,

frigate/util/services.py

@@ -255,7 +255,7 @@ def get_amd_gpu_stats() -> dict[str, str]:
         return results
 
 
-def get_intel_gpu_stats() -> dict[str, str]:
+def get_intel_gpu_stats(sriov: bool) -> dict[str, str]:
     """Get stats using intel_gpu_top."""
 
     def get_stats_manually(output: str) -> dict[str, str]:
@@ -302,6 +302,9 @@ def get_intel_gpu_stats() -> dict[str, str]:
         "1",
     ]
 
+    if sriov:
+        intel_gpu_top_command += ["-d", "drm:/dev/dri/card0"]
+
     p = sp.run(
         intel_gpu_top_command,
         encoding="ascii",

web/src/App.tsx

@@ -19,6 +19,7 @@ const ConfigEditor = lazy(() => import("@/pages/ConfigEditor"));
 const System = lazy(() => import("@/pages/System"));
 const Settings = lazy(() => import("@/pages/Settings"));
 const UIPlayground = lazy(() => import("@/pages/UIPlayground"));
+const FaceLibrary = lazy(() => import("@/pages/FaceLibrary"));
 const Logs = lazy(() => import("@/pages/Logs"));
 
 function App() {
@@ -51,6 +52,7 @@ function App() {
                   <Route path="/config" element={<ConfigEditor />} />
                   <Route path="/logs" element={<Logs />} />
                   <Route path="/playground" element={<UIPlayground />} />
+                  <Route path="/faces" element={<FaceLibrary />} />
                   <Route path="*" element={<Redirect to="/" />} />
                 </Routes>
               </Suspense>

web/src/components/camera/AutoUpdatingCameraImage.tsx

@@ -1,4 +1,4 @@
-import { useCallback, useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import CameraImage from "./CameraImage";
 
 type AutoUpdatingCameraImageProps = {
@@ -8,6 +8,7 @@ type AutoUpdatingCameraImageProps = {
   className?: string;
   cameraClasses?: string;
   reloadInterval?: number;
+  periodicCache?: boolean;
 };
 
 const MIN_LOAD_TIMEOUT_MS = 200;
@@ -19,6 +20,7 @@ export default function AutoUpdatingCameraImage({
   className,
   cameraClasses,
   reloadInterval = MIN_LOAD_TIMEOUT_MS,
+  periodicCache = false,
 }: AutoUpdatingCameraImageProps) {
   const [key, setKey] = useState(Date.now());
   const [fps, setFps] = useState<string>("0");
@@ -42,6 +44,8 @@ export default function AutoUpdatingCameraImage({
   }, [reloadInterval]);
 
   const handleLoad = useCallback(() => {
+    setIsCached(true);
+
     if (reloadInterval == -1) {
       return;
     }
@@ -66,12 +70,28 @@ export default function AutoUpdatingCameraImage({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [key, setFps]);
 
+  // periodic cache to reduce loading indicator
+
+  const [isCached, setIsCached] = useState(false);
+
+  const cacheKey = useMemo(() => {
+    let baseParam = "";
+
+    if (periodicCache && !isCached) {
+      baseParam = "store=1";
+    } else {
+      baseParam = `cache=${key}`;
+    }
+
+    return `${baseParam}${searchParams ? `&${searchParams}` : ""}`;
+  }, [isCached, periodicCache, key, searchParams]);
+
   return (
     <div className={className}>
       <CameraImage
         camera={camera}
         onload={handleLoad}
-        searchParams={`cache=${key}${searchParams ? `&${searchParams}` : ""}`}
+        searchParams={cacheKey}
         className={cameraClasses}
       />
       {showFps ? <span className="text-xs">Displaying at {fps}fps</span> : null}