Include necessary cudnn deps in cuda folder

* Add ability to view attempts

* Improve UI

* Cleanup

* Correctly refresh ui when item is deleted

* Select correct library by default

* Add min score

* Cleanup

.cspell/frigate-dictionary.txt

@@ -2,6 +2,7 @@ aarch
 absdiff
 airockchip
 Alloc
+alpr
 Amcrest
 amdgpu
 analyzeduration
@@ -61,6 +62,7 @@ dsize
 dtype
 ECONNRESET
 edgetpu
+facenet
 fastapi
 faststart
 fflags
@@ -114,6 +116,8 @@ itemsize
 Jellyfin
 jetson
 jetsons
+jina
+jinaai
 joserfc
 jsmpeg
 jsonify
@@ -187,6 +191,7 @@ openai
 opencv
 openvino
 OWASP
+paddleocr
 paho
 passwordless
 popleft
@@ -308,4 +313,4 @@ yolo
 yolonas
 yolox
 zeep
-zerolatency
+zerolatency

.github/workflows/ci.yml

@@ -7,7 +7,7 @@ on:
       - dev
       - master
     paths-ignore:
-      - 'docs/**'      
+      - "docs/**"
 
 # only run the latest commit to avoid cache overwrites
 concurrency:
@@ -24,6 +24,8 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: Set up QEMU and Buildx
         id: setup
         uses: ./.github/actions/setup
@@ -45,6 +47,8 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: Set up QEMU and Buildx
         id: setup
         uses: ./.github/actions/setup
@@ -71,21 +75,14 @@ jobs:
             rpi.tags=${{ steps.setup.outputs.image-name }}-rpi
             *.cache-from=type=registry,ref=${{ steps.setup.outputs.cache-name }}-arm64
             *.cache-to=type=registry,ref=${{ steps.setup.outputs.cache-name }}-arm64,mode=max
-      - name: Build and push Rockchip build
-        uses: docker/bake-action@v3
-        with:
-          push: true
-          targets: rk
-          files: docker/rockchip/rk.hcl
-          set: |
-            rk.tags=${{ steps.setup.outputs.image-name }}-rk
-            *.cache-from=type=gha
   jetson_jp4_build:
     runs-on: ubuntu-latest
     name: Jetson Jetpack 4
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: Set up QEMU and Buildx
         id: setup
         uses: ./.github/actions/setup
@@ -112,6 +109,8 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: Set up QEMU and Buildx
         id: setup
         uses: ./.github/actions/setup
@@ -140,6 +139,8 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: Set up QEMU and Buildx
         id: setup
         uses: ./.github/actions/setup
@@ -165,6 +166,8 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: Set up QEMU and Buildx
         id: setup
         uses: ./.github/actions/setup
@@ -188,6 +191,8 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: Set up QEMU and Buildx
         id: setup
         uses: ./.github/actions/setup

.github/workflows/dependabot-auto-merge.yaml

@@ -1,24 +0,0 @@
-name: dependabot-auto-merge
-on: pull_request
-
-permissions:
-  contents: write
-
-jobs:
-  dependabot-auto-merge:
-    runs-on: ubuntu-latest
-    if: github.actor == 'dependabot[bot]'
-    steps:
-      - name: Get Dependabot metadata
-        id: metadata
-        uses: dependabot/fetch-metadata@v2
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Enable auto-merge for Dependabot PRs
-        if: steps.metadata.outputs.dependency-type == 'direct:development' && (steps.metadata.outputs.update-type == 'version-update:semver-minor' || steps.metadata.outputs.update-type == 'version-update:semver-patch')
-        run: |
-          gh pr review --approve "$PR_URL"
-          gh pr merge --auto --squash "$PR_URL"
-        env:
-          PR_URL: ${{ github.event.pull_request.html_url }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pull_request.yml

@@ -3,7 +3,7 @@ name: On pull request
 on:
   pull_request:
     paths-ignore:
-      - 'docs/**'
+      - "docs/**"
 
 env:
   DEFAULT_PYTHON: 3.9
@@ -19,6 +19,8 @@ jobs:
       DOCKER_BUILDKIT: "1"
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - uses: actions/setup-node@master
         with:
           node-version: 16.x
@@ -38,6 +40,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - uses: actions/setup-node@master
         with:
           node-version: 16.x
@@ -52,6 +56,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - uses: actions/setup-node@master
         with:
           node-version: 20.x
@@ -67,8 +73,10 @@ jobs:
     steps:
       - name: Check out the repository
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: Set up Python ${{ env.DEFAULT_PYTHON }}
-        uses: actions/setup-python@v5.1.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: ${{ env.DEFAULT_PYTHON }}
       - name: Install requirements
@@ -88,6 +96,8 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - uses: actions/setup-node@master
         with:
           node-version: 16.x

.github/workflows/release.yml

@@ -11,6 +11,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - id: lowercaseRepo
         uses: ASzc/change-string-case-action@v6
         with:
@@ -22,10 +24,13 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Create tag variables
+        env:
+          TAG: ${{ github.ref_name }}
+          LOWERCASE_REPO: ${{ steps.lowercaseRepo.outputs.lowercase }}
         run: |
-          BUILD_TYPE=$([[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && echo "stable" || echo "beta")
+          BUILD_TYPE=$([[ "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && echo "stable" || echo "beta")
           echo "BUILD_TYPE=${BUILD_TYPE}" >> $GITHUB_ENV
-          echo "BASE=ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}" >> $GITHUB_ENV
+          echo "BASE=ghcr.io/${LOWERCASE_REPO}" >> $GITHUB_ENV
           echo "BUILD_TAG=${GITHUB_SHA::7}" >> $GITHUB_ENV
           echo "CLEAN_VERSION=$(echo ${GITHUB_REF##*/} | tr '[:upper:]' '[:lower:]' | sed 's/^[v]//')" >> $GITHUB_ENV
       - name: Tag and push the main image

.github/workflows/stale.yml

@@ -23,7 +23,9 @@ jobs:
           exempt-pr-labels: "pinned,security,dependencies"
           operations-per-run: 120
       - name: Print outputs
-        run: echo ${{ join(steps.stale.outputs.*, ',') }}
+        env:
+          STALE_OUTPUT: ${{ join(steps.stale.outputs.*, ',') }}
+        run: echo "$STALE_OUTPUT"
 
   # clean_ghcr:
   #   name: Delete outdated dev container images
@@ -38,4 +40,3 @@ jobs:
   #         account-type: personal
   #         token: ${{ secrets.GITHUB_TOKEN }}
   #         token-type: github-token
-

Makefile

@@ -1,7 +1,7 @@
 default_target: local
 
 COMMIT_HASH := $(shell git log -1 --pretty=format:"%h"|tail -1)
-VERSION = 0.15.0
+VERSION = 0.16.0
 IMAGE_REPO ?= ghcr.io/blakeblackshear/frigate
 GITHUB_REF_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
 BOARDS= #Initialized empty

docker/hailo8l/Dockerfile

@@ -5,6 +5,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 # Build Python wheels
 FROM wheels AS h8l-wheels
 
+RUN python3 -m pip config set global.break-system-packages true
 COPY docker/main/requirements-wheels.txt /requirements-wheels.txt
 COPY docker/hailo8l/requirements-wheels-h8l.txt /requirements-wheels-h8l.txt
 
@@ -30,6 +31,7 @@ COPY --from=hailort /hailo-wheels /deps/hailo-wheels
 COPY --from=hailort /rootfs/ /
 
 # Install the wheels
+RUN python3 -m pip config set global.break-system-packages true
 RUN pip3 install -U /deps/h8l-wheels/*.whl
 RUN pip3 install -U /deps/hailo-wheels/*.whl
 

docker/hailo8l/install_hailort.sh

@@ -15,5 +15,5 @@ wget -qO- "https://github.com/frigate-nvr/hailort/releases/download/v${hailo_ver
 
 mkdir -p /hailo-wheels
 
-wget -P /hailo-wheels/ "https://github.com/frigate-nvr/hailort/releases/download/v${hailo_version}/hailort-${hailo_version}-cp39-cp39-linux_${arch}.whl"
+wget -P /hailo-wheels/ "https://github.com/frigate-nvr/hailort/releases/download/v${hailo_version}/hailort-${hailo_version}-cp311-cp311-linux_${arch}.whl"
 

docker/hailo8l/requirements-wheels-h8l.txt

@@ -1,12 +1,12 @@
-appdirs==1.4.4
-argcomplete==2.0.0
-contextlib2==0.6.0.post1
-distlib==0.3.6
-filelock==3.8.0
-future==0.18.2
-importlib-metadata==5.1.0
-importlib-resources==5.1.2
-netaddr==0.8.0
-netifaces==0.10.9
-verboselogs==1.7
-virtualenv==20.17.0
+appdirs==1.4.*
+argcomplete==2.0.*
+contextlib2==0.6.*
+distlib==0.3.*
+filelock==3.8.*
+future==0.18.*
+importlib-metadata==5.1.*
+importlib-resources==5.1.*
+netaddr==0.8.*
+netifaces==0.10.*
+verboselogs==1.7.*
+virtualenv==20.17.*

docker/main/Dockerfile

@@ -3,12 +3,12 @@
 # https://askubuntu.com/questions/972516/debian-frontend-environment-variable
 ARG DEBIAN_FRONTEND=noninteractive
 
-ARG BASE_IMAGE=debian:11
-ARG SLIM_BASE=debian:11-slim
+ARG BASE_IMAGE=debian:12
+ARG SLIM_BASE=debian:12-slim
 
 FROM ${BASE_IMAGE} AS base
 
-FROM --platform=${BUILDPLATFORM} debian:11 AS base_host
+FROM --platform=${BUILDPLATFORM} debian:12 AS base_host
 
 FROM ${SLIM_BASE} AS slim-base
 
@@ -66,8 +66,8 @@ COPY docker/main/requirements-ov.txt /requirements-ov.txt
 RUN apt-get -qq update \
     && apt-get -qq install -y wget python3 python3-dev python3-distutils gcc pkg-config libhdf5-dev \
     && wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
-    && python3 get-pip.py "pip" \
-    && pip install -r /requirements-ov.txt
+    && python3 get-pip.py "pip" --break-system-packages \
+    && pip install --break-system-packages -r /requirements-ov.txt
 
 # Get OpenVino Model
 RUN --mount=type=bind,source=docker/main/build_ov_model.py,target=/build_ov_model.py \
@@ -139,24 +139,17 @@ ARG TARGETARCH
 # Use a separate container to build wheels to prevent build dependencies in final image
 RUN apt-get -qq update \
     && apt-get -qq install -y \
-    apt-transport-https \
-    gnupg \
-    wget \
-    # the key fingerprint can be obtained from https://ftp-master.debian.org/keys.html
-    && wget -qO- "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA4285295FC7B1A81600062A9605C66F00D6C9793" | \
-    gpg --dearmor > /usr/share/keyrings/debian-archive-bullseye-stable.gpg \
-    && echo "deb [signed-by=/usr/share/keyrings/debian-archive-bullseye-stable.gpg] http://deb.debian.org/debian bullseye main contrib non-free" | \
-    tee /etc/apt/sources.list.d/debian-bullseye-nonfree.list \
+    apt-transport-https wget \
     && apt-get -qq update \
     && apt-get -qq install -y \
-    python3.9 \
-    python3.9-dev \
+    python3 \
+    python3-dev \
     # opencv dependencies
     build-essential cmake git pkg-config libgtk-3-dev \
     libavcodec-dev libavformat-dev libswscale-dev libv4l-dev \
     libxvidcore-dev libx264-dev libjpeg-dev libpng-dev libtiff-dev \
     gfortran openexr libatlas-base-dev libssl-dev\
-    libtbb2 libtbb-dev libdc1394-22-dev libopenexr-dev \
+    libtbbmalloc2 libtbb-dev libdc1394-dev libopenexr-dev \
     libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev \
     # sqlite3 dependencies
     tclsh \
@@ -164,14 +157,11 @@ RUN apt-get -qq update \
     gcc gfortran libopenblas-dev liblapack-dev && \
     rm -rf /var/lib/apt/lists/*
 
-# Ensure python3 defaults to python3.9
-RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1
-
 RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
-    && python3 get-pip.py "pip"
+    && python3 get-pip.py "pip" --break-system-packages
 
 COPY docker/main/requirements.txt /requirements.txt
-RUN pip3 install -r /requirements.txt
+RUN pip3 install -r /requirements.txt --break-system-packages
 
 # Build pysqlite3 from source
 COPY docker/main/build_pysqlite3.sh /build_pysqlite3.sh
@@ -222,8 +212,8 @@ RUN --mount=type=bind,source=docker/main/install_deps.sh,target=/deps/install_de
     /deps/install_deps.sh
 
 RUN --mount=type=bind,from=wheels,source=/wheels,target=/deps/wheels \
-    python3 -m pip install --upgrade pip && \
-    pip3 install -U /deps/wheels/*.whl
+    python3 -m pip install --upgrade pip --break-system-packages && \
+    pip3 install -U /deps/wheels/*.whl --break-system-packages
 
 COPY --from=deps-rootfs / /
 
@@ -270,7 +260,7 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/*
 
 RUN --mount=type=bind,source=./docker/main/requirements-dev.txt,target=/workspace/frigate/requirements-dev.txt \
-    pip3 install -r requirements-dev.txt
+    pip3 install -r requirements-dev.txt --break-system-packages
 
 HEALTHCHECK NONE
 

docker/main/build_nginx.sh

@@ -8,8 +8,7 @@ SECURE_TOKEN_MODULE_VERSION="1.5"
 SET_MISC_MODULE_VERSION="v0.33"
 NGX_DEVEL_KIT_VERSION="v0.3.3"
 
-cp /etc/apt/sources.list /etc/apt/sources.list.d/sources-src.list
-sed -i 's|deb http|deb-src http|g' /etc/apt/sources.list.d/sources-src.list
+sed -i '/^Types:/s/deb/& deb-src/' /etc/apt/sources.list.d/debian.sources
 apt-get update
 
 apt-get -yqq build-dep nginx

docker/main/build_ov_model.py

@@ -4,7 +4,7 @@ from openvino.tools import mo
 ov_model = mo.convert_model(
     "/models/ssdlite_mobilenet_v2_coco_2018_05_09/frozen_inference_graph.pb",
     compress_to_fp16=True,
-    transformations_config="/usr/local/lib/python3.9/dist-packages/openvino/tools/mo/front/tf/ssd_v2_support.json",
+    transformations_config="/usr/local/lib/python3.11/dist-packages/openvino/tools/mo/front/tf/ssd_v2_support.json",
     tensorflow_object_detection_api_pipeline_config="/models/ssdlite_mobilenet_v2_coco_2018_05_09/pipeline.config",
     reverse_input_channels=True,
 )

docker/main/build_sqlite_vec.sh

@@ -4,8 +4,7 @@ set -euxo pipefail
 
 SQLITE_VEC_VERSION="0.1.3"
 
-cp /etc/apt/sources.list /etc/apt/sources.list.d/sources-src.list
-sed -i 's|deb http|deb-src http|g' /etc/apt/sources.list.d/sources-src.list
+sed -i '/^Types:/s/deb/& deb-src/' /etc/apt/sources.list.d/debian.sources
 apt-get update
 apt-get -yqq build-dep sqlite3 gettext git
 

docker/main/install_deps.sh

@@ -11,33 +11,34 @@ apt-get -qq install --no-install-recommends -y \
     lbzip2 \
     procps vainfo \
     unzip locales tzdata libxml2 xz-utils \
-    python3.9 \
+    python3 \
     python3-pip \
     curl \
     lsof \
     jq \
-    nethogs
-
-# ensure python3 defaults to python3.9
-update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1
+    nethogs \
+    libgl1 \
+    libglib2.0-0 \
+    libusb-1.0.0
 
 mkdir -p -m 600 /root/.gnupg
 
-# add coral repo
-curl -fsSLo - https://packages.cloud.google.com/apt/doc/apt-key.gpg | \
-    gpg --dearmor -o /etc/apt/trusted.gpg.d/google-cloud-packages-archive-keyring.gpg
-echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" | tee /etc/apt/sources.list.d/coral-edgetpu.list
-echo "libedgetpu1-max libedgetpu/accepted-eula select true" | debconf-set-selections
+# install coral runtime
+wget -q -O /tmp/libedgetpu1-max.deb "https://github.com/feranick/libedgetpu/releases/download/16.0TF2.17.0-1/libedgetpu1-max_16.0tf2.17.0-1.bookworm_${TARGETARCH}.deb"
+unset DEBIAN_FRONTEND
+yes | dpkg -i /tmp/libedgetpu1-max.deb && export DEBIAN_FRONTEND=noninteractive
+rm /tmp/libedgetpu1-max.deb
 
-# enable non-free repo in Debian
-if grep -q "Debian" /etc/issue; then
-    sed -i -e's/ main/ main contrib non-free/g' /etc/apt/sources.list
+# install python3 & tflite runtime
+if [[ "${TARGETARCH}" == "amd64" ]]; then
+    pip3 install --break-system-packages https://github.com/feranick/TFlite-builds/releases/download/v2.17.0/tflite_runtime-2.17.0-cp311-cp311-linux_x86_64.whl
+    pip3 install --break-system-packages https://github.com/feranick/pycoral/releases/download/2.0.2TF2.17.0/pycoral-2.0.2-cp311-cp311-linux_x86_64.whl
 fi
 
-# coral drivers
-apt-get -qq update
-apt-get -qq install --no-install-recommends --no-install-suggests -y \
-    libedgetpu1-max python3-tflite-runtime python3-pycoral
+if [[ "${TARGETARCH}" == "arm64" ]]; then
+    pip3 install --break-system-packages https://github.com/feranick/TFlite-builds/releases/download/v2.17.0/tflite_runtime-2.17.0-cp311-cp311-linux_aarch64.whl
+    pip3 install --break-system-packages https://github.com/feranick/pycoral/releases/download/2.0.2TF2.17.0/pycoral-2.0.2-cp311-cp311-linux_aarch64.whl
+fi
 
 # btbn-ffmpeg -> amd64
 if [[ "${TARGETARCH}" == "amd64" ]]; then
@@ -65,23 +66,15 @@ fi
 
 # arch specific packages
 if [[ "${TARGETARCH}" == "amd64" ]]; then
-    # use debian bookworm for amd / intel-i965 driver packages
-    echo 'deb https://deb.debian.org/debian bookworm main contrib non-free' >/etc/apt/sources.list.d/debian-bookworm.list
-    apt-get -qq update
+    # install amd / intel-i965 driver packages
     apt-get -qq install --no-install-recommends --no-install-suggests -y \
         i965-va-driver intel-gpu-tools onevpl-tools \
         libva-drm2 \
         mesa-va-drivers radeontop
 
-    # something about this dependency requires it to be installed in a separate call rather than in the line above
-    apt-get -qq install --no-install-recommends --no-install-suggests -y \
-        i965-va-driver-shaders
-
     # intel packages use zst compression so we need to update dpkg
     apt-get install -y dpkg
 
-    rm -f /etc/apt/sources.list.d/debian-bookworm.list
-
     # use intel apt intel packages
     wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | gpg --yes --dearmor --output /usr/share/keyrings/intel-graphics.gpg
     echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu jammy client" | tee /etc/apt/sources.list.d/intel-gpu-jammy.list

docker/main/requirements-wheels.txt

@@ -10,10 +10,10 @@ imutils == 0.5.*
 joserfc == 1.0.*
 pathvalidate == 3.2.*
 markupsafe == 2.1.*
+python-multipart == 0.0.12
+# General
 mypy == 1.6.1
-numpy == 1.26.*
 onvif_zeep == 0.2.12
-opencv-python-headless == 4.9.0.*
 paho-mqtt == 2.1.*
 pandas == 2.2.*
 peewee == 3.17.*
@@ -27,15 +27,19 @@ ruamel.yaml == 0.18.*
 tzlocal == 5.2
 requests == 2.32.*
 types-requests == 2.32.*
-scipy == 1.13.*
 norfair == 2.2.*
 setproctitle == 1.3.*
 ws4py == 0.5.*
 unidecode == 1.3.*
+# Image Manipulation
+numpy == 1.26.*
+opencv-python-headless == 4.10.0.*
+opencv-contrib-python == 4.9.0.*
+scipy == 1.14.*
 # OpenVino & ONNX
-openvino == 2024.3.*
-onnxruntime-openvino == 1.19.* ; platform_machine == 'x86_64'
-onnxruntime == 1.19.* ; platform_machine == 'aarch64'
+openvino == 2024.4.*
+onnxruntime-openvino == 1.20.* ; platform_machine == 'x86_64'
+onnxruntime == 1.20.* ; platform_machine == 'aarch64'
 # Embeddings
 transformers == 4.45.*
 # Generative AI
@@ -45,3 +49,6 @@ openai == 1.51.*
 # push notifications
 py-vapid == 1.9.*
 pywebpush == 2.0.*
+# alpr
+pyclipper == 1.3.*
+shapely == 2.0.*

docker/main/requirements.txt

@@ -1,2 +1,2 @@
-scikit-build == 0.17.*
+scikit-build == 0.18.*
 nvidia-pyindex

docker/main/rootfs/usr/local/nginx/conf/nginx.conf

@@ -81,6 +81,9 @@ http {
         open_file_cache_errors on;
         aio on;
 
+        # file upload size
+        client_max_body_size 10M;
+
         # https://github.com/kaltura/nginx-vod-module#vod_open_file_thread_pool
         vod_open_file_thread_pool default;
 

docker/rockchip/Dockerfile

@@ -7,13 +7,14 @@ FROM wheels as rk-wheels
 COPY docker/main/requirements-wheels.txt /requirements-wheels.txt
 COPY docker/rockchip/requirements-wheels-rk.txt /requirements-wheels-rk.txt
 RUN sed -i "/https:\/\//d" /requirements-wheels.txt
+RUN python3 -m pip config set global.break-system-packages true
 RUN pip3 wheel --wheel-dir=/rk-wheels -c /requirements-wheels.txt -r /requirements-wheels-rk.txt
 
 FROM deps AS rk-frigate
 ARG TARGETARCH
 
 RUN --mount=type=bind,from=rk-wheels,source=/rk-wheels,target=/deps/rk-wheels \
-    pip3 install -U /deps/rk-wheels/*.whl
+    pip3 install -U /deps/rk-wheels/*.whl --break-system-packages
 
 WORKDIR /opt/frigate/
 COPY --from=rootfs / /

docker/rockchip/requirements-wheels-rk.txt

@@ -1 +1 @@
-rknn-toolkit-lite2 @ https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.0.0/rknn_toolkit_lite2-2.0.0b0-cp39-cp39-linux_aarch64.whl
+rknn-toolkit-lite2 @ https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.0.0/rknn_toolkit_lite2-2.0.0b0-cp311-cp311-linux_aarch64.whl

docker/rocm/Dockerfile

@@ -34,7 +34,7 @@ RUN mkdir -p /opt/rocm-dist/etc/ld.so.conf.d/
 RUN echo /opt/rocm/lib|tee /opt/rocm-dist/etc/ld.so.conf.d/rocm.conf
 
 #######################################################################
-FROM --platform=linux/amd64 debian:11 as debian-base
+FROM --platform=linux/amd64 debian:12 as debian-base
 
 RUN apt-get update && apt-get -y upgrade
 RUN apt-get -y install --no-install-recommends libelf1 libdrm2 libdrm-amdgpu1 libnuma1 kmod
@@ -51,7 +51,7 @@ COPY --from=rocm /opt/rocm-$ROCM /opt/rocm-$ROCM
 RUN ln -s /opt/rocm-$ROCM /opt/rocm
 
 RUN apt-get -y install g++ cmake
-RUN apt-get -y install python3-pybind11 python3.9-distutils python3-dev
+RUN apt-get -y install python3-pybind11 python3-distutils python3-dev
 
 WORKDIR /opt/build
 
@@ -70,10 +70,11 @@ RUN apt-get -y install libnuma1
 WORKDIR /opt/frigate/
 COPY --from=rootfs / /
 
-COPY docker/rocm/requirements-wheels-rocm.txt /requirements.txt
-RUN python3 -m pip install --upgrade pip \
-    && pip3 uninstall -y onnxruntime-openvino \
-    && pip3 install -r /requirements.txt
+# Temporarily disabled to see if a new wheel can be built to support py3.11
+#COPY docker/rocm/requirements-wheels-rocm.txt /requirements.txt
+#RUN python3 -m pip install --upgrade pip \
+#    && pip3 uninstall -y onnxruntime-openvino \
+#    && pip3 install -r /requirements.txt
 
 #######################################################################
 FROM scratch AS rocm-dist
@@ -86,12 +87,12 @@ COPY --from=rocm /opt/rocm-$ROCM/share/miopen/db/*$AMDGPU* /opt/rocm-$ROCM/share
 COPY --from=rocm /opt/rocm-$ROCM/share/miopen/db/*gfx908* /opt/rocm-$ROCM/share/miopen/db/
 COPY --from=rocm /opt/rocm-$ROCM/lib/rocblas/library/*$AMDGPU* /opt/rocm-$ROCM/lib/rocblas/library/
 COPY --from=rocm /opt/rocm-dist/ /
-COPY --from=debian-build /opt/rocm/lib/migraphx.cpython-39-x86_64-linux-gnu.so /opt/rocm-$ROCM/lib/
+COPY --from=debian-build /opt/rocm/lib/migraphx.cpython-311-x86_64-linux-gnu.so /opt/rocm-$ROCM/lib/
 
 #######################################################################
 FROM deps-prelim AS rocm-prelim-hsa-override0
-
-ENV HSA_ENABLE_SDMA=0
+\
+    ENV HSA_ENABLE_SDMA=0
 
 COPY --from=rocm-dist / /
 

docker/rpi/install_deps.sh

@@ -24,7 +24,7 @@ sed -i -e's/ main/ main contrib non-free/g' /etc/apt/sources.list
 if [[ "${TARGETARCH}" == "arm64" ]]; then
     # add raspberry pi repo
     gpg --no-default-keyring --keyring /usr/share/keyrings/raspbian.gpg --keyserver keyserver.ubuntu.com --recv-keys 82B129927FA3303E
-    echo "deb [signed-by=/usr/share/keyrings/raspbian.gpg] https://archive.raspberrypi.org/debian/ bullseye main" | tee /etc/apt/sources.list.d/raspi.list
+    echo "deb [signed-by=/usr/share/keyrings/raspbian.gpg] https://archive.raspberrypi.org/debian/ bookworm main" | tee /etc/apt/sources.list.d/raspi.list
     apt-get -qq update
     apt-get -qq install --no-install-recommends --no-install-suggests -y ffmpeg
 fi

docker/tensorrt/Dockerfile.amd64

@@ -7,33 +7,19 @@ ARG DEBIAN_FRONTEND=noninteractive
 FROM wheels as trt-wheels
 ARG DEBIAN_FRONTEND
 ARG TARGETARCH
+RUN python3 -m pip config set global.break-system-packages true
 
 # Add TensorRT wheels to another folder
 COPY docker/tensorrt/requirements-amd64.txt /requirements-tensorrt.txt
 RUN mkdir -p /trt-wheels && pip3 wheel --wheel-dir=/trt-wheels -r /requirements-tensorrt.txt
 
-# Build CuDNN
-FROM wget AS cudnn-deps
-
-ARG COMPUTE_LEVEL
-
-RUN apt-get update \
-    && apt-get install -y git build-essential
-
-RUN wget https://developer.download.nvidia.com/compute/cuda/repos/debian11/x86_64/cuda-keyring_1.1-1_all.deb \
-&& dpkg -i cuda-keyring_1.1-1_all.deb \
-&& apt-get update \
-&& apt-get -y install cuda-toolkit \
-&& rm -rf /var/lib/apt/lists/*
-
 FROM tensorrt-base AS frigate-tensorrt
-ENV TRT_VER=8.5.3
+ENV TRT_VER=8.6.1
+RUN python3 -m pip config set global.break-system-packages true
 RUN --mount=type=bind,from=trt-wheels,source=/trt-wheels,target=/deps/trt-wheels \
-    pip3 install -U /deps/trt-wheels/*.whl && \
+    pip3 install -U /deps/trt-wheels/*.whl --break-system-packages && \
     ldconfig
-COPY --from=cudnn-deps /usr/local/cuda-12.6 /usr/local/cuda
 
-ENV LD_LIBRARY_PATH=/usr/local/lib/python3.9/dist-packages/tensorrt:/usr/local/cuda/lib64:/usr/local/lib/python3.9/dist-packages/nvidia/cufft/lib
 WORKDIR /opt/frigate/
 COPY --from=rootfs / /
 
@@ -42,8 +28,8 @@ FROM devcontainer AS devcontainer-trt
 
 COPY --from=trt-deps /usr/local/lib/libyolo_layer.so /usr/local/lib/libyolo_layer.so
 COPY --from=trt-deps /usr/local/src/tensorrt_demos /usr/local/src/tensorrt_demos
-COPY --from=cudnn-deps /usr/local/cuda-12.6 /usr/local/cuda
+COPY --from=trt-deps /usr/local/cuda-12.1 /usr/local/cuda
 COPY docker/tensorrt/detector/rootfs/ /
 COPY --from=trt-deps /usr/local/lib/libyolo_layer.so /usr/local/lib/libyolo_layer.so
 RUN --mount=type=bind,from=trt-wheels,source=/trt-wheels,target=/deps/trt-wheels \
-    pip3 install -U /deps/trt-wheels/*.whl
+    pip3 install -U /deps/trt-wheels/*.whl --break-system-packages

docker/tensorrt/Dockerfile.arm64

@@ -41,11 +41,11 @@ RUN --mount=type=bind,source=docker/tensorrt/detector/build_python_tensorrt.sh,t
     && TENSORRT_VER=$(cat /etc/TENSORRT_VER) /deps/build_python_tensorrt.sh
 
 COPY docker/tensorrt/requirements-arm64.txt /requirements-tensorrt.txt
-ADD https://nvidia.box.com/shared/static/9aemm4grzbbkfaesg5l7fplgjtmswhj8.whl /tmp/onnxruntime_gpu-1.15.1-cp39-cp39-linux_aarch64.whl
+ADD https://nvidia.box.com/shared/static/psl23iw3bh7hlgku0mjo1xekxpego3e3.whl /tmp/onnxruntime_gpu-1.15.1-cp311-cp311-linux_aarch64.whl
 
 RUN pip3 uninstall -y onnxruntime-openvino \
     && pip3 wheel --wheel-dir=/trt-wheels -r /requirements-tensorrt.txt \
-    && pip3 install --no-deps /tmp/onnxruntime_gpu-1.15.1-cp39-cp39-linux_aarch64.whl
+    && pip3 install --no-deps /tmp/onnxruntime_gpu-1.15.1-cp311-cp311-linux_aarch64.whl
 
 FROM build-wheels AS trt-model-wheels
 ARG DEBIAN_FRONTEND

docker/tensorrt/Dockerfile.base

@@ -3,18 +3,19 @@
 # https://askubuntu.com/questions/972516/debian-frontend-environment-variable
 ARG DEBIAN_FRONTEND=noninteractive
 
-ARG TRT_BASE=nvcr.io/nvidia/tensorrt:23.03-py3
+ARG TRT_BASE=nvcr.io/nvidia/tensorrt:24.10-py3
 
 # Build TensorRT-specific library
 FROM ${TRT_BASE} AS trt-deps
 
 ARG COMPUTE_LEVEL
 
-RUN apt-get update \
-    && apt-get install -y git build-essential cuda-nvcc-* cuda-nvtx-* libnvinfer-dev libnvinfer-plugin-dev libnvparsers-dev libnvonnxparsers-dev \
-    && rm -rf /var/lib/apt/lists/*
-RUN --mount=type=bind,source=docker/tensorrt/detector/tensorrt_libyolo.sh,target=/tensorrt_libyolo.sh \
-    /tensorrt_libyolo.sh
+# Need to wait for script to be adapted to newer version of tensorrt or perhaps decide that we want to remove the TRT detector in favor of using onnx runtime directly
+#RUN apt-get update \
+#    && apt-get install -y git build-essential cuda-nvcc-* cuda-nvtx-* libnvinfer-dev libnvinfer-plugin-dev libnvparsers-dev libnvonnxparsers-dev \
+#    && rm -rf /var/lib/apt/lists/*
+#RUN --mount=type=bind,source=docker/tensorrt/detector/tensorrt_libyolo.sh,target=/tensorrt_libyolo.sh \
+#    /tensorrt_libyolo.sh
 
 # Frigate w/ TensorRT Support as separate image
 FROM deps AS tensorrt-base
@@ -22,8 +23,12 @@ FROM deps AS tensorrt-base
 #Disable S6 Global timeout
 ENV S6_CMD_WAIT_FOR_SERVICES_MAXTIME=0
 
-COPY --from=trt-deps /usr/local/lib/libyolo_layer.so /usr/local/lib/libyolo_layer.so
-COPY --from=trt-deps /usr/local/src/tensorrt_demos /usr/local/src/tensorrt_demos
+#COPY --from=trt-deps /usr/local/lib/libyolo_layer.so /usr/local/lib/libyolo_layer.so
+#COPY --from=trt-deps /usr/local/src/tensorrt_demos /usr/local/src/tensorrt_demos
+
+COPY --from=trt-deps /usr/lib/x86_64-linux-gnu/libcudnn* /usr/local/cuda/lib64/
+COPY --from=trt-deps /usr/lib/x86_64-linux-gnu/libnv* /usr/local/cuda/lib64/
+
 COPY docker/tensorrt/detector/rootfs/ /
 ENV YOLO_MODELS=""
 

docker/tensorrt/detector/rootfs/etc/ld.so.conf.d/cuda_tensorrt.conf

@@ -1,6 +1,8 @@
 /usr/local/lib
-/usr/local/lib/python3.9/dist-packages/nvidia/cudnn/lib
-/usr/local/lib/python3.9/dist-packages/nvidia/cuda_runtime/lib
-/usr/local/lib/python3.9/dist-packages/nvidia/cublas/lib
-/usr/local/lib/python3.9/dist-packages/nvidia/cuda_nvrtc/lib
-/usr/local/lib/python3.9/dist-packages/tensorrt
+/usr/local/cuda/lib64
+/usr/local/lib/python3.11/dist-packages/nvidia/cudnn/lib
+/usr/local/lib/python3.11/dist-packages/nvidia/cuda_runtime/lib
+/usr/local/lib/python3.11/dist-packages/nvidia/cublas/lib
+/usr/local/lib/python3.11/dist-packages/nvidia/cuda_nvrtc/lib
+/usr/local/lib/python3.11/dist-packages/tensorrt
+/usr/local/lib/python3.11/dist-packages/nvidia/cufft/lib

docker/tensorrt/requirements-amd64.txt

@@ -1,14 +1,10 @@
 # NVidia TensorRT Support (amd64 only)
 --extra-index-url 'https://pypi.nvidia.com'
 numpy < 1.24; platform_machine == 'x86_64'
-tensorrt == 8.5.3.*; platform_machine == 'x86_64'
-cuda-python == 11.8; platform_machine == 'x86_64'
-cython == 0.29.*; platform_machine == 'x86_64'
+tensorrt == 10.5.0; platform_machine == 'x86_64'
+cuda-python == 12.6.*; platform_machine == 'x86_64'
+cython == 3.0.*; platform_machine == 'x86_64'
 nvidia-cuda-runtime-cu12 == 12.1.*; platform_machine == 'x86_64'
-nvidia-cuda-runtime-cu11 == 11.8.*; platform_machine == 'x86_64'
-nvidia-cublas-cu11 == 11.11.3.6; platform_machine == 'x86_64'
-nvidia-cudnn-cu11 == 8.6.0.*; platform_machine == 'x86_64'
-nvidia-cufft-cu11==10.*; platform_machine == 'x86_64'
 onnx==1.16.*; platform_machine == 'x86_64'
-onnxruntime-gpu==1.18.*; platform_machine == 'x86_64'
+onnxruntime-gpu==1.20.*; platform_machine == 'x86_64'
 protobuf==3.20.3; platform_machine == 'x86_64'

docs/docs/configuration/advanced.md

@@ -174,7 +174,7 @@ NOTE: The folder that is set for the config needs to be the folder that contains
 
 ### Custom go2rtc version
 
-Frigate currently includes go2rtc v1.9.4, there may be certain cases where you want to run a different version of go2rtc.
+Frigate currently includes go2rtc v1.9.2, there may be certain cases where you want to run a different version of go2rtc.
 
 To do this:
 

docs/docs/configuration/autotracking.md

@@ -41,6 +41,7 @@ cameras:
     ...
     onvif:
       # Required: host of the camera being connected to.
+      # NOTE: HTTP is assumed by default; HTTPS is supported if you specify the scheme, ex: "https://0.0.0.0".
       host: 0.0.0.0
       # Optional: ONVIF port for device (default: shown below).
       port: 8000
@@ -49,6 +50,8 @@ cameras:
       user: admin
       # Optional: password for login.
       password: admin
+      # Optional: Skip TLS verification from the ONVIF server (default: shown below)
+      tls_insecure: False
       # Optional: PTZ camera object autotracking. Keeps a moving object in
       # the center of the frame by automatically moving the PTZ camera.
       autotracking:

docs/docs/configuration/face_recognition.md

@@ -0,0 +1,35 @@
+---
+id: face_recognition
+title: Face Recognition
+---
+
+Face recognition allows people to be assigned names and when their face is recognized Frigate will assign the person's name as a sub label. This information is included in the UI, filters, as well as in notifications.
+
+Frigate has support for FaceNet to create face embeddings, which runs locally. Embeddings are then saved to Frigate's database.
+
+## Minimum System Requirements
+
+Face recognition works by running a large AI model locally on your system. Systems without a GPU will not run Face Recognition reliably or at all.
+
+## Configuration
+
+Face recognition is disabled by default and requires semantic search to be enabled, face recognition must be enabled in your config file before it can be used. Semantic Search and face recognition are global configuration settings.
+
+```yaml
+face_recognition:
+  enabled: true
+```
+
+## Dataset
+
+The number of images needed for a sufficient training set for face recognition varies depending on several factors:
+
+- Complexity of the task: A simple task like recognizing faces of known individuals may require fewer images than a complex task like identifying unknown individuals in a large crowd.
+- Diversity of the dataset: A dataset with diverse images, including variations in lighting, pose, and facial expressions, will require fewer images per person than a less diverse dataset.
+- Desired accuracy: The higher the desired accuracy, the more images are typically needed.
+
+However, here are some general guidelines:
+
+- Minimum: For basic face recognition tasks, a minimum of 10-20 images per person is often recommended.
+- Recommended: For more robust and accurate systems, 30-50 images per person is a good starting point.
+- Ideal: For optimal performance, especially in challenging conditions, 100 or more images per person can be beneficial.

docs/docs/configuration/genai.md

@@ -5,6 +5,8 @@ title: Generative AI
 
 Generative AI can be used to automatically generate descriptive text based on the thumbnails of your tracked objects. This helps with [Semantic Search](/configuration/semantic_search) in Frigate to provide more context about your tracked objects. Descriptions are accessed via the _Explore_ view in the Frigate UI by clicking on a tracked object's thumbnail.
 
+Requests for a description are sent off automatically to your AI provider at the end of the tracked object's lifecycle. Descriptions can also be regenerated manually via the Frigate UI.
+
 :::info
 
 Semantic Search must be enabled to use Generative AI.

docs/docs/configuration/hardware_acceleration.md

@@ -231,28 +231,11 @@ docker run -d \
 
 ### Setup Decoder
 
-The decoder you need to pass in the `hwaccel_args` will depend on the input video.
-
-A list of supported codecs (you can use `ffmpeg -decoders | grep cuvid` in the container to get the ones your card supports)
-
-```
- V..... h263_cuvid           Nvidia CUVID H263 decoder (codec h263)
- V..... h264_cuvid           Nvidia CUVID H264 decoder (codec h264)
- V..... hevc_cuvid           Nvidia CUVID HEVC decoder (codec hevc)
- V..... mjpeg_cuvid          Nvidia CUVID MJPEG decoder (codec mjpeg)
- V..... mpeg1_cuvid          Nvidia CUVID MPEG1VIDEO decoder (codec mpeg1video)
- V..... mpeg2_cuvid          Nvidia CUVID MPEG2VIDEO decoder (codec mpeg2video)
- V..... mpeg4_cuvid          Nvidia CUVID MPEG4 decoder (codec mpeg4)
- V..... vc1_cuvid            Nvidia CUVID VC1 decoder (codec vc1)
- V..... vp8_cuvid            Nvidia CUVID VP8 decoder (codec vp8)
- V..... vp9_cuvid            Nvidia CUVID VP9 decoder (codec vp9)
-```
-
-For example, for H264 video, you'll select `preset-nvidia-h264`.
+Using `preset-nvidia` ffmpeg will automatically select the necessary profile for the incoming video, and will log an error if the profile is not supported by your GPU.
 
 ```yaml
 ffmpeg:
-  hwaccel_args: preset-nvidia-h264
+  hwaccel_args: preset-nvidia
 ```
 
 If everything is working correctly, you should see a significant improvement in performance.

docs/docs/configuration/license_plate_recognition.md

@@ -0,0 +1,45 @@
+---
+id: license_plate_recognition
+title: License Plate Recognition (LPR)
+---
+
+Frigate can recognize license plates on vehicles and automatically add the detected characters as a `sub_label` to objects that are of type `car`. A common use case may be to read the license plates of cars pulling into a driveway or cars passing by on a street with a dedicated LPR camera.
+
+Users running a Frigate+ model should ensure that `license_plate` is added to the [list of objects to track](https://docs.frigate.video/plus/#available-label-types) either globally or for a specific camera. This will improve the accuracy and performance of the LPR model.
+
+LPR is most effective when the vehicle’s license plate is fully visible to the camera. For moving vehicles, Frigate will attempt to read the plate continuously, refining its detection and keeping the most confident result. LPR will not run on stationary vehicles.
+
+## Minimum System Requirements
+
+License plate recognition works by running AI models locally on your system. The models are relatively lightweight and run on your CPU. At least 4GB of RAM is required.
+
+## Configuration
+
+License plate recognition is disabled by default. Enable it in your config file:
+
+```yaml
+lpr:
+  enabled: true
+```
+
+## Advanced Configuration
+
+Several options are available to fine-tune the LPR feature. For example, you can adjust the `min_area` setting, which defines the minimum size in pixels a license plate must be before LPR runs. The default is 500 pixels.
+
+Additionally, you can define `known_plates` as strings or regular expressions, allowing Frigate to label tracked vehicles with custom sub_labels when a recognized plate is detected. This information is then accessible in the UI, filters, and notifications.
+
+```yaml
+lpr:
+  enabled: true
+  min_area: 500
+  known_plates:
+    Wife's Car:
+      - "ABC-1234"
+      - "ABC-I234"
+    Johnny:
+      - "J*N-*234" # Using wildcards for H/M and 1/I
+    Sally:
+      - "[S5]LL-1234" # Matches SLL-1234 and 5LL-1234
+```
+
+In this example, "Wife's Car" will appear as the label for any vehicle matching the plate "ABC-1234." The model might occasionally interpret the digit 1 as a capital I (e.g., "ABC-I234"), so both variations are listed. Similarly, multiple possible variations are specified for Johnny and Sally.

docs/docs/configuration/live.md

@@ -23,7 +23,7 @@ If you are using go2rtc, you should adjust the following settings in your camera
 
 - Video codec: **H.264** - provides the most compatible video codec with all Live view technologies and browsers. Avoid any kind of "smart codec" or "+" codec like _H.264+_ or _H.265+_. as these non-standard codecs remove keyframes (see below).
 - Audio codec: **AAC** - provides the most compatible audio codec with all Live view technologies and browsers that support audio.
-- I-frame interval (sometimes called the keyframe interval, the interframe space, or the GOP length): match your camera's frame rate, or choose "1x" (for interframe space on Reolink cameras). For example, if your stream outputs 20fps, your i-frame interval should be 20 (or 1x on Reolink). Values higher than the frame rate will cause the stream to take longer to begin playback. See [this page](https://gardinal.net/understanding-the-keyframe-interval/) for more on keyframes.
+- I-frame interval (sometimes called the keyframe interval, the interframe space, or the GOP length): match your camera's frame rate, or choose "1x" (for interframe space on Reolink cameras). For example, if your stream outputs 20fps, your i-frame interval should be 20 (or 1x on Reolink). Values higher than the frame rate will cause the stream to take longer to begin playback. See [this page](https://gardinal.net/understanding-the-keyframe-interval/) for more on keyframes. For many users this may not be an issue, but it should be noted that that a 1x i-frame interval will cause more storage utilization if you are using the stream for the `record` role as well.
 
 The default video and audio codec on your camera may not always be compatible with your browser, which is why setting them to H.264 and AAC is recommended. See the [go2rtc docs](https://github.com/AlexxIT/go2rtc?tab=readme-ov-file#codecs-madness) for codec support information.
 

docs/docs/configuration/reference.md

@@ -522,6 +522,14 @@ semantic_search:
   # NOTE: small model runs on CPU and large model runs on GPU
   model_size: "small"
 
+# Optional: Configuration for face recognition capability
+face_recognition:
+  # Optional: Enable semantic search (default: shown below)
+  enabled: False
+  # Optional: Set the model size used for embeddings. (default: shown below)
+  # NOTE: small model runs on CPU and large model runs on GPU
+  model_size: "small"
+
 # Optional: Configuration for AI generated tracked object descriptions
 # NOTE: Semantic Search must be enabled for this to do anything.
 # WARNING: Depending on the provider, this will send thumbnails over the internet
@@ -686,6 +694,7 @@ cameras:
     # to enable PTZ controls.
     onvif:
       # Required: host of the camera being connected to.
+      # NOTE: HTTP is assumed by default; HTTPS is supported if you specify the scheme, ex: "https://0.0.0.0".
       host: 0.0.0.0
       # Optional: ONVIF port for device (default: shown below).
       port: 8000
@@ -694,6 +703,8 @@ cameras:
       user: admin
       # Optional: password for login.
       password: admin
+      # Optional: Skip TLS verification from the ONVIF server (default: shown below)
+      tls_insecure: False
       # Optional: Ignores time synchronization mismatches between the camera and the server during authentication.
       # Using NTP on both ends is recommended and this should only be set to True in a "safe" environment due to the security risk it represents.
       ignore_time_mismatch: False
@@ -757,6 +768,8 @@ cameras:
         - cat
       # Optional: Restrict generation to objects that entered any of the listed zones (default: none, all zones qualify)
       required_zones: []
+      # Optional: Save thumbnails sent to generative AI for review/debugging purposes (default: shown below)
+      debug_save_thumbnails: False
 
 # Optional
 ui:

docs/docs/configuration/restream.md

@@ -132,6 +132,28 @@ cameras:
             - detect
 ```
 
+## Handling Complex Passwords
+
+go2rtc expects URL-encoded passwords in the config, [urlencoder.org](https://urlencoder.org) can be used for this purpose. 
+
+For example:
+
+```yaml
+go2rtc:
+  streams:
+    my_camera: rtsp://username:$@foo%@192.168.1.100
+```
+
+becomes
+
+```yaml
+go2rtc:
+  streams:
+    my_camera: rtsp://username:$%40foo%25@192.168.1.100
+```
+
+See [this comment(https://github.com/AlexxIT/go2rtc/issues/1217#issuecomment-2242296489) for more information.
+
 ## Advanced Restream Configurations
 
 The [exec](https://github.com/AlexxIT/go2rtc/tree/v1.9.2#source-exec) source in go2rtc can be used for custom ffmpeg commands. An example is below:

docs/docs/configuration/semantic_search.md

@@ -5,7 +5,7 @@ title: Using Semantic Search
 
 Semantic Search in Frigate allows you to find tracked objects within your review items using either the image itself, a user-defined text description, or an automatically generated one. This feature works by creating _embeddings_ — numerical vector representations — for both the images and text descriptions of your tracked objects. By comparing these embeddings, Frigate assesses their similarities to deliver relevant search results.
 
-Frigate has support for [Jina AI's CLIP model](https://huggingface.co/jinaai/jina-clip-v1) to create embeddings, which runs locally. Embeddings are then saved to Frigate's database.
+Frigate uses [Jina AI's CLIP model](https://huggingface.co/jinaai/jina-clip-v1) to create and save embeddings to Frigate's database. All of this runs locally.
 
 Semantic Search is accessed via the _Explore_ view in the Frigate UI.
 
@@ -19,7 +19,7 @@ For best performance, 16GB or more of RAM and a dedicated GPU are recommended.
 
 ## Configuration
 
-Semantic Search is disabled by default, and must be enabled in your config file before it can be used. Semantic Search is a global configuration setting.
+Semantic Search is disabled by default, and must be enabled in your config file or in the UI's Settings page before it can be used. Semantic Search is a global configuration setting.
 
 ```yaml
 semantic_search:
@@ -29,9 +29,9 @@ semantic_search:
 
 :::tip
 
-The embeddings database can be re-indexed from the existing tracked objects in your database by adding `reindex: True` to your `semantic_search` configuration. Depending on the number of tracked objects you have, it can take a long while to complete and may max out your CPU while indexing. Make sure to set the config back to `False` before restarting Frigate again.
+The embeddings database can be re-indexed from the existing tracked objects in your database by adding `reindex: True` to your `semantic_search` configuration or by toggling the switch on the Search Settings page in the UI and restarting Frigate. Depending on the number of tracked objects you have, it can take a long while to complete and may max out your CPU while indexing. Make sure to turn the UI's switch off or set the config back to `False` before restarting Frigate again.
 
-If you are enabling the Search feature for the first time, be advised that Frigate does not automatically index older tracked objects. You will need to enable the `reindex` feature in order to do that.
+If you are enabling Semantic Search for the first time, be advised that Frigate does not automatically index older tracked objects. You will need to enable the `reindex` feature in order to do that.
 
 :::
 
@@ -39,9 +39,9 @@ If you are enabling the Search feature for the first time, be advised that Friga
 
 The vision model is able to embed both images and text into the same vector space, which allows `image -> image` and `text -> image` similarity searches. Frigate uses this model on tracked objects to encode the thumbnail image and store it in the database. When searching for tracked objects via text in the search box, Frigate will perform a `text -> image` similarity search against this embedding. When clicking "Find Similar" in the tracked object detail pane, Frigate will perform an `image -> image` similarity search to retrieve the closest matching thumbnails.
 
-The text model is used to embed tracked object descriptions and perform searches against them. Descriptions can be created, viewed, and modified on the Search page when clicking on the gray tracked object chip at the top left of each review item. See [the Generative AI docs](/configuration/genai.md) for more information on how to automatically generate tracked object descriptions.
+The text model is used to embed tracked object descriptions and perform searches against them. Descriptions can be created, viewed, and modified on the Explore page when clicking on thumbnail of a tracked object. See [the Generative AI docs](/configuration/genai.md) for more information on how to automatically generate tracked object descriptions.
 
-Differently weighted CLIP models are available and can be selected by setting the `model_size` config option as `small` or `large`:
+Differently weighted versions of the Jina model are available and can be selected by setting the `model_size` config option as `small` or `large`:
 
 ```yaml
 semantic_search:
@@ -50,7 +50,7 @@ semantic_search:
 ```
 
 - Configuring the `large` model employs the full Jina model and will automatically run on the GPU if applicable.
-- Configuring the `small` model employs a quantized version of the model that uses less RAM and runs on CPU with a very negligible difference in embedding quality.
+- Configuring the `small` model employs a quantized version of the Jina model that uses less RAM and runs on CPU with a very negligible difference in embedding quality.
 
 ### GPU Acceleration
 
@@ -84,7 +84,7 @@ If the correct build is used for your GPU and the `large` model is configured, t
 
 ## Usage and Best Practices
 
-1. Semantic Search is used in conjunction with the other filters available on the Search page. Use a combination of traditional filtering and Semantic Search for the best results.
+1. Semantic Search is used in conjunction with the other filters available on the Explore page. Use a combination of traditional filtering and Semantic Search for the best results.
 2. Use the thumbnail search type when searching for particular objects in the scene. Use the description search type when attempting to discern the intent of your object.
 3. Because of how the AI models Frigate uses have been trained, the comparison between text and image embedding distances generally means that with multi-modal (`thumbnail` and `description`) searches, results matching `description` will appear first, even if a `thumbnail` embedding may be a better match. Play with the "Search Type" setting to help find what you are looking for. Note that if you are generating descriptions for specific objects or zones only, this may cause search results to prioritize the objects with descriptions even if the the ones without them are more relevant.
 4. Make your search language and tone closely match exactly what you're looking for. If you are using thumbnail search, **phrase your query as an image caption**. Searching for "red car" may not work as well as "red sedan driving down a residential street on a sunny day".

docs/docs/frigate/camera_setup.md

@@ -28,7 +28,7 @@ For the Dahua/Loryta 5442 camera, I use the following settings:
 - Encode Mode: H.264
 - Resolution: 2688\*1520
 - Frame Rate(FPS): 15
-- I Frame Interval: 30
+- I Frame Interval: 30 (15 can also be used to prioritize streaming performance - see the [camera settings recommendations](../configuration/live) for more info)
 
 **Sub Stream (Detection)**
 

docs/docs/troubleshooting/faqs.md

@@ -98,3 +98,11 @@ docker run -d \
   -p 8555:8555/udp \
   ghcr.io/blakeblackshear/frigate:stable
 ```
+
+### My RTSP stream works fine in VLC, but it does not work when I put the same URL in my Frigate config. Is this a bug?
+
+No. Frigate uses the TCP protocol to connect to your camera's RTSP URL. VLC automatically switches between UDP and TCP depending on network conditions and stream availability. So a stream that works in VLC but not in Frigate is likely due to VLC selecting UDP as the transfer protocol.
+
+TCP ensures that all data packets arrive in the correct order. This is crucial for video recording, decoding, and stream processing, which is why Frigate enforces a TCP connection. UDP is faster but less reliable, as it does not guarantee packet delivery or order, and VLC does not have the same requirements as Frigate.
+
+You can still configure Frigate to use UDP by using ffmpeg input args or the preset `preset-rtsp-udp`. See the [ffmpeg presets](/configuration/ffmpeg_presets) documentation.

docs/package.json

@@ -17,15 +17,15 @@
     "write-heading-ids": "docusaurus write-heading-ids"
   },
   "dependencies": {
-    "@docusaurus/core": "^3.5.2",
-    "@docusaurus/preset-classic": "^3.5.2",
-    "@docusaurus/theme-mermaid": "^3.5.2",
-    "@docusaurus/plugin-content-docs": "^3.5.2",
-    "@mdx-js/react": "^3.0.1",
+    "@docusaurus/core": "^3.6.3",
+    "@docusaurus/preset-classic": "^3.6.3",
+    "@docusaurus/theme-mermaid": "^3.6.3",
+    "@docusaurus/plugin-content-docs": "^3.6.3",
+    "@mdx-js/react": "^3.1.0",
     "clsx": "^2.1.1",
-    "docusaurus-plugin-openapi-docs": "^4.1.0",
-    "docusaurus-theme-openapi-docs": "^4.1.0",
-    "prism-react-renderer": "^2.4.0",
+    "docusaurus-plugin-openapi-docs": "^4.3.1",
+    "docusaurus-theme-openapi-docs": "^4.3.1",
+    "prism-react-renderer": "^2.4.1",
     "raw-loader": "^4.0.2",
     "react": "^18.3.1",
     "react-dom": "^18.3.1"

docs/sidebars.ts

@@ -36,6 +36,8 @@ const sidebars: SidebarsConfig = {
       'Semantic Search': [
         'configuration/semantic_search',
         'configuration/genai',
+        'configuration/face_recognition',
+        'configuration/license_plate_recognition',
       ],
       Cameras: [
         'configuration/cameras',

frigate/__main__.py

@@ -3,12 +3,15 @@ import faulthandler
 import signal
 import sys
 import threading
+from typing import Union
 
+import ruamel.yaml
 from pydantic import ValidationError
 
 from frigate.app import FrigateApp
 from frigate.config import FrigateConfig
 from frigate.log import setup_logging
+from frigate.util.config import find_config_file
 
 
 def main() -> None:
@@ -42,10 +45,50 @@ def main() -> None:
         print("*************************************************************")
         print("*************************************************************")
         print("***    Config Validation Errors                           ***")
-        print("*************************************************************")
+        print("*************************************************************\n")
+        # Attempt to get the original config file for line number tracking
+        config_path = find_config_file()
+        with open(config_path, "r") as f:
+            yaml_config = ruamel.yaml.YAML()
+            yaml_config.preserve_quotes = True
+            full_config = yaml_config.load(f)
+
         for error in e.errors():
-            location = ".".join(str(item) for item in error["loc"])
-            print(f"{location}: {error['msg']}")
+            error_path = error["loc"]
+
+            current = full_config
+            line_number = "Unknown"
+            last_line_number = "Unknown"
+
+            try:
+                for i, part in enumerate(error_path):
+                    key: Union[int, str] = (
+                        int(part) if isinstance(part, str) and part.isdigit() else part
+                    )
+
+                    if isinstance(current, ruamel.yaml.comments.CommentedMap):
+                        current = current[key]
+                    elif isinstance(current, list):
+                        if isinstance(key, int):
+                            current = current[key]
+
+                    if hasattr(current, "lc"):
+                        last_line_number = current.lc.line
+
+                    if i == len(error_path) - 1:
+                        if hasattr(current, "lc"):
+                            line_number = current.lc.line
+                        else:
+                            line_number = last_line_number
+
+            except Exception as traverse_error:
+                print(f"Could not determine exact line number: {traverse_error}")
+
+            print(f"Line #  : {line_number}")
+            print(f"Key     : {' -> '.join(map(str, error_path))}")
+            print(f"Value   : {error.get('input','-')}")
+            print(f"Message : {error.get('msg', error.get('type', 'Unknown'))}\n")
+
         print("*************************************************************")
         print("***    End Config Validation Errors                       ***")
         print("*************************************************************")

frigate/api/app.py

@@ -7,27 +7,30 @@ import os
 import traceback
 from datetime import datetime, timedelta
 from functools import reduce
+from io import StringIO
 from typing import Any, Optional
 
 import requests
+import ruamel.yaml
 from fastapi import APIRouter, Body, Path, Request, Response
 from fastapi.encoders import jsonable_encoder
 from fastapi.params import Depends
 from fastapi.responses import JSONResponse, PlainTextResponse
 from markupsafe import escape
 from peewee import operator
+from pydantic import ValidationError
 
-from frigate.api.defs.app_body import AppConfigSetBody
-from frigate.api.defs.app_query_parameters import AppTimelineHourlyQueryParameters
+from frigate.api.defs.query.app_query_parameters import AppTimelineHourlyQueryParameters
+from frigate.api.defs.request.app_body import AppConfigSetBody
 from frigate.api.defs.tags import Tags
 from frigate.config import FrigateConfig
-from frigate.const import CONFIG_DIR
 from frigate.models import Event, Timeline
 from frigate.util.builtin import (
     clean_camera_user_pass,
     get_tz_modifiers,
     update_yaml_from_url,
 )
+from frigate.util.config import find_config_file
 from frigate.util.services import (
     ffprobe_stream,
     get_nvidia_driver_info,
@@ -134,9 +137,25 @@ def config(request: Request):
         for zone_name, zone in config_obj.cameras[camera_name].zones.items():
             camera_dict["zones"][zone_name]["color"] = zone.color
 
+    # remove go2rtc stream passwords
+    go2rtc: dict[str, any] = config_obj.go2rtc.model_dump(
+        mode="json", warnings="none", exclude_none=True
+    )
+    for stream_name, stream in go2rtc.get("streams", {}).items():
+        if isinstance(stream, str):
+            cleaned = clean_camera_user_pass(stream)
+        else:
+            cleaned = []
+
+            for item in stream:
+                cleaned.append(clean_camera_user_pass(item))
+
+        config["go2rtc"]["streams"][stream_name] = cleaned
+
     config["plus"] = {"enabled": request.app.frigate_config.plus_api.is_active()}
     config["model"]["colormap"] = config_obj.model.colormap
 
+    # use merged labelamp
     for detector_config in config["detectors"].values():
         detector_config["model"]["labelmap"] = (
             request.app.frigate_config.model.merged_labelmap
@@ -147,13 +166,7 @@ def config(request: Request):
 
 @router.get("/config/raw")
 def config_raw():
-    config_file = os.environ.get("CONFIG_FILE", "/config/config.yml")
-
-    # Check if we can use .yaml instead of .yml
-    config_file_yaml = config_file.replace(".yml", ".yaml")
-
-    if os.path.isfile(config_file_yaml):
-        config_file = config_file_yaml
+    config_file = find_config_file()
 
     if not os.path.isfile(config_file):
         return JSONResponse(
@@ -173,7 +186,6 @@ def config_raw():
 @router.post("/config/save")
 def config_save(save_option: str, body: Any = Body(media_type="text/plain")):
     new_config = body.decode()
-
     if not new_config:
         return JSONResponse(
             content=(
@@ -184,13 +196,64 @@ def config_save(save_option: str, body: Any = Body(media_type="text/plain")):
 
     # Validate the config schema
     try:
+        # Use ruamel to parse and preserve line numbers
+        yaml_config = ruamel.yaml.YAML()
+        yaml_config.preserve_quotes = True
+        full_config = yaml_config.load(StringIO(new_config))
+
         FrigateConfig.parse_yaml(new_config)
+
+    except ValidationError as e:
+        error_message = []
+
+        for error in e.errors():
+            error_path = error["loc"]
+            current = full_config
+            line_number = "Unknown"
+            last_line_number = "Unknown"
+
+            try:
+                for i, part in enumerate(error_path):
+                    key = int(part) if part.isdigit() else part
+
+                    if isinstance(current, ruamel.yaml.comments.CommentedMap):
+                        current = current[key]
+                    elif isinstance(current, list):
+                        current = current[key]
+
+                    if hasattr(current, "lc"):
+                        last_line_number = current.lc.line
+
+                    if i == len(error_path) - 1:
+                        if hasattr(current, "lc"):
+                            line_number = current.lc.line
+                        else:
+                            line_number = last_line_number
+
+            except Exception:
+                line_number = "Unable to determine"
+
+            error_message.append(
+                f"Line {line_number}: {' -> '.join(map(str, error_path))} - {error.get('msg', error.get('type', 'Unknown'))}"
+            )
+
+        return JSONResponse(
+            content=(
+                {
+                    "success": False,
+                    "message": "Your configuration is invalid.\nSee the official documentation at docs.frigate.video.\n\n"
+                    + "\n".join(error_message),
+                }
+            ),
+            status_code=400,
+        )
+
     except Exception:
         return JSONResponse(
             content=(
                 {
                     "success": False,
-                    "message": f"\nConfig Error:\n\n{escape(str(traceback.format_exc()))}",
+                    "message": f"\nYour configuration is invalid.\nSee the official documentation at docs.frigate.video.\n\n{escape(str(traceback.format_exc()))}",
                 }
             ),
             status_code=400,
@@ -198,13 +261,7 @@ def config_save(save_option: str, body: Any = Body(media_type="text/plain")):
 
     # Save the config to file
     try:
-        config_file = os.environ.get("CONFIG_FILE", "/config/config.yml")
-
-        # Check if we can use .yaml instead of .yml
-        config_file_yaml = config_file.replace(".yml", ".yaml")
-
-        if os.path.isfile(config_file_yaml):
-            config_file = config_file_yaml
+        config_file = find_config_file()
 
         with open(config_file, "w") as f:
             f.write(new_config)
@@ -253,13 +310,7 @@ def config_save(save_option: str, body: Any = Body(media_type="text/plain")):
 
 @router.put("/config/set")
 def config_set(request: Request, body: AppConfigSetBody):
-    config_file = os.environ.get("CONFIG_FILE", f"{CONFIG_DIR}/config.yml")
-
-    # Check if we can use .yaml instead of .yml
-    config_file_yaml = config_file.replace(".yml", ".yaml")
-
-    if os.path.isfile(config_file_yaml):
-        config_file = config_file_yaml
+    config_file = find_config_file()
 
     with open(config_file, "r") as f:
         old_raw_config = f.read()

frigate/api/auth.py

@@ -18,7 +18,7 @@ from joserfc import jwt
 from peewee import DoesNotExist
 from slowapi import Limiter
 
-from frigate.api.defs.app_body import (
+from frigate.api.defs.request.app_body import (
     AppPostLoginBody,
     AppPostUsersBody,
     AppPutPasswordBody,
@@ -85,7 +85,12 @@ def get_remote_addr(request: Request):
             return str(ip)
 
     # if there wasn't anything in the route, just return the default
-    return request.remote_addr or "127.0.0.1"
+    remote_addr = None
+
+    if hasattr(request, "remote_addr"):
+        remote_addr = request.remote_addr
+
+    return remote_addr or "127.0.0.1"
 
 
 def get_jwt_secret() -> str:
@@ -324,7 +329,7 @@ def login(request: Request, body: AppPostLoginBody):
     try:
         db_user: User = User.get_by_id(user)
     except DoesNotExist:
-        return JSONResponse(content={"message": "Login failed"}, status_code=400)
+        return JSONResponse(content={"message": "Login failed"}, status_code=401)
 
     password_hash = db_user.password_hash
     if verify_password(password, password_hash):
@@ -335,7 +340,7 @@ def login(request: Request, body: AppPostLoginBody):
             response, JWT_COOKIE_NAME, encoded_jwt, expiration, JWT_COOKIE_SECURE
         )
         return response
-    return JSONResponse(content={"message": "Login failed"}, status_code=400)
+    return JSONResponse(content={"message": "Login failed"}, status_code=401)
 
 
 @router.get("/users")

frigate/api/classification.py

@@ -0,0 +1,59 @@
+"""Object classification APIs."""
+
+import logging
+import os
+
+from fastapi import APIRouter, Request, UploadFile
+from fastapi.responses import JSONResponse
+from pathvalidate import sanitize_filename
+
+from frigate.api.defs.tags import Tags
+from frigate.const import FACE_DIR
+from frigate.embeddings import EmbeddingsContext
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=[Tags.events])
+
+
+@router.get("/faces")
+def get_faces():
+    face_dict: dict[str, list[str]] = {}
+
+    for name in os.listdir(FACE_DIR):
+        face_dict[name] = []
+        for file in os.listdir(os.path.join(FACE_DIR, name)):
+            face_dict[name].append(file)
+
+    return JSONResponse(status_code=200, content=face_dict)
+
+
+@router.post("/faces/{name}")
+async def register_face(request: Request, name: str, file: UploadFile):
+    context: EmbeddingsContext = request.app.embeddings
+    context.register_face(name, await file.read())
+    return JSONResponse(
+        status_code=200,
+        content={"success": True, "message": "Successfully registered face."},
+    )
+
+
+@router.post("/faces/{name}/delete")
+def deregister_faces(request: Request, name: str, body: dict = None):
+    json: dict[str, any] = body or {}
+    list_of_ids = json.get("ids", "")
+
+    if not list_of_ids or len(list_of_ids) == 0:
+        return JSONResponse(
+            content=({"success": False, "message": "Not a valid list of ids"}),
+            status_code=404,
+        )
+
+    context: EmbeddingsContext = request.app.embeddings
+    context.delete_face_ids(
+        name, map(lambda file: sanitize_filename(file), list_of_ids)
+    )
+    return JSONResponse(
+        content=({"success": True, "message": "Successfully deleted faces."}),
+        status_code=200,
+    )

frigate/api/defs/query/media_query_parameters.py

@@ -20,6 +20,7 @@ class MediaLatestFrameQueryParams(BaseModel):
     regions: Optional[int] = None
     quality: Optional[int] = 70
     height: Optional[int] = None
+    store: Optional[int] = None
 
 
 class MediaEventsSnapshotQueryParams(BaseModel):

frigate/api/defs/query/review_query_parameters.py

@@ -3,7 +3,7 @@ from typing import Union
 from pydantic import BaseModel
 from pydantic.json_schema import SkipJsonSchema
 
-from frigate.review.maintainer import SeverityEnum
+from frigate.review.types import SeverityEnum
 
 
 class ReviewQueryParams(BaseModel):

frigate/api/defs/request/events_body.py

@@ -8,6 +8,9 @@ class EventsSubLabelBody(BaseModel):
     subLabelScore: Optional[float] = Field(
         title="Score for sub label", default=None, gt=0.0, le=1.0
     )
+    camera: Optional[str] = Field(
+        title="Camera this object is detected on.", default=None
+    )
 
 
 class EventsDescriptionBody(BaseModel):

frigate/api/defs/response/event_response.py

@@ -0,0 +1,42 @@
+from typing import Any, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+
+class EventResponse(BaseModel):
+    id: str
+    label: str
+    sub_label: Optional[str]
+    camera: str
+    start_time: float
+    end_time: Optional[float]
+    false_positive: Optional[bool]
+    zones: list[str]
+    thumbnail: str
+    has_clip: bool
+    has_snapshot: bool
+    retain_indefinitely: bool
+    plus_id: Optional[str]
+    model_hash: Optional[str]
+    detector_type: Optional[str]
+    model_type: Optional[str]
+    data: dict[str, Any]
+
+    model_config = ConfigDict(protected_namespaces=())
+
+
+class EventCreateResponse(BaseModel):
+    success: bool
+    message: str
+    event_id: str
+
+
+class EventMultiDeleteResponse(BaseModel):
+    success: bool
+    deleted_events: list[str]
+    not_found_events: list[str]
+
+
+class EventUploadPlusResponse(BaseModel):
+    success: bool
+    plus_id: str

frigate/api/defs/response/review_response.py

@@ -3,7 +3,7 @@ from typing import Dict
 
 from pydantic import BaseModel, Json
 
-from frigate.review.maintainer import SeverityEnum
+from frigate.review.types import SeverityEnum
 
 
 class ReviewSegmentResponse(BaseModel):

frigate/api/defs/tags.py

@@ -10,4 +10,5 @@ class Tags(Enum):
     review = "Review"
     export = "Export"
     events = "Events"
+    classification = "classification"
     auth = "Auth"

frigate/api/event.py

@@ -14,7 +14,16 @@ from fastapi.responses import JSONResponse
 from peewee import JOIN, DoesNotExist, fn, operator
 from playhouse.shortcuts import model_to_dict
 
-from frigate.api.defs.events_body import (
+from frigate.api.defs.query.events_query_parameters import (
+    DEFAULT_TIME_RANGE,
+    EventsQueryParams,
+    EventsSearchQueryParams,
+    EventsSummaryQueryParams,
+)
+from frigate.api.defs.query.regenerate_query_parameters import (
+    RegenerateQueryParameters,
+)
+from frigate.api.defs.request.events_body import (
     EventsCreateBody,
     EventsDeleteBody,
     EventsDescriptionBody,
@@ -22,19 +31,15 @@ from frigate.api.defs.events_body import (
     EventsSubLabelBody,
     SubmitPlusBody,
 )
-from frigate.api.defs.events_query_parameters import (
-    DEFAULT_TIME_RANGE,
-    EventsQueryParams,
-    EventsSearchQueryParams,
-    EventsSummaryQueryParams,
-)
-from frigate.api.defs.regenerate_query_parameters import (
-    RegenerateQueryParameters,
+from frigate.api.defs.response.event_response import (
+    EventCreateResponse,
+    EventMultiDeleteResponse,
+    EventResponse,
+    EventUploadPlusResponse,
 )
+from frigate.api.defs.response.generic_response import GenericResponse
 from frigate.api.defs.tags import Tags
-from frigate.const import (
-    CLIPS_DIR,
-)
+from frigate.const import CLIPS_DIR
 from frigate.embeddings import EmbeddingsContext
 from frigate.events.external import ExternalEventProcessor
 from frigate.models import Event, ReviewSegment, Timeline
@@ -46,7 +51,7 @@ logger = logging.getLogger(__name__)
 router = APIRouter(tags=[Tags.events])
 
 
-@router.get("/events")
+@router.get("/events", response_model=list[EventResponse])
 def events(params: EventsQueryParams = Depends()):
     camera = params.camera
     cameras = params.cameras
@@ -248,6 +253,8 @@ def events(params: EventsQueryParams = Depends()):
             order_by = Event.start_time.asc()
         elif sort == "date_desc":
             order_by = Event.start_time.desc()
+        else:
+            order_by = Event.start_time.desc()
     else:
         order_by = Event.start_time.desc()
 
@@ -263,7 +270,7 @@ def events(params: EventsQueryParams = Depends()):
     return JSONResponse(content=list(events))
 
 
-@router.get("/events/explore")
+@router.get("/events/explore", response_model=list[EventResponse])
 def events_explore(limit: int = 10):
     # get distinct labels for all events
     distinct_labels = Event.select(Event.label).distinct().order_by(Event.label)
@@ -308,7 +315,8 @@ def events_explore(limit: int = 10):
                 "data": {
                     k: v
                     for k, v in event.data.items()
-                    if k in ["type", "score", "top_score", "description"]
+                    if k
+                    in ["type", "score", "top_score", "description", "sub_label_score"]
                 },
                 "event_count": label_counts[event.label],
             }
@@ -324,7 +332,7 @@ def events_explore(limit: int = 10):
     return JSONResponse(content=processed_events)
 
 
-@router.get("/event_ids")
+@router.get("/event_ids", response_model=list[EventResponse])
 def event_ids(ids: str):
     ids = ids.split(",")
 
@@ -582,19 +590,17 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends())
 
         processed_events.append(processed_event)
 
-    # Sort by search distance if search_results are available, otherwise by start_time as default
-    if search_results:
+    if (sort is None or sort == "relevance") and search_results:
         processed_events.sort(key=lambda x: x.get("search_distance", float("inf")))
+    elif min_score is not None and max_score is not None and sort == "score_asc":
+        processed_events.sort(key=lambda x: x["score"])
+    elif min_score is not None and max_score is not None and sort == "score_desc":
+        processed_events.sort(key=lambda x: x["score"], reverse=True)
+    elif sort == "date_asc":
+        processed_events.sort(key=lambda x: x["start_time"])
     else:
-        if sort == "score_asc":
-            processed_events.sort(key=lambda x: x["score"])
-        elif sort == "score_desc":
-            processed_events.sort(key=lambda x: x["score"], reverse=True)
-        elif sort == "date_asc":
-            processed_events.sort(key=lambda x: x["start_time"])
-        else:
-            # "date_desc" default
-            processed_events.sort(key=lambda x: x["start_time"], reverse=True)
+        # "date_desc" default
+        processed_events.sort(key=lambda x: x["start_time"], reverse=True)
 
     # Limit the number of events returned
     processed_events = processed_events[:limit]
@@ -647,7 +653,7 @@ def events_summary(params: EventsSummaryQueryParams = Depends()):
     return JSONResponse(content=[e for e in groups.dicts()])
 
 
-@router.get("/events/{event_id}")
+@router.get("/events/{event_id}", response_model=EventResponse)
 def event(event_id: str):
     try:
         return model_to_dict(Event.get(Event.id == event_id))
@@ -655,7 +661,7 @@ def event(event_id: str):
         return JSONResponse(content="Event not found", status_code=404)
 
 
-@router.post("/events/{event_id}/retain")
+@router.post("/events/{event_id}/retain", response_model=GenericResponse)
 def set_retain(event_id: str):
     try:
         event = Event.get(Event.id == event_id)
@@ -674,7 +680,7 @@ def set_retain(event_id: str):
     )
 
 
-@router.post("/events/{event_id}/plus")
+@router.post("/events/{event_id}/plus", response_model=EventUploadPlusResponse)
 def send_to_plus(request: Request, event_id: str, body: SubmitPlusBody = None):
     if not request.app.frigate_config.plus_api.is_active():
         message = "PLUS_API_KEY environment variable is not set"
@@ -786,7 +792,7 @@ def send_to_plus(request: Request, event_id: str, body: SubmitPlusBody = None):
     )
 
 
-@router.put("/events/{event_id}/false_positive")
+@router.put("/events/{event_id}/false_positive", response_model=EventUploadPlusResponse)
 def false_positive(request: Request, event_id: str):
     if not request.app.frigate_config.plus_api.is_active():
         message = "PLUS_API_KEY environment variable is not set"
@@ -875,7 +881,7 @@ def false_positive(request: Request, event_id: str):
     )
 
 
-@router.delete("/events/{event_id}/retain")
+@router.delete("/events/{event_id}/retain", response_model=GenericResponse)
 def delete_retain(event_id: str):
     try:
         event = Event.get(Event.id == event_id)
@@ -894,7 +900,7 @@ def delete_retain(event_id: str):
     )
 
 
-@router.post("/events/{event_id}/sub_label")
+@router.post("/events/{event_id}/sub_label", response_model=GenericResponse)
 def set_sub_label(
     request: Request,
     event_id: str,
@@ -903,38 +909,59 @@ def set_sub_label(
     try:
         event: Event = Event.get(Event.id == event_id)
     except DoesNotExist:
+        if not body.camera:
+            return JSONResponse(
+                content=(
+                    {
+                        "success": False,
+                        "message": "Event "
+                        + event_id
+                        + " not found and camera is not provided.",
+                    }
+                ),
+                status_code=404,
+            )
+
+        event = None
+
+    if request.app.detected_frames_processor:
+        tracked_obj: TrackedObject = (
+            request.app.detected_frames_processor.camera_states[
+                event.camera if event else body.camera
+            ].tracked_objects.get(event_id)
+        )
+    else:
+        tracked_obj = None
+
+    if not event and not tracked_obj:
         return JSONResponse(
-            content=({"success": False, "message": "Event " + event_id + " not found"}),
+            content=(
+                {"success": False, "message": "Event " + event_id + " not found."}
+            ),
             status_code=404,
         )
 
     new_sub_label = body.subLabel
     new_score = body.subLabelScore
 
-    if not event.end_time:
-        # update tracked object
-        tracked_obj: TrackedObject = (
-            request.app.detected_frames_processor.camera_states[
-                event.camera
-            ].tracked_objects.get(event.id)
-        )
-
-        if tracked_obj:
-            tracked_obj.obj_data["sub_label"] = (new_sub_label, new_score)
+    if tracked_obj:
+        tracked_obj.obj_data["sub_label"] = (new_sub_label, new_score)
 
         # update timeline items
         Timeline.update(
             data=Timeline.data.update({"sub_label": (new_sub_label, new_score)})
         ).where(Timeline.source_id == event_id).execute()
 
-    event.sub_label = new_sub_label
+    if event:
+        event.sub_label = new_sub_label
 
-    if new_score:
-        data = event.data
-        data["sub_label_score"] = new_score
-        event.data = data
+        if new_score:
+            data = event.data
+            data["sub_label_score"] = new_score
+            event.data = data
+
+        event.save()
 
-    event.save()
     return JSONResponse(
         content=(
             {
@@ -946,7 +973,7 @@ def set_sub_label(
     )
 
 
-@router.post("/events/{event_id}/description")
+@router.post("/events/{event_id}/description", response_model=GenericResponse)
 def set_description(
     request: Request,
     event_id: str,
@@ -993,7 +1020,7 @@ def set_description(
     )
 
 
-@router.put("/events/{event_id}/description/regenerate")
+@router.put("/events/{event_id}/description/regenerate", response_model=GenericResponse)
 def regenerate_description(
     request: Request, event_id: str, params: RegenerateQueryParameters = Depends()
 ):
@@ -1064,14 +1091,14 @@ def delete_single_event(event_id: str, request: Request) -> dict:
     return {"success": True, "message": f"Event {event_id} deleted"}
 
 
-@router.delete("/events/{event_id}")
+@router.delete("/events/{event_id}", response_model=GenericResponse)
 def delete_event(request: Request, event_id: str):
     result = delete_single_event(event_id, request)
     status_code = 200 if result["success"] else 404
     return JSONResponse(content=result, status_code=status_code)
 
 
-@router.delete("/events/")
+@router.delete("/events/", response_model=EventMultiDeleteResponse)
 def delete_events(request: Request, body: EventsDeleteBody):
     if not body.event_ids:
         return JSONResponse(
@@ -1097,7 +1124,7 @@ def delete_events(request: Request, body: EventsDeleteBody):
     return JSONResponse(content=response, status_code=200)
 
 
-@router.post("/events/{camera_name}/{label}/create")
+@router.post("/events/{camera_name}/{label}/create", response_model=EventCreateResponse)
 def create_event(
     request: Request,
     camera_name: str,
@@ -1153,7 +1180,7 @@ def create_event(
     )
 
 
-@router.put("/events/{event_id}/end")
+@router.put("/events/{event_id}/end", response_model=GenericResponse)
 def end_event(request: Request, event_id: str, body: EventsEndBody):
     try:
         end_time = body.end_time or datetime.datetime.now().timestamp()

frigate/api/export.py

@@ -9,6 +9,7 @@ import psutil
 from fastapi import APIRouter, Request
 from fastapi.responses import JSONResponse
 from peewee import DoesNotExist
+from playhouse.shortcuts import model_to_dict
 
 from frigate.api.defs.request.export_recordings_body import ExportRecordingsBody
 from frigate.api.defs.tags import Tags
@@ -207,3 +208,14 @@ def export_delete(event_id: str):
         ),
         status_code=200,
     )
+
+
+@router.get("/exports/{export_id}")
+def get_export(export_id: str):
+    try:
+        return JSONResponse(content=model_to_dict(Export.get(Export.id == export_id)))
+    except DoesNotExist:
+        return JSONResponse(
+            content={"success": False, "message": "Export not found"},
+            status_code=404,
+        )

frigate/api/fastapi_app.py

@@ -11,7 +11,16 @@ from starlette_context import middleware, plugins
 from starlette_context.plugins import Plugin
 
 from frigate.api import app as main_app
-from frigate.api import auth, event, export, media, notification, preview, review
+from frigate.api import (
+    auth,
+    classification,
+    event,
+    export,
+    media,
+    notification,
+    preview,
+    review,
+)
 from frigate.api.auth import get_jwt_secret, limiter
 from frigate.comms.event_metadata_updater import (
     EventMetadataPublisher,
@@ -87,7 +96,11 @@ def create_fastapi_app(
         logger.info("FastAPI started")
 
     # Rate limiter (used for login endpoint)
-    auth.rateLimiter.set_limit(frigate_config.auth.failed_login_rate_limit or "")
+    if frigate_config.auth.failed_login_rate_limit is None:
+        limiter.enabled = False
+    else:
+        auth.rateLimiter.set_limit(frigate_config.auth.failed_login_rate_limit)
+
     app.state.limiter = limiter
     app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
     app.add_middleware(SlowAPIMiddleware)
@@ -95,6 +108,7 @@ def create_fastapi_app(
     # Routes
     # Order of include_router matters: https://fastapi.tiangolo.com/tutorial/path-params/#order-matters
     app.include_router(auth.router)
+    app.include_router(classification.router)
     app.include_router(review.router)
     app.include_router(main_app.router)
     app.include_router(preview.router)

frigate/api/media.py

@@ -20,7 +20,7 @@ from pathvalidate import sanitize_filename
 from peewee import DoesNotExist, fn
 from tzlocal import get_localzone_name
 
-from frigate.api.defs.media_query_parameters import (
+from frigate.api.defs.query.media_query_parameters import (
     Extension,
     MediaEventsSnapshotQueryParams,
     MediaLatestFrameQueryParams,
@@ -179,7 +179,12 @@ def latest_frame(
         return Response(
             content=img.tobytes(),
             media_type=f"image/{extension}",
-            headers={"Content-Type": f"image/{extension}", "Cache-Control": "no-store"},
+            headers={
+                "Content-Type": f"image/{extension}",
+                "Cache-Control": "no-store"
+                if not params.store
+                else "private, max-age=60",
+            },
         )
     elif camera_name == "birdseye" and request.app.frigate_config.birdseye.restream:
         frame = cv2.cvtColor(
@@ -198,7 +203,12 @@ def latest_frame(
         return Response(
             content=img.tobytes(),
             media_type=f"image/{extension}",
-            headers={"Content-Type": f"image/{extension}", "Cache-Control": "no-store"},
+            headers={
+                "Content-Type": f"image/{extension}",
+                "Cache-Control": "no-store"
+                if not params.store
+                else "private, max-age=60",
+            },
         )
     else:
         return JSONResponse(

frigate/api/review.py

@@ -12,20 +12,21 @@ from fastapi.responses import JSONResponse
 from peewee import Case, DoesNotExist, fn, operator
 from playhouse.shortcuts import model_to_dict
 
-from frigate.api.defs.generic_response import GenericResponse
-from frigate.api.defs.review_body import ReviewModifyMultipleBody
-from frigate.api.defs.review_query_parameters import (
+from frigate.api.defs.query.review_query_parameters import (
     ReviewActivityMotionQueryParams,
     ReviewQueryParams,
     ReviewSummaryQueryParams,
 )
-from frigate.api.defs.review_responses import (
+from frigate.api.defs.request.review_body import ReviewModifyMultipleBody
+from frigate.api.defs.response.generic_response import GenericResponse
+from frigate.api.defs.response.review_response import (
     ReviewActivityMotionResponse,
     ReviewSegmentResponse,
     ReviewSummaryResponse,
 )
 from frigate.api.defs.tags import Tags
 from frigate.models import Recordings, ReviewSegment
+from frigate.review.types import SeverityEnum
 from frigate.util.builtin import get_tz_modifiers
 
 logger = logging.getLogger(__name__)
@@ -161,7 +162,7 @@ def review_summary(params: ReviewSummaryQueryParams = Depends()):
                     None,
                     [
                         (
-                            (ReviewSegment.severity == "alert"),
+                            (ReviewSegment.severity == SeverityEnum.alert),
                             ReviewSegment.has_been_reviewed,
                         )
                     ],
@@ -173,7 +174,7 @@ def review_summary(params: ReviewSummaryQueryParams = Depends()):
                     None,
                     [
                         (
-                            (ReviewSegment.severity == "detection"),
+                            (ReviewSegment.severity == SeverityEnum.detection),
                             ReviewSegment.has_been_reviewed,
                         )
                     ],
@@ -185,7 +186,7 @@ def review_summary(params: ReviewSummaryQueryParams = Depends()):
                     None,
                     [
                         (
-                            (ReviewSegment.severity == "alert"),
+                            (ReviewSegment.severity == SeverityEnum.alert),
                             1,
                         )
                     ],
@@ -197,7 +198,7 @@ def review_summary(params: ReviewSummaryQueryParams = Depends()):
                     None,
                     [
                         (
-                            (ReviewSegment.severity == "detection"),
+                            (ReviewSegment.severity == SeverityEnum.detection),
                             1,
                         )
                     ],
@@ -230,6 +231,7 @@ def review_summary(params: ReviewSummaryQueryParams = Depends()):
         label_clause = reduce(operator.or_, label_clauses)
         clauses.append((label_clause))
 
+    day_in_seconds = 60 * 60 * 24
     last_month = (
         ReviewSegment.select(
             fn.strftime(
@@ -246,7 +248,7 @@ def review_summary(params: ReviewSummaryQueryParams = Depends()):
                     None,
                     [
                         (
-                            (ReviewSegment.severity == "alert"),
+                            (ReviewSegment.severity == SeverityEnum.alert),
                             ReviewSegment.has_been_reviewed,
                         )
                     ],
@@ -258,7 +260,7 @@ def review_summary(params: ReviewSummaryQueryParams = Depends()):
                     None,
                     [
                         (
-                            (ReviewSegment.severity == "detection"),
+                            (ReviewSegment.severity == SeverityEnum.detection),
                             ReviewSegment.has_been_reviewed,
                         )
                     ],
@@ -270,7 +272,7 @@ def review_summary(params: ReviewSummaryQueryParams = Depends()):
                     None,
                     [
                         (
-                            (ReviewSegment.severity == "alert"),
+                            (ReviewSegment.severity == SeverityEnum.alert),
                             1,
                         )
                     ],
@@ -282,7 +284,7 @@ def review_summary(params: ReviewSummaryQueryParams = Depends()):
                     None,
                     [
                         (
-                            (ReviewSegment.severity == "detection"),
+                            (ReviewSegment.severity == SeverityEnum.detection),
                             1,
                         )
                     ],
@@ -292,7 +294,7 @@ def review_summary(params: ReviewSummaryQueryParams = Depends()):
         )
         .where(reduce(operator.and_, clauses))
         .group_by(
-            (ReviewSegment.start_time + seconds_offset).cast("int") / (3600 * 24),
+            (ReviewSegment.start_time + seconds_offset).cast("int") / day_in_seconds,
         )
         .order_by(ReviewSegment.start_time.desc())
     )
@@ -362,7 +364,7 @@ def delete_reviews(body: ReviewModifyMultipleBody):
     ReviewSegment.delete().where(ReviewSegment.id << list_of_ids).execute()
 
     return JSONResponse(
-        content=({"success": True, "message": "Delete reviews"}), status_code=200
+        content=({"success": True, "message": "Deleted review items."}), status_code=200
     )
 
 

frigate/app.py

@@ -437,7 +437,7 @@ class FrigateApp:
             # pre-create shms
             for i in range(shm_frame_count):
                 frame_size = config.frame_shape_yuv[0] * config.frame_shape_yuv[1]
-                self.frame_manager.create(f"{config.name}_{i}", frame_size)
+                self.frame_manager.create(f"{config.name}_frame{i}", frame_size)
 
             capture_process = util.Process(
                 target=capture_camera,

frigate/comms/embeddings_updater.py

@@ -12,6 +12,7 @@ class EmbeddingsRequestEnum(Enum):
     embed_description = "embed_description"
     embed_thumbnail = "embed_thumbnail"
     generate_search = "generate_search"
+    register_face = "register_face"
 
 
 class EmbeddingsResponder:
@@ -22,7 +23,7 @@ class EmbeddingsResponder:
 
     def check_for_request(self, process: Callable) -> None:
         while True:  # load all messages that are queued
-            has_message, _, _ = zmq.select([self.socket], [], [], 0.1)
+            has_message, _, _ = zmq.select([self.socket], [], [], 0.01)
 
             if not has_message:
                 break

frigate/config/camera/genai.py

@@ -38,6 +38,10 @@ class GenAICameraConfig(BaseModel):
         default_factory=list,
         title="List of required zones to be entered in order to run generative AI.",
     )
+    debug_save_thumbnails: bool = Field(
+        default=False,
+        title="Save thumbnails sent to generative AI for debugging purposes.",
+    )
 
     @field_validator("required_zones", mode="before")
     @classmethod

frigate/config/camera/objects.py

@@ -1,6 +1,6 @@
 from typing import Any, Optional, Union
 
-from pydantic import Field, field_serializer
+from pydantic import Field, PrivateAttr, field_serializer
 
 from ..base import FrigateBaseModel
 
@@ -53,3 +53,20 @@ class ObjectConfig(FrigateBaseModel):
         default_factory=dict, title="Object filters."
     )
     mask: Union[str, list[str]] = Field(default="", title="Object mask.")
+    _all_objects: list[str] = PrivateAttr()
+
+    @property
+    def all_objects(self) -> list[str]:
+        return self._all_objects
+
+    def parse_all_objects(self, cameras):
+        if "_all_objects" in self:
+            return
+
+        # get list of unique enabled labels for tracking
+        enabled_labels = set(self.track)
+
+        for camera in cameras.values():
+            enabled_labels.update(camera.objects.track)
+
+        self._all_objects = list(enabled_labels)

frigate/config/camera/onvif.py

@@ -74,6 +74,7 @@ class OnvifConfig(FrigateBaseModel):
     port: int = Field(default=8000, title="Onvif Port")
     user: Optional[EnvString] = Field(default=None, title="Onvif Username")
     password: Optional[EnvString] = Field(default=None, title="Onvif Password")
+    tls_insecure: bool = Field(default=False, title="Onvif Disable TLS verification")
     autotracking: PtzAutotrackConfig = Field(
         default_factory=PtzAutotrackConfig,
         title="PTZ auto tracking config.",

frigate/config/camera/record.py

@@ -4,6 +4,7 @@ from typing import Optional
 from pydantic import Field
 
 from frigate.const import MAX_PRE_CAPTURE
+from frigate.review.types import SeverityEnum
 
 from ..base import FrigateBaseModel
 
@@ -101,3 +102,15 @@ class RecordConfig(FrigateBaseModel):
             self.alerts.pre_capture,
             self.detections.pre_capture,
         )
+
+    def get_review_pre_capture(self, severity: SeverityEnum) -> int:
+        if severity == SeverityEnum.alert:
+            return self.alerts.pre_capture
+        else:
+            return self.detections.pre_capture
+
+    def get_review_post_capture(self, severity: SeverityEnum) -> int:
+        if severity == SeverityEnum.alert:
+            return self.alerts.post_capture
+        else:
+            return self.detections.post_capture

frigate/config/config.py

@@ -29,6 +29,7 @@ from frigate.util.builtin import (
 )
 from frigate.util.config import (
     StreamInfoRetriever,
+    find_config_file,
     get_relative_coordinates,
     migrate_frigate_config,
 )
@@ -56,7 +57,11 @@ from .logger import LoggerConfig
 from .mqtt import MqttConfig
 from .notification import NotificationConfig
 from .proxy import ProxyConfig
-from .semantic_search import SemanticSearchConfig
+from .semantic_search import (
+    FaceRecognitionConfig,
+    LicensePlateRecognitionConfig,
+    SemanticSearchConfig,
+)
 from .telemetry import TelemetryConfig
 from .tls import TlsConfig
 from .ui import UIConfig
@@ -67,7 +72,6 @@ logger = logging.getLogger(__name__)
 
 yaml = YAML()
 
-DEFAULT_CONFIG_FILE = "/config/config.yml"
 DEFAULT_CONFIG = """
 mqtt:
   enabled: False
@@ -159,6 +163,16 @@ class RestreamConfig(BaseModel):
     model_config = ConfigDict(extra="allow")
 
 
+def verify_semantic_search_dependent_configs(config: FrigateConfig) -> None:
+    """Verify that semantic search is enabled if required features are enabled."""
+    if not config.semantic_search.enabled:
+        if config.genai.enabled:
+            raise ValueError("Genai requires semantic search to be enabled.")
+
+        if config.face_recognition.enabled:
+            raise ValueError("Face recognition requires semantic to be enabled.")
+
+
 def verify_config_roles(camera_config: CameraConfig) -> None:
     """Verify that roles are setup in the config correctly."""
     assigned_roles = list(
@@ -320,6 +334,13 @@ class FrigateConfig(FrigateBaseModel):
     semantic_search: SemanticSearchConfig = Field(
         default_factory=SemanticSearchConfig, title="Semantic search configuration."
     )
+    face_recognition: FaceRecognitionConfig = Field(
+        default_factory=FaceRecognitionConfig, title="Face recognition config."
+    )
+    lpr: LicensePlateRecognitionConfig = Field(
+        default_factory=LicensePlateRecognitionConfig,
+        title="License Plate recognition config.",
+    )
     ui: UIConfig = Field(default_factory=UIConfig, title="UI configuration.")
 
     # Detector config
@@ -578,13 +599,8 @@ class FrigateConfig(FrigateBaseModel):
             verify_autotrack_zones(camera_config)
             verify_motion_and_detect(camera_config)
 
-        # get list of unique enabled labels for tracking
-        enabled_labels = set(self.objects.track)
-
-        for camera in self.cameras.values():
-            enabled_labels.update(camera.objects.track)
-
-        self.model.create_colormap(sorted(enabled_labels))
+        self.objects.parse_all_objects(self.cameras)
+        self.model.create_colormap(sorted(self.objects.all_objects))
         self.model.check_and_load_plus_model(self.plus_api)
 
         for key, detector in self.detectors.items():
@@ -625,6 +641,7 @@ class FrigateConfig(FrigateBaseModel):
             detector_config.model.compute_model_hash()
             self.detectors[key] = detector_config
 
+        verify_semantic_search_dependent_configs(self)
         return self
 
     @field_validator("cameras")
@@ -638,16 +655,13 @@ class FrigateConfig(FrigateBaseModel):
 
     @classmethod
     def load(cls, **kwargs):
-        config_path = os.environ.get("CONFIG_FILE", DEFAULT_CONFIG_FILE)
-
-        if not os.path.isfile(config_path):
-            config_path = config_path.replace("yml", "yaml")
+        config_path = find_config_file()
 
         # No configuration file found, create one.
         new_config = False
         if not os.path.isfile(config_path):
             logger.info("No config file found, saving default config")
-            config_path = DEFAULT_CONFIG_FILE
+            config_path = config_path
             new_config = True
         else:
             # Check if the config file needs to be migrated.

frigate/config/semantic_search.py

@@ -1,10 +1,14 @@
-from typing import Optional
+from typing import Dict, List, Optional
 
 from pydantic import Field
 
 from .base import FrigateBaseModel
 
-__all__ = ["SemanticSearchConfig"]
+__all__ = [
+    "FaceRecognitionConfig",
+    "SemanticSearchConfig",
+    "LicensePlateRecognitionConfig",
+]
 
 
 class SemanticSearchConfig(FrigateBaseModel):
@@ -15,3 +19,40 @@ class SemanticSearchConfig(FrigateBaseModel):
     model_size: str = Field(
         default="small", title="The size of the embeddings model used."
     )
+
+
+class FaceRecognitionConfig(FrigateBaseModel):
+    enabled: bool = Field(default=False, title="Enable face recognition.")
+    min_score: float = Field(
+        title="Minimum face distance score required to save the attempt.",
+        default=0.8,
+        gt=0.0,
+        le=1.0,
+    )
+    threshold: float = Field(
+        default=0.9,
+        title="Minimum face distance score required to be considered a match.",
+        gt=0.0,
+        le=1.0,
+    )
+    min_area: int = Field(
+        default=500, title="Min area of face box to consider running face recognition."
+    )
+    save_attempts: bool = Field(
+        default=True, title="Save images of face detections for training."
+    )
+
+
+class LicensePlateRecognitionConfig(FrigateBaseModel):
+    enabled: bool = Field(default=False, title="Enable license plate recognition.")
+    threshold: float = Field(
+        default=0.9,
+        title="License plate confidence score required to be added to the object as a sub label.",
+    )
+    min_area: int = Field(
+        default=500,
+        title="Min area of license plate to consider running license plate recognition.",
+    )
+    known_plates: Optional[Dict[str, List[str]]] = Field(
+        default={}, title="Known plates to track."
+    )

frigate/const.py

@@ -5,8 +5,9 @@ DEFAULT_DB_PATH = f"{CONFIG_DIR}/frigate.db"
 MODEL_CACHE_DIR = f"{CONFIG_DIR}/model_cache"
 BASE_DIR = "/media/frigate"
 CLIPS_DIR = f"{BASE_DIR}/clips"
-RECORD_DIR = f"{BASE_DIR}/recordings"
 EXPORT_DIR = f"{BASE_DIR}/exports"
+FACE_DIR = f"{CLIPS_DIR}/faces"
+RECORD_DIR = f"{BASE_DIR}/recordings"
 BIRDSEYE_PIPE = "/tmp/cache/birdseye"
 CACHE_DIR = "/tmp/cache"
 FRIGATE_LOCALHOST = "http://127.0.0.1:5000"

frigate/detectors/plugins/deepstack.py

@@ -32,6 +32,7 @@ class DeepStack(DetectionApi):
         self.api_timeout = detector_config.api_timeout
         self.api_key = detector_config.api_key
         self.labels = detector_config.model.merged_labelmap
+        self.session = requests.Session()
 
     def get_label_index(self, label_value):
         if label_value.lower() == "truck":
@@ -51,7 +52,7 @@ class DeepStack(DetectionApi):
         data = {"api_key": self.api_key}
 
         try:
-            response = requests.post(
+            response = self.session.post(
                 self.api_url,
                 data=data,
                 files={"image": image_bytes},

frigate/detectors/plugins/rknn.py

@@ -136,17 +136,17 @@ class Rknn(DetectionApi):
     def check_config(self, config):
         if (config.model.width != 320) or (config.model.height != 320):
             raise Exception(
-                "Make sure to set the model width and height to 320 in your config.yml."
+                "Make sure to set the model width and height to 320 in your config."
             )
 
         if config.model.input_pixel_format != "bgr":
             raise Exception(
-                'Make sure to set the model input_pixel_format to "bgr" in your config.yml.'
+                'Make sure to set the model input_pixel_format to "bgr" in your config.'
             )
 
         if config.model.input_tensor != "nhwc":
             raise Exception(
-                'Make sure to set the model input_tensor to "nhwc" in your config.yml.'
+                'Make sure to set the model input_tensor to "nhwc" in your config.'
             )
 
     def detect_raw(self, tensor_input):

frigate/embeddings/__init__.py

@@ -1,5 +1,6 @@
 """SQLite-vec embeddings database."""
 
+import base64
 import json
 import logging
 import multiprocessing as mp
@@ -13,7 +14,7 @@ from setproctitle import setproctitle
 
 from frigate.comms.embeddings_updater import EmbeddingsRequestEnum, EmbeddingsRequestor
 from frigate.config import FrigateConfig
-from frigate.const import CONFIG_DIR
+from frigate.const import CONFIG_DIR, FACE_DIR
 from frigate.db.sqlitevecq import SqliteVecQueueDatabase
 from frigate.models import Event
 from frigate.util.builtin import serialize
@@ -189,6 +190,33 @@ class EmbeddingsContext:
 
         return results
 
+    def register_face(self, face_name: str, image_data: bytes) -> None:
+        self.requestor.send_data(
+            EmbeddingsRequestEnum.register_face.value,
+            {
+                "face_name": face_name,
+                "image": base64.b64encode(image_data).decode("ASCII"),
+            },
+        )
+
+    def get_face_ids(self, name: str) -> list[str]:
+        sql_query = f"""
+            SELECT
+                id
+            FROM vec_descriptions
+            WHERE id LIKE '%{name}%'
+        """
+
+        return self.db.execute_sql(sql_query).fetchall()
+
+    def delete_face_ids(self, face: str, ids: list[str]) -> None:
+        folder = os.path.join(FACE_DIR, face)
+        for id in ids:
+            file_path = os.path.join(folder, id)
+
+            if os.path.isfile(file_path):
+                os.unlink(file_path)
+
     def update_description(self, event_id: str, description: str) -> None:
         self.requestor.send_data(
             EmbeddingsRequestEnum.embed_description.value,

frigate/embeddings/embeddings.py

@@ -9,7 +9,7 @@ from numpy import ndarray
 from playhouse.shortcuts import model_to_dict
 
 from frigate.comms.inter_process import InterProcessRequestor
-from frigate.config.semantic_search import SemanticSearchConfig
+from frigate.config import FrigateConfig
 from frigate.const import (
     CONFIG_DIR,
     UPDATE_EMBEDDINGS_REINDEX_PROGRESS,
@@ -59,9 +59,7 @@ def get_metadata(event: Event) -> dict:
 class Embeddings:
     """SQLite-vec embeddings database."""
 
-    def __init__(
-        self, config: SemanticSearchConfig, db: SqliteVecQueueDatabase
-    ) -> None:
+    def __init__(self, config: FrigateConfig, db: SqliteVecQueueDatabase) -> None:
         self.config = config
         self.db = db
         self.requestor = InterProcessRequestor()
@@ -73,9 +71,13 @@ class Embeddings:
             "jinaai/jina-clip-v1-text_model_fp16.onnx",
             "jinaai/jina-clip-v1-tokenizer",
             "jinaai/jina-clip-v1-vision_model_fp16.onnx"
-            if config.model_size == "large"
+            if config.semantic_search.model_size == "large"
             else "jinaai/jina-clip-v1-vision_model_quantized.onnx",
             "jinaai/jina-clip-v1-preprocessor_config.json",
+            "facenet-facenet.onnx",
+            "paddleocr-onnx-detection.onnx",
+            "paddleocr-onnx-classification.onnx",
+            "paddleocr-onnx-recognition.onnx",
         ]
 
         for model in models:
@@ -94,7 +96,7 @@ class Embeddings:
             download_urls={
                 "text_model_fp16.onnx": "https://huggingface.co/jinaai/jina-clip-v1/resolve/main/onnx/text_model_fp16.onnx",
             },
-            model_size=config.model_size,
+            model_size=config.semantic_search.model_size,
             model_type=ModelTypeEnum.text,
             requestor=self.requestor,
             device="CPU",
@@ -102,7 +104,7 @@ class Embeddings:
 
         model_file = (
             "vision_model_fp16.onnx"
-            if self.config.model_size == "large"
+            if self.config.semantic_search.model_size == "large"
             else "vision_model_quantized.onnx"
         )
 
@@ -115,12 +117,66 @@ class Embeddings:
             model_name="jinaai/jina-clip-v1",
             model_file=model_file,
             download_urls=download_urls,
-            model_size=config.model_size,
+            model_size=config.semantic_search.model_size,
             model_type=ModelTypeEnum.vision,
             requestor=self.requestor,
-            device="GPU" if config.model_size == "large" else "CPU",
+            device="GPU" if config.semantic_search.model_size == "large" else "CPU",
         )
 
+        if self.config.face_recognition.enabled:
+            self.face_embedding = GenericONNXEmbedding(
+                model_name="facedet",
+                model_file="facedet.onnx",
+                download_urls={
+                    "facedet.onnx": "https://github.com/NickM-27/facenet-onnx/releases/download/v1.0/facedet.onnx",
+                    "landmarkdet.yaml": "https://github.com/NickM-27/facenet-onnx/releases/download/v1.0/landmarkdet.yaml",
+                },
+                model_size="small",
+                model_type=ModelTypeEnum.face,
+                requestor=self.requestor,
+            )
+
+        self.lpr_detection_model = None
+        self.lpr_classification_model = None
+        self.lpr_recognition_model = None
+
+        if self.config.lpr.enabled:
+            self.lpr_detection_model = GenericONNXEmbedding(
+                model_name="paddleocr-onnx",
+                model_file="detection.onnx",
+                download_urls={
+                    "detection.onnx": "https://github.com/hawkeye217/paddleocr-onnx/raw/refs/heads/master/models/detection.onnx"
+                },
+                model_size="large",
+                model_type=ModelTypeEnum.lpr_detect,
+                requestor=self.requestor,
+                device="CPU",
+            )
+
+            self.lpr_classification_model = GenericONNXEmbedding(
+                model_name="paddleocr-onnx",
+                model_file="classification.onnx",
+                download_urls={
+                    "classification.onnx": "https://github.com/hawkeye217/paddleocr-onnx/raw/refs/heads/master/models/classification.onnx"
+                },
+                model_size="large",
+                model_type=ModelTypeEnum.lpr_classify,
+                requestor=self.requestor,
+                device="CPU",
+            )
+
+            self.lpr_recognition_model = GenericONNXEmbedding(
+                model_name="paddleocr-onnx",
+                model_file="recognition.onnx",
+                download_urls={
+                    "recognition.onnx": "https://github.com/hawkeye217/paddleocr-onnx/raw/refs/heads/master/models/recognition.onnx"
+                },
+                model_size="large",
+                model_type=ModelTypeEnum.lpr_recognize,
+                requestor=self.requestor,
+                device="CPU",
+            )
+
     def embed_thumbnail(
         self, event_id: str, thumbnail: bytes, upsert: bool = True
     ) -> ndarray:

frigate/embeddings/functions/onnx.py

@@ -31,11 +31,16 @@ warnings.filterwarnings(
 disable_progress_bar()
 logger = logging.getLogger(__name__)
 
+FACE_EMBEDDING_SIZE = 160
+
 
 class ModelTypeEnum(str, Enum):
     face = "face"
     vision = "vision"
     text = "text"
+    lpr_detect = "lpr_detect"
+    lpr_classify = "lpr_classify"
+    lpr_recognize = "lpr_recognize"
 
 
 class GenericONNXEmbedding:
@@ -47,7 +52,7 @@ class GenericONNXEmbedding:
         model_file: str,
         download_urls: Dict[str, str],
         model_size: str,
-        model_type: str,
+        model_type: ModelTypeEnum,
         requestor: InterProcessRequestor,
         tokenizer_file: Optional[str] = None,
         device: str = "AUTO",
@@ -57,7 +62,7 @@ class GenericONNXEmbedding:
         self.tokenizer_file = tokenizer_file
         self.requestor = requestor
         self.download_urls = download_urls
-        self.model_type = model_type  # 'text' or 'vision'
+        self.model_type = model_type
         self.model_size = model_size
         self.device = device
         self.download_path = os.path.join(MODEL_CACHE_DIR, self.model_name)
@@ -87,12 +92,13 @@ class GenericONNXEmbedding:
                 files_names,
                 ModelStatusTypesEnum.downloaded,
             )
-            self._load_model_and_tokenizer()
+            self._load_model_and_utils()
             logger.debug(f"models are already downloaded for {self.model_name}")
 
     def _download_model(self, path: str):
         try:
             file_name = os.path.basename(path)
+
             if file_name in self.download_urls:
                 ModelDownloader.download_from_url(self.download_urls[file_name], path)
             elif (
@@ -101,6 +107,7 @@ class GenericONNXEmbedding:
             ):
                 if not os.path.exists(path + "/" + self.model_name):
                     logger.info(f"Downloading {self.model_name} tokenizer")
+
                 tokenizer = AutoTokenizer.from_pretrained(
                     self.model_name,
                     trust_remote_code=True,
@@ -125,14 +132,23 @@ class GenericONNXEmbedding:
                 },
             )
 
-    def _load_model_and_tokenizer(self):
+    def _load_model_and_utils(self):
         if self.runner is None:
             if self.downloader:
                 self.downloader.wait_for_download()
             if self.model_type == ModelTypeEnum.text:
                 self.tokenizer = self._load_tokenizer()
-            else:
+            elif self.model_type == ModelTypeEnum.vision:
                 self.feature_extractor = self._load_feature_extractor()
+            elif self.model_type == ModelTypeEnum.face:
+                self.feature_extractor = []
+            elif self.model_type == ModelTypeEnum.lpr_detect:
+                self.feature_extractor = []
+            elif self.model_type == ModelTypeEnum.lpr_classify:
+                self.feature_extractor = []
+            elif self.model_type == ModelTypeEnum.lpr_recognize:
+                self.feature_extractor = []
+
             self.runner = ONNXModelRunner(
                 os.path.join(self.download_path, self.model_file),
                 self.device,
@@ -172,23 +188,72 @@ class GenericONNXEmbedding:
                 self.feature_extractor(images=image, return_tensors="np")
                 for image in processed_images
             ]
+        elif self.model_type == ModelTypeEnum.face:
+            if isinstance(raw_inputs, list):
+                raise ValueError("Face embedding does not support batch inputs.")
+
+            pil = self._process_image(raw_inputs)
+
+            # handle images larger than input size
+            width, height = pil.size
+            if width != FACE_EMBEDDING_SIZE or height != FACE_EMBEDDING_SIZE:
+                if width > height:
+                    new_height = int(((height / width) * FACE_EMBEDDING_SIZE) // 4 * 4)
+                    pil = pil.resize((FACE_EMBEDDING_SIZE, new_height))
+                else:
+                    new_width = int(((width / height) * FACE_EMBEDDING_SIZE) // 4 * 4)
+                    pil = pil.resize((new_width, FACE_EMBEDDING_SIZE))
+
+            og = np.array(pil).astype(np.float32)
+
+            # Image must be FACE_EMBEDDING_SIZExFACE_EMBEDDING_SIZE
+            og_h, og_w, channels = og.shape
+            frame = np.full(
+                (FACE_EMBEDDING_SIZE, FACE_EMBEDDING_SIZE, channels),
+                (0, 0, 0),
+                dtype=np.float32,
+            )
+
+            # compute center offset
+            x_center = (FACE_EMBEDDING_SIZE - og_w) // 2
+            y_center = (FACE_EMBEDDING_SIZE - og_h) // 2
+
+            # copy img image into center of result image
+            frame[y_center : y_center + og_h, x_center : x_center + og_w] = og
+            frame = np.expand_dims(frame, axis=0)
+            return [{"input_2": frame}]
+        elif self.model_type == ModelTypeEnum.lpr_detect:
+            preprocessed = []
+            for x in raw_inputs:
+                preprocessed.append(x)
+            return [{"x": preprocessed[0]}]
+        elif self.model_type == ModelTypeEnum.lpr_classify:
+            processed = []
+            for img in raw_inputs:
+                processed.append({"x": img})
+            return processed
+        elif self.model_type == ModelTypeEnum.lpr_recognize:
+            processed = []
+            for img in raw_inputs:
+                processed.append({"x": img})
+            return processed
         else:
             raise ValueError(f"Unable to preprocess inputs for {self.model_type}")
 
-    def _process_image(self, image):
+    def _process_image(self, image, output: str = "RGB") -> Image.Image:
         if isinstance(image, str):
             if image.startswith("http"):
                 response = requests.get(image)
-                image = Image.open(BytesIO(response.content)).convert("RGB")
+                image = Image.open(BytesIO(response.content)).convert(output)
         elif isinstance(image, bytes):
-            image = Image.open(BytesIO(image)).convert("RGB")
+            image = Image.open(BytesIO(image)).convert(output)
 
         return image
 
     def __call__(
         self, inputs: Union[List[str], List[Image.Image], List[str]]
     ) -> List[np.ndarray]:
-        self._load_model_and_tokenizer()
+        self._load_model_and_utils()
         if self.runner is None or (
             self.tokenizer is None and self.feature_extractor is None
         ):

frigate/embeddings/lpr/lpr.py

@@ -0,0 +1,808 @@
+import logging
+import math
+from typing import List, Tuple
+
+import cv2
+import numpy as np
+from pyclipper import ET_CLOSEDPOLYGON, JT_ROUND, PyclipperOffset
+from shapely.geometry import Polygon
+
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config.semantic_search import LicensePlateRecognitionConfig
+from frigate.embeddings.embeddings import Embeddings
+
+logger = logging.getLogger(__name__)
+
+MIN_PLATE_LENGTH = 3
+
+
+class LicensePlateRecognition:
+    def __init__(
+        self,
+        config: LicensePlateRecognitionConfig,
+        requestor: InterProcessRequestor,
+        embeddings: Embeddings,
+    ):
+        self.lpr_config = config
+        self.requestor = requestor
+        self.embeddings = embeddings
+        self.detection_model = self.embeddings.lpr_detection_model
+        self.classification_model = self.embeddings.lpr_classification_model
+        self.recognition_model = self.embeddings.lpr_recognition_model
+        self.ctc_decoder = CTCDecoder()
+
+        self.batch_size = 6
+
+        # Detection specific parameters
+        self.min_size = 3
+        self.max_size = 960
+        self.box_thresh = 0.8
+        self.mask_thresh = 0.8
+
+        if self.lpr_config.enabled:
+            # all models need to be loaded to run LPR
+            self.detection_model._load_model_and_utils()
+            self.classification_model._load_model_and_utils()
+            self.recognition_model._load_model_and_utils()
+
+    def detect(self, image: np.ndarray) -> List[np.ndarray]:
+        """
+        Detect possible license plates in the input image by first resizing and normalizing it,
+        running a detection model, and filtering out low-probability regions.
+
+        Args:
+            image (np.ndarray): The input image in which license plates will be detected.
+
+        Returns:
+            List[np.ndarray]: A list of bounding box coordinates representing detected license plates.
+        """
+        h, w = image.shape[:2]
+
+        if sum([h, w]) < 64:
+            image = self.zero_pad(image)
+
+        resized_image = self.resize_image(image)
+        normalized_image = self.normalize_image(resized_image)
+
+        outputs = self.detection_model([normalized_image])[0]
+        outputs = outputs[0, :, :]
+
+        boxes, _ = self.boxes_from_bitmap(outputs, outputs > self.mask_thresh, w, h)
+        return self.filter_polygon(boxes, (h, w))
+
+    def classify(
+        self, images: List[np.ndarray]
+    ) -> Tuple[List[np.ndarray], List[Tuple[str, float]]]:
+        """
+        Classify the orientation or category of each detected license plate.
+
+        Args:
+            images (List[np.ndarray]): A list of images of detected license plates.
+
+        Returns:
+            Tuple[List[np.ndarray], List[Tuple[str, float]]]: A tuple of rotated/normalized plate images
+                                                            and classification results with confidence scores.
+        """
+        num_images = len(images)
+        indices = np.argsort([x.shape[1] / x.shape[0] for x in images])
+
+        for i in range(0, num_images, self.batch_size):
+            norm_images = []
+            for j in range(i, min(num_images, i + self.batch_size)):
+                norm_img = self._preprocess_classification_image(images[indices[j]])
+                norm_img = norm_img[np.newaxis, :]
+                norm_images.append(norm_img)
+
+        outputs = self.classification_model(norm_images)
+
+        return self._process_classification_output(images, outputs)
+
+    def recognize(
+        self, images: List[np.ndarray]
+    ) -> Tuple[List[str], List[List[float]]]:
+        """
+        Recognize the characters on the detected license plates using the recognition model.
+
+        Args:
+            images (List[np.ndarray]): A list of images of license plates to recognize.
+
+        Returns:
+            Tuple[List[str], List[List[float]]]: A tuple of recognized license plate texts and confidence scores.
+        """
+        input_shape = [3, 48, 320]
+        num_images = len(images)
+
+        # sort images by aspect ratio for processing
+        indices = np.argsort(np.array([x.shape[1] / x.shape[0] for x in images]))
+
+        for index in range(0, num_images, self.batch_size):
+            input_h, input_w = input_shape[1], input_shape[2]
+            max_wh_ratio = input_w / input_h
+            norm_images = []
+
+            # calculate the maximum aspect ratio in the current batch
+            for i in range(index, min(num_images, index + self.batch_size)):
+                h, w = images[indices[i]].shape[0:2]
+                max_wh_ratio = max(max_wh_ratio, w * 1.0 / h)
+
+            # preprocess the images based on the max aspect ratio
+            for i in range(index, min(num_images, index + self.batch_size)):
+                norm_image = self._preprocess_recognition_image(
+                    images[indices[i]], max_wh_ratio
+                )
+                norm_image = norm_image[np.newaxis, :]
+                norm_images.append(norm_image)
+
+        outputs = self.recognition_model(norm_images)
+        return self.ctc_decoder(outputs)
+
+    def process_license_plate(
+        self, image: np.ndarray
+    ) -> Tuple[List[str], List[float], List[int]]:
+        """
+        Complete pipeline for detecting, classifying, and recognizing license plates in the input image.
+
+        Args:
+            image (np.ndarray): The input image in which to detect, classify, and recognize license plates.
+
+        Returns:
+            Tuple[List[str], List[float], List[int]]: Detected license plate texts, confidence scores, and areas of the plates.
+        """
+        if (
+            self.detection_model.runner is None
+            or self.classification_model.runner is None
+            or self.recognition_model.runner is None
+        ):
+            # we might still be downloading the models
+            logger.debug("Model runners not loaded")
+            return [], [], []
+
+        plate_points = self.detect(image)
+        if len(plate_points) == 0:
+            return [], [], []
+
+        plate_points = self.sort_polygon(list(plate_points))
+        plate_images = [self._crop_license_plate(image, x) for x in plate_points]
+        rotated_images, _ = self.classify(plate_images)
+
+        # keep track of the index of each image for correct area calc later
+        sorted_indices = np.argsort([x.shape[1] / x.shape[0] for x in rotated_images])
+        reverse_mapping = {
+            idx: original_idx for original_idx, idx in enumerate(sorted_indices)
+        }
+
+        results, confidences = self.recognize(rotated_images)
+
+        if results:
+            license_plates = [""] * len(rotated_images)
+            average_confidences = [[0.0]] * len(rotated_images)
+            areas = [0] * len(rotated_images)
+
+            # map results back to original image order
+            for i, (plate, conf) in enumerate(zip(results, confidences)):
+                original_idx = reverse_mapping[i]
+
+                height, width = rotated_images[original_idx].shape[:2]
+                area = height * width
+
+                average_confidence = conf
+
+                # set to True to write each cropped image for debugging
+                if False:
+                    save_image = cv2.cvtColor(
+                        rotated_images[original_idx], cv2.COLOR_RGB2BGR
+                    )
+                    filename = f"/config/plate_{original_idx}_{plate}_{area}.jpg"
+                    cv2.imwrite(filename, save_image)
+
+                license_plates[original_idx] = plate
+                average_confidences[original_idx] = average_confidence
+                areas[original_idx] = area
+
+            # Filter out plates that have a length of less than 3 characters
+            # Sort by area, then by plate length, then by confidence all desc
+            sorted_data = sorted(
+                [
+                    (plate, conf, area)
+                    for plate, conf, area in zip(
+                        license_plates, average_confidences, areas
+                    )
+                    if len(plate) >= MIN_PLATE_LENGTH
+                ],
+                key=lambda x: (x[2], len(x[0]), x[1]),
+                reverse=True,
+            )
+
+            if sorted_data:
+                return map(list, zip(*sorted_data))
+
+        return [], [], []
+
+    def resize_image(self, image: np.ndarray) -> np.ndarray:
+        """
+        Resize the input image while maintaining the aspect ratio, ensuring dimensions are multiples of 32.
+
+        Args:
+            image (np.ndarray): The input image to resize.
+
+        Returns:
+            np.ndarray: The resized image.
+        """
+        h, w = image.shape[:2]
+        ratio = min(self.max_size / max(h, w), 1.0)
+        resize_h = max(int(round(int(h * ratio) / 32) * 32), 32)
+        resize_w = max(int(round(int(w * ratio) / 32) * 32), 32)
+        return cv2.resize(image, (resize_w, resize_h))
+
+    def normalize_image(self, image: np.ndarray) -> np.ndarray:
+        """
+        Normalize the input image by subtracting the mean and multiplying by the standard deviation.
+
+        Args:
+            image (np.ndarray): The input image to normalize.
+
+        Returns:
+            np.ndarray: The normalized image, transposed to match the model's expected input format.
+        """
+        mean = np.array([123.675, 116.28, 103.53]).reshape(1, -1).astype("float64")
+        std = 1 / np.array([58.395, 57.12, 57.375]).reshape(1, -1).astype("float64")
+
+        image = image.astype("float32")
+        cv2.subtract(image, mean, image)
+        cv2.multiply(image, std, image)
+        return image.transpose((2, 0, 1))[np.newaxis, ...]
+
+    def boxes_from_bitmap(
+        self, output: np.ndarray, mask: np.ndarray, dest_width: int, dest_height: int
+    ) -> Tuple[np.ndarray, List[float]]:
+        """
+        Process the binary mask to extract bounding boxes and associated confidence scores.
+
+        Args:
+            output (np.ndarray): Output confidence map from the model.
+            mask (np.ndarray): Binary mask of detected regions.
+            dest_width (int): Target width for scaling the box coordinates.
+            dest_height (int): Target height for scaling the box coordinates.
+
+        Returns:
+            Tuple[np.ndarray, List[float]]: Array of bounding boxes and list of corresponding scores.
+        """
+
+        mask = (mask * 255).astype(np.uint8)
+        height, width = mask.shape
+        outs = cv2.findContours(mask, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
+
+        # handle different return values of findContours between OpenCV versions
+        contours = outs[0] if len(outs) == 2 else outs[1]
+
+        boxes = []
+        scores = []
+
+        for index in range(len(contours)):
+            contour = contours[index]
+
+            # get minimum bounding box (rotated rectangle) around the contour and the smallest side length.
+            points, min_side = self.get_min_boxes(contour)
+
+            if min_side < self.min_size:
+                continue
+
+            points = np.array(points)
+
+            score = self.box_score(output, contour)
+            if self.box_thresh > score:
+                continue
+
+            polygon = Polygon(points)
+            distance = polygon.area / polygon.length
+
+            # Use pyclipper to shrink the polygon slightly based on the computed distance.
+            offset = PyclipperOffset()
+            offset.AddPath(points, JT_ROUND, ET_CLOSEDPOLYGON)
+            points = np.array(offset.Execute(distance * 1.5)).reshape((-1, 1, 2))
+
+            # get the minimum bounding box around the shrunken polygon.
+            box, min_side = self.get_min_boxes(points)
+
+            if min_side < self.min_size + 2:
+                continue
+
+            box = np.array(box)
+
+            # normalize and clip box coordinates to fit within the destination image size.
+            box[:, 0] = np.clip(np.round(box[:, 0] / width * dest_width), 0, dest_width)
+            box[:, 1] = np.clip(
+                np.round(box[:, 1] / height * dest_height), 0, dest_height
+            )
+
+            boxes.append(box.astype("int32"))
+            scores.append(score)
+
+        return np.array(boxes, dtype="int32"), scores
+
+    @staticmethod
+    def get_min_boxes(contour: np.ndarray) -> Tuple[List[Tuple[float, float]], float]:
+        """
+        Calculate the minimum bounding box (rotated rectangle) for a given contour.
+
+        Args:
+            contour (np.ndarray): The contour points of the detected shape.
+
+        Returns:
+            Tuple[List[Tuple[float, float]], float]: A list of four points representing the
+            corners of the bounding box, and the length of the shortest side.
+        """
+        bounding_box = cv2.minAreaRect(contour)
+        points = sorted(cv2.boxPoints(bounding_box), key=lambda x: x[0])
+        index_1, index_4 = (0, 1) if points[1][1] > points[0][1] else (1, 0)
+        index_2, index_3 = (2, 3) if points[3][1] > points[2][1] else (3, 2)
+        box = [points[index_1], points[index_2], points[index_3], points[index_4]]
+        return box, min(bounding_box[1])
+
+    @staticmethod
+    def box_score(bitmap: np.ndarray, contour: np.ndarray) -> float:
+        """
+        Calculate the average score within the bounding box of a contour.
+
+        Args:
+            bitmap (np.ndarray): The output confidence map from the model.
+            contour (np.ndarray): The contour of the detected shape.
+
+        Returns:
+            float: The average score of the pixels inside the contour region.
+        """
+        h, w = bitmap.shape[:2]
+        contour = contour.reshape(-1, 2)
+        x1, y1 = np.clip(contour.min(axis=0), 0, [w - 1, h - 1])
+        x2, y2 = np.clip(contour.max(axis=0), 0, [w - 1, h - 1])
+        mask = np.zeros((y2 - y1 + 1, x2 - x1 + 1), dtype=np.uint8)
+        cv2.fillPoly(mask, [contour - [x1, y1]], 1)
+        return cv2.mean(bitmap[y1 : y2 + 1, x1 : x2 + 1], mask)[0]
+
+    @staticmethod
+    def expand_box(points: List[Tuple[float, float]]) -> np.ndarray:
+        """
+        Expand a polygonal shape slightly by a factor determined by the area-to-perimeter ratio.
+
+        Args:
+            points (List[Tuple[float, float]]): Points of the polygon to expand.
+
+        Returns:
+            np.ndarray: Expanded polygon points.
+        """
+        polygon = Polygon(points)
+        distance = polygon.area / polygon.length
+        offset = PyclipperOffset()
+        offset.AddPath(points, JT_ROUND, ET_CLOSEDPOLYGON)
+        expanded = np.array(offset.Execute(distance * 1.5)).reshape((-1, 2))
+        return expanded
+
+    def filter_polygon(
+        self, points: List[np.ndarray], shape: Tuple[int, int]
+    ) -> np.ndarray:
+        """
+        Filter a set of polygons to include only valid ones that fit within an image shape
+        and meet size constraints.
+
+        Args:
+            points (List[np.ndarray]): List of polygons to filter.
+            shape (Tuple[int, int]): Shape of the image (height, width).
+
+        Returns:
+            np.ndarray: List of filtered polygons.
+        """
+        height, width = shape
+        return np.array(
+            [
+                self.clockwise_order(point)
+                for point in points
+                if self.is_valid_polygon(point, width, height)
+            ]
+        )
+
+    @staticmethod
+    def is_valid_polygon(point: np.ndarray, width: int, height: int) -> bool:
+        """
+        Check if a polygon is valid, meaning it fits within the image bounds
+        and has sides of a minimum length.
+
+        Args:
+            point (np.ndarray): The polygon to validate.
+            width (int): Image width.
+            height (int): Image height.
+
+        Returns:
+            bool: Whether the polygon is valid or not.
+        """
+        return (
+            point[:, 0].min() >= 0
+            and point[:, 0].max() < width
+            and point[:, 1].min() >= 0
+            and point[:, 1].max() < height
+            and np.linalg.norm(point[0] - point[1]) > 3
+            and np.linalg.norm(point[0] - point[3]) > 3
+        )
+
+    @staticmethod
+    def clockwise_order(point: np.ndarray) -> np.ndarray:
+        """
+        Arrange the points of a polygon in clockwise order based on their angular positions
+        around the polygon's center.
+
+        Args:
+            point (np.ndarray): Array of points of the polygon.
+
+        Returns:
+            np.ndarray: Points ordered in clockwise direction.
+        """
+        center = point.mean(axis=0)
+        return point[
+            np.argsort(np.arctan2(point[:, 1] - center[1], point[:, 0] - center[0]))
+        ]
+
+    @staticmethod
+    def sort_polygon(points):
+        """
+        Sort polygons based on their position in the image. If polygons are close in vertical
+        position (within 10 pixels), sort them by horizontal position.
+
+        Args:
+            points: List of polygons to sort.
+
+        Returns:
+            List: Sorted list of polygons.
+        """
+        points.sort(key=lambda x: (x[0][1], x[0][0]))
+        for i in range(len(points) - 1):
+            for j in range(i, -1, -1):
+                if abs(points[j + 1][0][1] - points[j][0][1]) < 10 and (
+                    points[j + 1][0][0] < points[j][0][0]
+                ):
+                    temp = points[j]
+                    points[j] = points[j + 1]
+                    points[j + 1] = temp
+                else:
+                    break
+        return points
+
+    @staticmethod
+    def zero_pad(image: np.ndarray) -> np.ndarray:
+        """
+        Apply zero-padding to an image, ensuring its dimensions are at least 32x32.
+        The padding is added only if needed.
+
+        Args:
+            image (np.ndarray): Input image.
+
+        Returns:
+            np.ndarray: Zero-padded image.
+        """
+        h, w, c = image.shape
+        pad = np.zeros((max(32, h), max(32, w), c), np.uint8)
+        pad[:h, :w, :] = image
+        return pad
+
+    @staticmethod
+    def _preprocess_classification_image(image: np.ndarray) -> np.ndarray:
+        """
+        Preprocess a single image for classification by resizing, normalizing, and padding.
+
+        This method resizes the input image to a fixed height of 48 pixels while adjusting
+        the width dynamically up to a maximum of 192 pixels. The image is then normalized and
+        padded to fit the required input dimensions for classification.
+
+        Args:
+            image (np.ndarray): Input image to preprocess.
+
+        Returns:
+            np.ndarray: Preprocessed and padded image.
+        """
+        # fixed height of 48, dynamic width up to 192
+        input_shape = (3, 48, 192)
+        input_c, input_h, input_w = input_shape
+
+        h, w = image.shape[:2]
+        ratio = w / h
+        resized_w = min(input_w, math.ceil(input_h * ratio))
+
+        resized_image = cv2.resize(image, (resized_w, input_h))
+
+        # handle single-channel images (grayscale) if needed
+        if input_c == 1 and resized_image.ndim == 2:
+            resized_image = resized_image[np.newaxis, :, :]
+        else:
+            resized_image = resized_image.transpose((2, 0, 1))
+
+        # normalize
+        resized_image = (resized_image.astype("float32") / 255.0 - 0.5) / 0.5
+
+        padded_image = np.zeros((input_c, input_h, input_w), dtype=np.float32)
+        padded_image[:, :, :resized_w] = resized_image
+
+        return padded_image
+
+    def _process_classification_output(
+        self, images: List[np.ndarray], outputs: List[np.ndarray]
+    ) -> Tuple[List[np.ndarray], List[Tuple[str, float]]]:
+        """
+        Process the classification model output by matching labels with confidence scores.
+
+        This method processes the outputs from the classification model and rotates images
+        with high confidence of being labeled "180". It ensures that results are mapped to
+        the original image order.
+
+        Args:
+            images (List[np.ndarray]): List of input images.
+            outputs (List[np.ndarray]): Corresponding model outputs.
+
+        Returns:
+            Tuple[List[np.ndarray], List[Tuple[str, float]]]: A tuple of processed images and
+            classification results (label and confidence score).
+        """
+        labels = ["0", "180"]
+        results = [["", 0.0]] * len(images)
+        indices = np.argsort(np.array([x.shape[1] / x.shape[0] for x in images]))
+
+        outputs = np.stack(outputs)
+
+        outputs = [
+            (labels[idx], outputs[i, idx])
+            for i, idx in enumerate(outputs.argmax(axis=1))
+        ]
+
+        for i in range(0, len(images), self.batch_size):
+            for j in range(len(outputs)):
+                label, score = outputs[j]
+                results[indices[i + j]] = [label, score]
+                if "180" in label and score >= self.lpr_config.threshold:
+                    images[indices[i + j]] = cv2.rotate(images[indices[i + j]], 1)
+
+        return images, results
+
+    def _preprocess_recognition_image(
+        self, image: np.ndarray, max_wh_ratio: float
+    ) -> np.ndarray:
+        """
+        Preprocess an image for recognition by dynamically adjusting its width.
+
+        This method adjusts the width of the image based on the maximum width-to-height ratio
+        while keeping the height fixed at 48 pixels. The image is then normalized and padded
+        to fit the required input dimensions for recognition.
+
+        Args:
+            image (np.ndarray): Input image to preprocess.
+            max_wh_ratio (float): Maximum width-to-height ratio for resizing.
+
+        Returns:
+            np.ndarray: Preprocessed and padded image.
+        """
+        # fixed height of 48, dynamic width based on ratio
+        input_shape = [3, 48, 320]
+        input_h, input_w = input_shape[1], input_shape[2]
+
+        assert image.shape[2] == input_shape[0], "Unexpected number of image channels."
+
+        # dynamically adjust input width based on max_wh_ratio
+        input_w = int(input_h * max_wh_ratio)
+
+        # check for model-specific input width
+        model_input_w = self.recognition_model.runner.ort.get_inputs()[0].shape[3]
+        if isinstance(model_input_w, int) and model_input_w > 0:
+            input_w = model_input_w
+
+        h, w = image.shape[:2]
+        aspect_ratio = w / h
+        resized_w = min(input_w, math.ceil(input_h * aspect_ratio))
+
+        resized_image = cv2.resize(image, (resized_w, input_h))
+        resized_image = resized_image.transpose((2, 0, 1))
+        resized_image = (resized_image.astype("float32") / 255.0 - 0.5) / 0.5
+
+        padded_image = np.zeros((input_shape[0], input_h, input_w), dtype=np.float32)
+        padded_image[:, :, :resized_w] = resized_image
+
+        return padded_image
+
+    @staticmethod
+    def _crop_license_plate(image: np.ndarray, points: np.ndarray) -> np.ndarray:
+        """
+        Crop the license plate from the image using four corner points.
+
+        This method crops the region containing the license plate by using the perspective
+        transformation based on four corner points. If the resulting image is significantly
+        taller than wide, the image is rotated to the correct orientation.
+
+        Args:
+            image (np.ndarray): Input image containing the license plate.
+            points (np.ndarray): Four corner points defining the plate's position.
+
+        Returns:
+            np.ndarray: Cropped and potentially rotated license plate image.
+        """
+        assert len(points) == 4, "shape of points must be 4*2"
+        points = points.astype(np.float32)
+        crop_width = int(
+            max(
+                np.linalg.norm(points[0] - points[1]),
+                np.linalg.norm(points[2] - points[3]),
+            )
+        )
+        crop_height = int(
+            max(
+                np.linalg.norm(points[0] - points[3]),
+                np.linalg.norm(points[1] - points[2]),
+            )
+        )
+        pts_std = np.float32(
+            [[0, 0], [crop_width, 0], [crop_width, crop_height], [0, crop_height]]
+        )
+        matrix = cv2.getPerspectiveTransform(points, pts_std)
+        image = cv2.warpPerspective(
+            image,
+            matrix,
+            (crop_width, crop_height),
+            borderMode=cv2.BORDER_REPLICATE,
+            flags=cv2.INTER_CUBIC,
+        )
+        height, width = image.shape[0:2]
+        if height * 1.0 / width >= 1.5:
+            image = np.rot90(image, k=3)
+        return image
+
+
+class CTCDecoder:
+    """
+    A decoder for interpreting the output of a CTC (Connectionist Temporal Classification) model.
+
+    This decoder converts the model's output probabilities into readable sequences of characters
+    while removing duplicates and handling blank tokens. It also calculates the confidence scores
+    for each decoded character sequence.
+    """
+
+    def __init__(self):
+        """
+        Initialize the CTCDecoder with a list of characters and a character map.
+
+        The character set includes digits, letters, special characters, and a "blank" token
+        (used by the CTC model for decoding purposes). A character map is created to map
+        indices to characters.
+        """
+        self.characters = [
+            "blank",
+            "0",
+            "1",
+            "2",
+            "3",
+            "4",
+            "5",
+            "6",
+            "7",
+            "8",
+            "9",
+            ":",
+            ";",
+            "<",
+            "=",
+            ">",
+            "?",
+            "@",
+            "A",
+            "B",
+            "C",
+            "D",
+            "E",
+            "F",
+            "G",
+            "H",
+            "I",
+            "J",
+            "K",
+            "L",
+            "M",
+            "N",
+            "O",
+            "P",
+            "Q",
+            "R",
+            "S",
+            "T",
+            "U",
+            "V",
+            "W",
+            "X",
+            "Y",
+            "Z",
+            "[",
+            "\\",
+            "]",
+            "^",
+            "_",
+            "`",
+            "a",
+            "b",
+            "c",
+            "d",
+            "e",
+            "f",
+            "g",
+            "h",
+            "i",
+            "j",
+            "k",
+            "l",
+            "m",
+            "n",
+            "o",
+            "p",
+            "q",
+            "r",
+            "s",
+            "t",
+            "u",
+            "v",
+            "w",
+            "x",
+            "y",
+            "z",
+            "{",
+            "|",
+            "}",
+            "~",
+            "!",
+            '"',
+            "#",
+            "$",
+            "%",
+            "&",
+            "'",
+            "(",
+            ")",
+            "*",
+            "+",
+            ",",
+            "-",
+            ".",
+            "/",
+            " ",
+            " ",
+        ]
+        self.char_map = {i: char for i, char in enumerate(self.characters)}
+
+    def __call__(
+        self, outputs: List[np.ndarray]
+    ) -> Tuple[List[str], List[List[float]]]:
+        """
+        Decode a batch of model outputs into character sequences and their confidence scores.
+
+        The method takes the output probability distributions for each time step and uses
+        the best path decoding strategy. It then merges repeating characters and ignores
+        blank tokens. Confidence scores for each decoded character are also calculated.
+
+        Args:
+            outputs (List[np.ndarray]): A list of model outputs, where each element is
+                                        a probability distribution for each time step.
+
+        Returns:
+            Tuple[List[str], List[List[float]]]: A tuple of decoded character sequences
+                                                and confidence scores for each sequence.
+        """
+        results = []
+        confidences = []
+        for output in outputs:
+            seq_log_probs = np.log(output + 1e-8)
+            best_path = np.argmax(seq_log_probs, axis=1)
+
+            merged_path = []
+            merged_probs = []
+            for t, char_index in enumerate(best_path):
+                if char_index != 0 and (t == 0 or char_index != best_path[t - 1]):
+                    merged_path.append(char_index)
+                    merged_probs.append(seq_log_probs[t, char_index])
+
+            result = "".join(self.char_map[idx] for idx in merged_path)
+            results.append(result)
+
+            confidence = np.exp(merged_probs).tolist()
+            confidences.append(confidence)
+
+        return results, confidences

frigate/embeddings/maintainer.py

@@ -3,12 +3,17 @@
 import base64
 import logging
 import os
+import random
+import re
+import string
 import threading
 from multiprocessing.synchronize import Event as MpEvent
+from pathlib import Path
 from typing import Optional
 
 import cv2
 import numpy as np
+import requests
 from peewee import DoesNotExist
 from playhouse.sqliteq import SqliteQueueDatabase
 
@@ -20,13 +25,20 @@ from frigate.comms.event_metadata_updater import (
 from frigate.comms.events_updater import EventEndSubscriber, EventUpdateSubscriber
 from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config import FrigateConfig
-from frigate.const import CLIPS_DIR, UPDATE_EVENT_DESCRIPTION
+from frigate.const import (
+    CLIPS_DIR,
+    FACE_DIR,
+    FRIGATE_LOCALHOST,
+    UPDATE_EVENT_DESCRIPTION,
+)
+from frigate.embeddings.lpr.lpr import LicensePlateRecognition
 from frigate.events.types import EventTypeEnum
 from frigate.genai import get_genai_client
 from frigate.models import Event
 from frigate.types import TrackedObjectUpdateTypesEnum
 from frigate.util.builtin import serialize
-from frigate.util.image import SharedMemoryFrameManager, calculate_region
+from frigate.util.image import SharedMemoryFrameManager, area, calculate_region
+from frigate.util.model import FaceClassificationModel
 
 from .embeddings import Embeddings
 
@@ -46,7 +58,7 @@ class EmbeddingMaintainer(threading.Thread):
     ) -> None:
         super().__init__(name="embeddings_maintainer")
         self.config = config
-        self.embeddings = Embeddings(config.semantic_search, db)
+        self.embeddings = Embeddings(config, db)
 
         # Check if we need to re-index events
         if config.semantic_search.reindex:
@@ -59,12 +71,48 @@ class EmbeddingMaintainer(threading.Thread):
         )
         self.embeddings_responder = EmbeddingsResponder()
         self.frame_manager = SharedMemoryFrameManager()
+
+        # set face recognition conditions
+        self.face_recognition_enabled = self.config.face_recognition.enabled
+        self.requires_face_detection = "face" not in self.config.objects.all_objects
+        self.detected_faces: dict[str, float] = {}
+        self.face_classifier = (
+            FaceClassificationModel(self.config.face_recognition, db)
+            if self.face_recognition_enabled
+            else None
+        )
+
         # create communication for updating event descriptions
         self.requestor = InterProcessRequestor()
         self.stop_event = stop_event
-        self.tracked_events = {}
+        self.tracked_events: dict[str, list[any]] = {}
         self.genai_client = get_genai_client(config)
 
+        # set license plate recognition conditions
+        self.lpr_config = self.config.lpr
+        self.requires_license_plate_detection = (
+            "license_plate" not in self.config.objects.all_objects
+        )
+        self.detected_license_plates: dict[str, dict[str, any]] = {}
+
+        if self.lpr_config.enabled:
+            self.license_plate_recognition = LicensePlateRecognition(
+                self.lpr_config, self.requestor, self.embeddings
+            )
+
+    @property
+    def face_detector(self) -> cv2.FaceDetectorYN:
+        # Lazily create the classifier.
+        if "face_detector" not in self.__dict__:
+            self.__dict__["face_detector"] = cv2.FaceDetectorYN.create(
+                "/config/model_cache/facedet/facedet.onnx",
+                config="",
+                input_size=(320, 320),
+                score_threshold=0.8,
+                nms_threshold=0.3,
+            )
+        return self.__dict__["face_detector"]
+
     def run(self) -> None:
         """Maintain a SQLite-vec database for semantic search."""
         while not self.stop_event.is_set():
@@ -83,7 +131,7 @@ class EmbeddingMaintainer(threading.Thread):
     def _process_requests(self) -> None:
         """Process embeddings requests"""
 
-        def _handle_request(topic: str, data: str) -> str:
+        def _handle_request(topic: str, data: dict[str, any]) -> str:
             try:
                 if topic == EmbeddingsRequestEnum.embed_description.value:
                     return serialize(
@@ -102,6 +150,46 @@ class EmbeddingMaintainer(threading.Thread):
                     return serialize(
                         self.embeddings.text_embedding([data])[0], pack=False
                     )
+                elif topic == EmbeddingsRequestEnum.register_face.value:
+                    if not self.face_recognition_enabled:
+                        return False
+
+                    rand_id = "".join(
+                        random.choices(string.ascii_lowercase + string.digits, k=6)
+                    )
+                    label = data["face_name"]
+                    id = f"{label}-{rand_id}"
+
+                    if data.get("cropped"):
+                        pass
+                    else:
+                        img = cv2.imdecode(
+                            np.frombuffer(
+                                base64.b64decode(data["image"]), dtype=np.uint8
+                            ),
+                            cv2.IMREAD_COLOR,
+                        )
+                        face_box = self._detect_face(img)
+
+                        if not face_box:
+                            return False
+
+                        face = img[face_box[1] : face_box[3], face_box[0] : face_box[2]]
+                        ret, thumbnail = cv2.imencode(
+                            ".webp", face, [int(cv2.IMWRITE_WEBP_QUALITY), 100]
+                        )
+
+                    # write face to library
+                    folder = os.path.join(FACE_DIR, label)
+                    file = os.path.join(folder, f"{id}.webp")
+                    os.makedirs(folder, exist_ok=True)
+
+                    # save face image
+                    with open(file, "wb") as output:
+                        output.write(thumbnail.tobytes())
+
+                self.face_classifier.clear_classifier()
+                return True
             except Exception as e:
                 logger.error(f"Unable to handle embeddings request {e}")
 
@@ -109,7 +197,7 @@ class EmbeddingMaintainer(threading.Thread):
 
     def _process_updates(self) -> None:
         """Process event updates"""
-        update = self.event_subscriber.check_for_update(timeout=0.1)
+        update = self.event_subscriber.check_for_update(timeout=0.01)
 
         if update is None:
             return
@@ -120,42 +208,56 @@ class EmbeddingMaintainer(threading.Thread):
             return
 
         camera_config = self.config.cameras[camera]
-        # no need to save our own thumbnails if genai is not enabled
-        # or if the object has become stationary
+
+        # no need to process updated objects if face recognition, lpr, genai are disabled
         if (
             not camera_config.genai.enabled
-            or self.genai_client is None
-            or data["stationary"]
+            and not self.face_recognition_enabled
+            and not self.lpr_config.enabled
         ):
             return
 
-        if data["id"] not in self.tracked_events:
-            self.tracked_events[data["id"]] = []
-
         # Create our own thumbnail based on the bounding box and the frame time
         try:
             yuv_frame = self.frame_manager.get(
                 frame_name, camera_config.frame_shape_yuv
             )
-
-            if yuv_frame is not None:
-                data["thumbnail"] = self._create_thumbnail(yuv_frame, data["box"])
-
-                # Limit the number of thumbnails saved
-                if len(self.tracked_events[data["id"]]) >= MAX_THUMBNAILS:
-                    # Always keep the first thumbnail for the event
-                    self.tracked_events[data["id"]].pop(1)
-
-                self.tracked_events[data["id"]].append(data)
-
-                self.frame_manager.close(frame_name)
         except FileNotFoundError:
             pass
 
+        if yuv_frame is None:
+            logger.debug(
+                "Unable to process object update because frame is unavailable."
+            )
+            return
+
+        if self.face_recognition_enabled:
+            self._process_face(data, yuv_frame)
+
+        if self.lpr_config.enabled:
+            self._process_license_plate(data, yuv_frame)
+
+        # no need to save our own thumbnails if genai is not enabled
+        # or if the object has become stationary
+        if self.genai_client is not None and not data["stationary"]:
+            if data["id"] not in self.tracked_events:
+                self.tracked_events[data["id"]] = []
+
+            data["thumbnail"] = self._create_thumbnail(yuv_frame, data["box"])
+
+            # Limit the number of thumbnails saved
+            if len(self.tracked_events[data["id"]]) >= MAX_THUMBNAILS:
+                # Always keep the first thumbnail for the event
+                self.tracked_events[data["id"]].pop(1)
+
+            self.tracked_events[data["id"]].append(data)
+
+        self.frame_manager.close(frame_name)
+
     def _process_finalized(self) -> None:
         """Process the end of an event."""
         while True:
-            ended = self.event_end_subscriber.check_for_update(timeout=0.1)
+            ended = self.event_end_subscriber.check_for_update(timeout=0.01)
 
             if ended == None:
                 break
@@ -163,6 +265,12 @@ class EmbeddingMaintainer(threading.Thread):
             event_id, camera, updated_db = ended
             camera_config = self.config.cameras[camera]
 
+            if event_id in self.detected_faces:
+                self.detected_faces.pop(event_id)
+
+            if event_id in self.detected_license_plates:
+                self.detected_license_plates.pop(event_id)
+
             if updated_db:
                 try:
                     event: Event = Event.get(Event.id == event_id)
@@ -217,16 +325,47 @@ class EmbeddingMaintainer(threading.Thread):
                             _, buffer = cv2.imencode(".jpg", cropped_image)
                             snapshot_image = buffer.tobytes()
 
+                    num_thumbnails = len(self.tracked_events.get(event_id, []))
+
                     embed_image = (
                         [snapshot_image]
                         if event.has_snapshot and camera_config.genai.use_snapshot
                         else (
-                            [thumbnail for data in self.tracked_events[event_id]]
-                            if len(self.tracked_events.get(event_id, [])) > 0
+                            [
+                                data["thumbnail"]
+                                for data in self.tracked_events[event_id]
+                            ]
+                            if num_thumbnails > 0
                             else [thumbnail]
                         )
                     )
 
+                    if camera_config.genai.debug_save_thumbnails and num_thumbnails > 0:
+                        logger.debug(
+                            f"Saving {num_thumbnails} thumbnails for event {event.id}"
+                        )
+
+                        Path(
+                            os.path.join(CLIPS_DIR, f"genai-requests/{event.id}")
+                        ).mkdir(parents=True, exist_ok=True)
+
+                        for idx, data in enumerate(self.tracked_events[event_id], 1):
+                            jpg_bytes: bytes = data["thumbnail"]
+
+                            if jpg_bytes is None:
+                                logger.warning(
+                                    f"Unable to save thumbnail {idx} for {event.id}."
+                                )
+                            else:
+                                with open(
+                                    os.path.join(
+                                        CLIPS_DIR,
+                                        f"genai-requests/{event.id}/{idx}.jpg",
+                                    ),
+                                    "wb",
+                                ) as j:
+                                    j.write(jpg_bytes)
+
                     # Generate the description. Call happens in a thread since it is network bound.
                     threading.Thread(
                         target=self._embed_description,
@@ -245,7 +384,7 @@ class EmbeddingMaintainer(threading.Thread):
     def _process_event_metadata(self):
         # Check for regenerate description requests
         (topic, event_id, source) = self.event_metadata_subscriber.check_for_update(
-            timeout=0.1
+            timeout=0.01
         )
 
         if topic is None:
@@ -254,6 +393,350 @@ class EmbeddingMaintainer(threading.Thread):
         if event_id:
             self.handle_regenerate_description(event_id, source)
 
+    def _detect_face(self, input: np.ndarray) -> tuple[int, int, int, int]:
+        """Detect faces in input image."""
+        self.face_detector.setInputSize((input.shape[1], input.shape[0]))
+        faces = self.face_detector.detect(input)
+
+        if faces[1] is None:
+            return None
+
+        face = None
+
+        for _, potential_face in enumerate(faces[1]):
+            raw_bbox = potential_face[0:4].astype(np.uint16)
+            x: int = max(raw_bbox[0], 0)
+            y: int = max(raw_bbox[1], 0)
+            w: int = raw_bbox[2]
+            h: int = raw_bbox[3]
+            bbox = (x, y, x + w, y + h)
+
+            if face is None or area(bbox) > area(face):
+                face = bbox
+
+        return face
+
+    def _process_face(self, obj_data: dict[str, any], frame: np.ndarray) -> None:
+        """Look for faces in image."""
+        id = obj_data["id"]
+
+        # don't run for non person objects
+        if obj_data.get("label") != "person":
+            logger.debug("Not a processing face for non person object.")
+            return
+
+        # don't overwrite sub label for objects that have a sub label
+        # that is not a face
+        if obj_data.get("sub_label") and id not in self.detected_faces:
+            logger.debug(
+                f"Not processing face due to existing sub label: {obj_data.get('sub_label')}."
+            )
+            return
+
+        face: Optional[dict[str, any]] = None
+
+        if self.requires_face_detection:
+            logger.debug("Running manual face detection.")
+            person_box = obj_data.get("box")
+
+            if not person_box:
+                return None
+
+            rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420)
+            left, top, right, bottom = person_box
+            person = rgb[top:bottom, left:right]
+            face_box = self._detect_face(person)
+
+            if not face_box:
+                logger.debug("Detected no faces for person object.")
+                return
+
+            margin = int((face_box[2] - face_box[0]) * 0.25)
+            face_frame = person[
+                max(0, face_box[1] - margin) : min(
+                    frame.shape[0], face_box[3] + margin
+                ),
+                max(0, face_box[0] - margin) : min(
+                    frame.shape[1], face_box[2] + margin
+                ),
+            ]
+            face_frame = cv2.cvtColor(face_frame, cv2.COLOR_RGB2BGR)
+        else:
+            # don't run for object without attributes
+            if not obj_data.get("current_attributes"):
+                logger.debug("No attributes to parse.")
+                return
+
+            attributes: list[dict[str, any]] = obj_data.get("current_attributes", [])
+            for attr in attributes:
+                if attr.get("label") != "face":
+                    continue
+
+                if face is None or attr.get("score", 0.0) > face.get("score", 0.0):
+                    face = attr
+
+            # no faces detected in this frame
+            if not face:
+                return
+
+            face_box = face.get("box")
+
+            # check that face is valid
+            if not face_box or area(face_box) < self.config.face_recognition.min_area:
+                logger.debug(f"Invalid face box {face}")
+                return
+
+            face_frame = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
+            margin = int((face_box[2] - face_box[0]) * 0.25)
+
+            face_frame = face_frame[
+                max(0, face_box[1] - margin) : min(
+                    frame.shape[0], face_box[3] + margin
+                ),
+                max(0, face_box[0] - margin) : min(
+                    frame.shape[1], face_box[2] + margin
+                ),
+            ]
+
+        res = self.face_classifier.classify_face(face_frame)
+
+        if not res:
+            return
+
+        sub_label, score = res
+
+        # calculate the overall face score as the probability * area of face
+        # this will help to reduce false positives from small side-angle faces
+        # if a large front-on face image may have scored slightly lower but
+        # is more likely to be accurate due to the larger face area
+        face_score = round(score * face_frame.shape[0] * face_frame.shape[1], 2)
+
+        logger.debug(
+            f"Detected best face for person as: {sub_label} with probability {score} and overall face score {face_score}"
+        )
+
+        if self.config.face_recognition.save_attempts:
+            # write face to library
+            folder = os.path.join(FACE_DIR, "debug")
+            file = os.path.join(folder, f"{id}-{sub_label}-{score}-{face_score}.webp")
+            os.makedirs(folder, exist_ok=True)
+            cv2.imwrite(file, face_frame)
+
+        if score < self.config.face_recognition.threshold:
+            logger.debug(
+                f"Recognized face distance {score} is less than threshold {self.config.face_recognition.threshold}"
+            )
+            return
+
+        if id in self.detected_faces and face_score <= self.detected_faces[id]:
+            logger.debug(
+                f"Recognized face distance {score} and overall score {face_score} is less than previous overall face score ({self.detected_faces.get(id)})."
+            )
+            return
+
+        resp = requests.post(
+            f"{FRIGATE_LOCALHOST}/api/events/{id}/sub_label",
+            json={
+                "camera": obj_data.get("camera"),
+                "subLabel": sub_label,
+                "subLabelScore": score,
+            },
+        )
+
+        if resp.status_code == 200:
+            self.detected_faces[id] = face_score
+
+    def _detect_license_plate(self, input: np.ndarray) -> tuple[int, int, int, int]:
+        """Return the dimensions of the input image as [x, y, width, height]."""
+        height, width = input.shape[:2]
+        return (0, 0, width, height)
+
+    def _process_license_plate(
+        self, obj_data: dict[str, any], frame: np.ndarray
+    ) -> None:
+        """Look for license plates in image."""
+        id = obj_data["id"]
+
+        # don't run for non car objects
+        if obj_data.get("label") != "car":
+            logger.debug("Not a processing license plate for non car object.")
+            return
+
+        # don't run for stationary car objects
+        if obj_data.get("stationary") == True:
+            logger.debug("Not a processing license plate for a stationary car object.")
+            return
+
+        # don't overwrite sub label for objects that have a sub label
+        # that is not a license plate
+        if obj_data.get("sub_label") and id not in self.detected_license_plates:
+            logger.debug(
+                f"Not processing license plate due to existing sub label: {obj_data.get('sub_label')}."
+            )
+            return
+
+        license_plate: Optional[dict[str, any]] = None
+
+        if self.requires_license_plate_detection:
+            logger.debug("Running manual license_plate detection.")
+            car_box = obj_data.get("box")
+
+            if not car_box:
+                return None
+
+            rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420)
+            left, top, right, bottom = car_box
+            car = rgb[top:bottom, left:right]
+            license_plate = self._detect_license_plate(car)
+
+            if not license_plate:
+                logger.debug("Detected no license plates for car object.")
+                return
+
+            license_plate_frame = car[
+                license_plate[1] : license_plate[3], license_plate[0] : license_plate[2]
+            ]
+            license_plate_frame = cv2.cvtColor(license_plate_frame, cv2.COLOR_RGB2BGR)
+        else:
+            # don't run for object without attributes
+            if not obj_data.get("current_attributes"):
+                logger.debug("No attributes to parse.")
+                return
+
+            attributes: list[dict[str, any]] = obj_data.get("current_attributes", [])
+            for attr in attributes:
+                if attr.get("label") != "license_plate":
+                    continue
+
+                if license_plate is None or attr.get("score", 0.0) > license_plate.get(
+                    "score", 0.0
+                ):
+                    license_plate = attr
+
+            # no license plates detected in this frame
+            if not license_plate:
+                return
+
+            license_plate_box = license_plate.get("box")
+
+            # check that license plate is valid
+            if (
+                not license_plate_box
+                or area(license_plate_box) < self.config.lpr.min_area
+            ):
+                logger.debug(f"Invalid license plate box {license_plate}")
+                return
+
+            license_plate_frame = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
+            license_plate_frame = license_plate_frame[
+                license_plate_box[1] : license_plate_box[3],
+                license_plate_box[0] : license_plate_box[2],
+            ]
+
+        # run detection, returns results sorted by confidence, best first
+        license_plates, confidences, areas = (
+            self.license_plate_recognition.process_license_plate(license_plate_frame)
+        )
+
+        logger.debug(f"Text boxes: {license_plates}")
+        logger.debug(f"Confidences: {confidences}")
+        logger.debug(f"Areas: {areas}")
+
+        if license_plates:
+            for plate, confidence, text_area in zip(license_plates, confidences, areas):
+                avg_confidence = (
+                    (sum(confidence) / len(confidence)) if confidence else 0
+                )
+
+                logger.debug(
+                    f"Detected text: {plate} (average confidence: {avg_confidence:.2f}, area: {text_area} pixels)"
+                )
+        else:
+            # no plates found
+            logger.debug("No text detected")
+            return
+
+        top_plate, top_char_confidences, top_area = (
+            license_plates[0],
+            confidences[0],
+            areas[0],
+        )
+        avg_confidence = (
+            (sum(top_char_confidences) / len(top_char_confidences))
+            if top_char_confidences
+            else 0
+        )
+
+        # Check if we have a previously detected plate for this ID
+        if id in self.detected_license_plates:
+            prev_plate = self.detected_license_plates[id]["plate"]
+            prev_char_confidences = self.detected_license_plates[id]["char_confidences"]
+            prev_area = self.detected_license_plates[id]["area"]
+            prev_avg_confidence = (
+                (sum(prev_char_confidences) / len(prev_char_confidences))
+                if prev_char_confidences
+                else 0
+            )
+
+            # Define conditions for keeping the previous plate
+            shorter_than_previous = len(top_plate) < len(prev_plate)
+            lower_avg_confidence = avg_confidence <= prev_avg_confidence
+            smaller_area = top_area < prev_area
+
+            # Compare character-by-character confidence where possible
+            min_length = min(len(top_plate), len(prev_plate))
+            char_confidence_comparison = sum(
+                1
+                for i in range(min_length)
+                if top_char_confidences[i] <= prev_char_confidences[i]
+            )
+            worse_char_confidences = char_confidence_comparison >= min_length / 2
+
+            if (shorter_than_previous or smaller_area) and (
+                lower_avg_confidence and worse_char_confidences
+            ):
+                logger.debug(
+                    f"Keeping previous plate. New plate stats: "
+                    f"length={len(top_plate)}, avg_conf={avg_confidence:.2f}, area={top_area} "
+                    f"vs Previous: length={len(prev_plate)}, avg_conf={prev_avg_confidence:.2f}, area={prev_area}"
+                )
+                return
+
+        # Check against minimum confidence threshold
+        if avg_confidence < self.lpr_config.threshold:
+            logger.debug(
+                f"Average confidence {avg_confidence} is less than threshold ({self.lpr_config.threshold})"
+            )
+            return
+
+        # Determine subLabel based on known plates, use regex matching
+        # Default to the detected plate, use label name if there's a match
+        sub_label = next(
+            (
+                label
+                for label, plates in self.lpr_config.known_plates.items()
+                if any(re.match(f"^{plate}$", top_plate) for plate in plates)
+            ),
+            top_plate,
+        )
+
+        # Send the result to the API
+        resp = requests.post(
+            f"{FRIGATE_LOCALHOST}/api/events/{id}/sub_label",
+            json={
+                "camera": obj_data.get("camera"),
+                "subLabel": sub_label,
+                "subLabelScore": avg_confidence,
+            },
+        )
+
+        if resp.status_code == 200:
+            self.detected_license_plates[id] = {
+                "plate": top_plate,
+                "char_confidences": top_char_confidences,
+                "area": top_area,
+            }
+
     def _create_thumbnail(self, yuv_frame, box, height=500) -> Optional[bytes]:
         """Return jpg thumbnail of a region of the frame."""
         frame = cv2.cvtColor(yuv_frame, cv2.COLOR_YUV2BGR_I420)
@@ -325,18 +808,25 @@ class EmbeddingMaintainer(threading.Thread):
         )
 
         if event.has_snapshot and source == "snapshot":
-            with open(
-                os.path.join(CLIPS_DIR, f"{event.camera}-{event.id}.jpg"),
-                "rb",
-            ) as image_file:
+            snapshot_file = os.path.join(CLIPS_DIR, f"{event.camera}-{event.id}.jpg")
+
+            if not os.path.isfile(snapshot_file):
+                logger.error(
+                    f"Cannot regenerate description for {event.id}, snapshot file not found: {snapshot_file}"
+                )
+                return
+
+            with open(snapshot_file, "rb") as image_file:
                 snapshot_image = image_file.read()
                 img = cv2.imdecode(
                     np.frombuffer(snapshot_image, dtype=np.int8), cv2.IMREAD_COLOR
                 )
 
                 # crop snapshot based on region before sending off to genai
+                # provide full image if region doesn't exist (manual events)
+                region = event.data.get("region", [0, 0, 1, 1])
                 height, width = img.shape[:2]
-                x1_rel, y1_rel, width_rel, height_rel = event.data["region"]
+                x1_rel, y1_rel, width_rel, height_rel = region
 
                 x1, y1 = int(x1_rel * width), int(y1_rel * height)
                 cropped_image = img[
@@ -350,7 +840,7 @@ class EmbeddingMaintainer(threading.Thread):
             [snapshot_image]
             if event.has_snapshot and source == "snapshot"
             else (
-                [thumbnail for data in self.tracked_events[event_id]]
+                [data["thumbnail"] for data in self.tracked_events[event_id]]
                 if len(self.tracked_events.get(event_id, [])) > 0
                 else [thumbnail]
             )

frigate/events/cleanup.py

@@ -4,7 +4,6 @@ import datetime
 import logging
 import os
 import threading
-from enum import Enum
 from multiprocessing.synchronize import Event as MpEvent
 from pathlib import Path
 
@@ -16,11 +15,6 @@ from frigate.models import Event, Timeline
 logger = logging.getLogger(__name__)
 
 
-class EventCleanupType(str, Enum):
-    clips = "clips"
-    snapshots = "snapshots"
-
-
 CHUNK_SIZE = 50
 
 
@@ -67,19 +61,11 @@ class EventCleanup(threading.Thread):
 
         return self.camera_labels[camera]["labels"]
 
-    def expire(self, media_type: EventCleanupType) -> list[str]:
+    def expire_snapshots(self) -> list[str]:
         ## Expire events from unlisted cameras based on the global config
-        if media_type == EventCleanupType.clips:
-            expire_days = max(
-                self.config.record.alerts.retain.days,
-                self.config.record.detections.retain.days,
-            )
-            file_extension = None  # mp4 clips are no longer stored in /clips
-            update_params = {"has_clip": False}
-        else:
-            retain_config = self.config.snapshots.retain
-            file_extension = "jpg"
-            update_params = {"has_snapshot": False}
+        retain_config = self.config.snapshots.retain
+        file_extension = "jpg"
+        update_params = {"has_snapshot": False}
 
         distinct_labels = self.get_removed_camera_labels()
 
@@ -87,10 +73,7 @@ class EventCleanup(threading.Thread):
         # loop over object types in db
         for event in distinct_labels:
             # get expiration time for this label
-            if media_type == EventCleanupType.snapshots:
-                expire_days = retain_config.objects.get(
-                    event.label, retain_config.default
-                )
+            expire_days = retain_config.objects.get(event.label, retain_config.default)
 
             expire_after = (
                 datetime.datetime.now() - datetime.timedelta(days=expire_days)
@@ -162,13 +145,7 @@ class EventCleanup(threading.Thread):
 
         ## Expire events from cameras based on the camera config
         for name, camera in self.config.cameras.items():
-            if media_type == EventCleanupType.clips:
-                expire_days = max(
-                    camera.record.alerts.retain.days,
-                    camera.record.detections.retain.days,
-                )
-            else:
-                retain_config = camera.snapshots.retain
+            retain_config = camera.snapshots.retain
 
             # get distinct objects in database for this camera
             distinct_labels = self.get_camera_labels(name)
@@ -176,10 +153,9 @@ class EventCleanup(threading.Thread):
             # loop over object types in db
             for event in distinct_labels:
                 # get expiration time for this label
-                if media_type == EventCleanupType.snapshots:
-                    expire_days = retain_config.objects.get(
-                        event.label, retain_config.default
-                    )
+                expire_days = retain_config.objects.get(
+                    event.label, retain_config.default
+                )
 
                 expire_after = (
                     datetime.datetime.now() - datetime.timedelta(days=expire_days)
@@ -206,19 +182,144 @@ class EventCleanup(threading.Thread):
                 for event in expired_events:
                     events_to_update.append(event.id)
 
-                    if media_type == EventCleanupType.snapshots:
-                        try:
-                            media_name = f"{event.camera}-{event.id}"
-                            media_path = Path(
-                                f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}"
-                            )
-                            media_path.unlink(missing_ok=True)
-                            media_path = Path(
-                                f"{os.path.join(CLIPS_DIR, media_name)}-clean.png"
-                            )
-                            media_path.unlink(missing_ok=True)
-                        except OSError as e:
-                            logger.warning(f"Unable to delete event images: {e}")
+                    try:
+                        media_name = f"{event.camera}-{event.id}"
+                        media_path = Path(
+                            f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}"
+                        )
+                        media_path.unlink(missing_ok=True)
+                        media_path = Path(
+                            f"{os.path.join(CLIPS_DIR, media_name)}-clean.png"
+                        )
+                        media_path.unlink(missing_ok=True)
+                    except OSError as e:
+                        logger.warning(f"Unable to delete event images: {e}")
+
+        # update the clips attribute for the db entry
+        for i in range(0, len(events_to_update), CHUNK_SIZE):
+            batch = events_to_update[i : i + CHUNK_SIZE]
+            logger.debug(f"Updating {update_params} for {len(batch)} events")
+            Event.update(update_params).where(Event.id << batch).execute()
+
+        return events_to_update
+
+    def expire_clips(self) -> list[str]:
+        ## Expire events from unlisted cameras based on the global config
+        expire_days = max(
+            self.config.record.alerts.retain.days,
+            self.config.record.detections.retain.days,
+        )
+        file_extension = None  # mp4 clips are no longer stored in /clips
+        update_params = {"has_clip": False}
+
+        # get expiration time for this label
+
+        expire_after = (
+            datetime.datetime.now() - datetime.timedelta(days=expire_days)
+        ).timestamp()
+        # grab all events after specific time
+        expired_events: list[Event] = (
+            Event.select(
+                Event.id,
+                Event.camera,
+            )
+            .where(
+                Event.camera.not_in(self.camera_keys),
+                Event.start_time < expire_after,
+                Event.retain_indefinitely == False,
+            )
+            .namedtuples()
+            .iterator()
+        )
+        logger.debug(f"{len(list(expired_events))} events can be expired")
+        # delete the media from disk
+        for expired in expired_events:
+            media_name = f"{expired.camera}-{expired.id}"
+            media_path = Path(f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}")
+
+            try:
+                media_path.unlink(missing_ok=True)
+                if file_extension == "jpg":
+                    media_path = Path(
+                        f"{os.path.join(CLIPS_DIR, media_name)}-clean.png"
+                    )
+                    media_path.unlink(missing_ok=True)
+            except OSError as e:
+                logger.warning(f"Unable to delete event images: {e}")
+
+        # update the clips attribute for the db entry
+        query = Event.select(Event.id).where(
+            Event.camera.not_in(self.camera_keys),
+            Event.start_time < expire_after,
+            Event.retain_indefinitely == False,
+        )
+
+        events_to_update = []
+
+        for event in query.iterator():
+            events_to_update.append(event)
+
+            if len(events_to_update) >= CHUNK_SIZE:
+                logger.debug(
+                    f"Updating {update_params} for {len(events_to_update)} events"
+                )
+                Event.update(update_params).where(
+                    Event.id << events_to_update
+                ).execute()
+                events_to_update = []
+
+        # Update any remaining events
+        if events_to_update:
+            logger.debug(
+                f"Updating clips/snapshots attribute for {len(events_to_update)} events"
+            )
+            Event.update(update_params).where(Event.id << events_to_update).execute()
+
+        events_to_update = []
+        now = datetime.datetime.now()
+
+        ## Expire events from cameras based on the camera config
+        for name, camera in self.config.cameras.items():
+            expire_days = max(
+                camera.record.alerts.retain.days,
+                camera.record.detections.retain.days,
+            )
+            alert_expire_date = (
+                now - datetime.timedelta(days=camera.record.alerts.retain.days)
+            ).timestamp()
+            detection_expire_date = (
+                now - datetime.timedelta(days=camera.record.detections.retain.days)
+            ).timestamp()
+            # grab all events after specific time
+            expired_events = (
+                Event.select(
+                    Event.id,
+                    Event.camera,
+                )
+                .where(
+                    Event.camera == name,
+                    Event.retain_indefinitely == False,
+                    (
+                        (
+                            (Event.data["max_severity"] != "detection")
+                            | (Event.data["max_severity"].is_null())
+                        )
+                        & (Event.end_time < alert_expire_date)
+                    )
+                    | (
+                        (Event.data["max_severity"] == "detection")
+                        & (Event.end_time < detection_expire_date)
+                    ),
+                )
+                .namedtuples()
+                .iterator()
+            )
+
+            # delete the grabbed clips from disk
+            # only snapshots are stored in /clips
+            # so no need to delete mp4 files
+            for event in expired_events:
+                events_to_update.append(event.id)
 
         # update the clips attribute for the db entry
         for i in range(0, len(events_to_update), CHUNK_SIZE):
@@ -231,7 +332,7 @@ class EventCleanup(threading.Thread):
     def run(self) -> None:
         # only expire events every 5 minutes
         while not self.stop_event.wait(300):
-            events_with_expired_clips = self.expire(EventCleanupType.clips)
+            events_with_expired_clips = self.expire_clips()
 
             # delete timeline entries for events that have expired recordings
             # delete up to 100,000 at a time
@@ -242,7 +343,7 @@ class EventCleanup(threading.Thread):
                     Timeline.source_id << deleted_events_list[i : i + max_deletes]
                 ).execute()
 
-            self.expire(EventCleanupType.snapshots)
+            self.expire_snapshots()
 
             # drop events from db where has_clip and has_snapshot are false
             events = (

frigate/events/maintainer.py

@@ -82,18 +82,23 @@ class EventProcessor(threading.Thread):
             )
 
             if source_type == EventTypeEnum.tracked_object:
+                id = event_data["id"]
                 self.timeline_queue.put(
                     (
                         camera,
                         source_type,
                         event_type,
-                        self.events_in_process.get(event_data["id"]),
+                        self.events_in_process.get(id),
                         event_data,
                     )
                 )
 
-                if event_type == EventStateEnum.start:
-                    self.events_in_process[event_data["id"]] = event_data
+                # if this is the first message, just store it and continue, its not time to insert it in the db
+                if (
+                    event_type == EventStateEnum.start
+                    or id not in self.events_in_process
+                ):
+                    self.events_in_process[id] = event_data
                     continue
 
                 self.handle_object_detection(event_type, camera, event_data)
@@ -123,10 +128,6 @@ class EventProcessor(threading.Thread):
         """handle tracked object event updates."""
         updated_db = False
 
-        # if this is the first message, just store it and continue, its not time to insert it in the db
-        if event_type == EventStateEnum.start:
-            self.events_in_process[event_data["id"]] = event_data
-
         if should_update_db(self.events_in_process[event_data["id"]], event_data):
             updated_db = True
             camera_config = self.config.cameras[camera]
@@ -210,6 +211,7 @@ class EventProcessor(threading.Thread):
                     "top_score": event_data["top_score"],
                     "attributes": attributes,
                     "type": "object",
+                    "max_severity": event_data.get("max_severity"),
                 },
             }
 

frigate/genai/ollama.py

@@ -38,6 +38,11 @@ class OllamaClient(GenAIClient):
 
     def _send(self, prompt: str, images: list[bytes]) -> Optional[str]:
         """Submit a request to Ollama"""
+        if self.provider is None:
+            logger.warning(
+                "Ollama provider has not been initialized, a description will not be generated. Check your Ollama configuration."
+            )
+            return None
         try:
             result = self.provider.generate(
                 self.genai_config.model,

frigate/log.py

@@ -18,12 +18,19 @@ LOG_HANDLER.setFormatter(
     )
 )
 
+# filter out norfair warning
 LOG_HANDLER.addFilter(
     lambda record: not record.getMessage().startswith(
         "You are using a scalar distance function"
     )
 )
 
+# filter out tflite logging
+LOG_HANDLER.addFilter(
+    lambda record: "Created TensorFlow Lite XNNPACK delegate for CPU."
+    not in record.getMessage()
+)
+
 log_listener: Optional[QueueListener] = None
 
 

frigate/object_processing.py

@@ -702,30 +702,7 @@ class TrackedObjectProcessor(threading.Thread):
             return False
 
         # If the object is not considered an alert or detection
-        review_config = self.config.cameras[camera].review
-        if not (
-            (
-                obj.obj_data["label"] in review_config.alerts.labels
-                and (
-                    not review_config.alerts.required_zones
-                    or set(obj.entered_zones) & set(review_config.alerts.required_zones)
-                )
-            )
-            or (
-                (
-                    not review_config.detections.labels
-                    or obj.obj_data["label"] in review_config.detections.labels
-                )
-                and (
-                    not review_config.detections.required_zones
-                    or set(obj.entered_zones)
-                    & set(review_config.detections.required_zones)
-                )
-            )
-        ):
-            logger.debug(
-                f"Not creating clip for {obj.obj_data['id']} because it did not qualify as an alert or detection"
-            )
+        if obj.max_severity is None:
             return False
 
         return True

frigate/ptz/autotrack.py

@@ -2,7 +2,6 @@
 
 import copy
 import logging
-import os
 import queue
 import threading
 import time
@@ -29,11 +28,11 @@ from frigate.const import (
     AUTOTRACKING_ZOOM_EDGE_THRESHOLD,
     AUTOTRACKING_ZOOM_IN_HYSTERESIS,
     AUTOTRACKING_ZOOM_OUT_HYSTERESIS,
-    CONFIG_DIR,
 )
 from frigate.ptz.onvif import OnvifController
 from frigate.track.tracked_object import TrackedObject
 from frigate.util.builtin import update_yaml_file
+from frigate.util.config import find_config_file
 from frigate.util.image import SharedMemoryFrameManager, intersection_over_union
 
 logger = logging.getLogger(__name__)
@@ -328,13 +327,7 @@ class PtzAutoTracker:
         self.autotracker_init[camera] = True
 
     def _write_config(self, camera):
-        config_file = os.environ.get("CONFIG_FILE", f"{CONFIG_DIR}/config.yml")
-
-        # Check if we can use .yaml instead of .yml
-        config_file_yaml = config_file.replace(".yml", ".yaml")
-
-        if os.path.isfile(config_file_yaml):
-            config_file = config_file_yaml
+        config_file = find_config_file()
 
         logger.debug(
             f"{camera}: Writing new config with autotracker motion coefficients: {self.config.cameras[camera].onvif.autotracking.movement_weights}"

frigate/ptz/onvif.py

@@ -6,6 +6,7 @@ from importlib.util import find_spec
 from pathlib import Path
 
 import numpy
+import requests
 from onvif import ONVIFCamera, ONVIFError
 from zeep.exceptions import Fault, TransportError
 from zeep.transports import Transport
@@ -48,7 +49,11 @@ class OnvifController:
 
             if cam.onvif.host:
                 try:
-                    transport = Transport(timeout=10, operation_timeout=10)
+                    session = requests.Session()
+                    session.verify = not cam.onvif.tls_insecure
+                    transport = Transport(
+                        timeout=10, operation_timeout=10, session=session
+                    )
                     self.cams[cam_name] = {
                         "onvif": ONVIFCamera(
                             cam.onvif.host,
@@ -558,22 +563,26 @@ class OnvifController:
             if not self._init_onvif(camera_name):
                 return
 
-        if command == OnvifCommandEnum.init:
-            # already init
-            return
-        elif command == OnvifCommandEnum.stop:
-            self._stop(camera_name)
-        elif command == OnvifCommandEnum.preset:
-            self._move_to_preset(camera_name, param)
-        elif command == OnvifCommandEnum.move_relative:
-            _, pan, tilt = param.split("_")
-            self._move_relative(camera_name, float(pan), float(tilt), 0, 1)
-        elif (
-            command == OnvifCommandEnum.zoom_in or command == OnvifCommandEnum.zoom_out
-        ):
-            self._zoom(camera_name, command)
-        else:
-            self._move(camera_name, command)
+        try:
+            if command == OnvifCommandEnum.init:
+                # already init
+                return
+            elif command == OnvifCommandEnum.stop:
+                self._stop(camera_name)
+            elif command == OnvifCommandEnum.preset:
+                self._move_to_preset(camera_name, param)
+            elif command == OnvifCommandEnum.move_relative:
+                _, pan, tilt = param.split("_")
+                self._move_relative(camera_name, float(pan), float(tilt), 0, 1)
+            elif (
+                command == OnvifCommandEnum.zoom_in
+                or command == OnvifCommandEnum.zoom_out
+            ):
+                self._zoom(camera_name, command)
+            else:
+                self._move(camera_name, command)
+        except ONVIFError as e:
+            logger.error(f"Unable to handle onvif command: {e}")
 
     def get_camera_info(self, camera_name: str) -> dict[str, any]:
         if camera_name not in self.cams.keys():

frigate/record/maintainer.py

@@ -29,6 +29,7 @@ from frigate.const import (
     RECORD_DIR,
 )
 from frigate.models import Recordings, ReviewSegment
+from frigate.review.types import SeverityEnum
 from frigate.util.services import get_video_properties
 
 logger = logging.getLogger(__name__)
@@ -194,6 +195,7 @@ class RecordingMaintainer(threading.Thread):
                 ReviewSegment.select(
                     ReviewSegment.start_time,
                     ReviewSegment.end_time,
+                    ReviewSegment.severity,
                     ReviewSegment.data,
                 )
                 .where(
@@ -219,11 +221,15 @@ class RecordingMaintainer(threading.Thread):
             [r for r in recordings_to_insert if r is not None],
         )
 
+    def drop_segment(self, cache_path: str) -> None:
+        Path(cache_path).unlink(missing_ok=True)
+        self.end_time_cache.pop(cache_path, None)
+
     async def validate_and_move_segment(
         self, camera: str, reviews: list[ReviewSegment], recording: dict[str, any]
     ) -> None:
-        cache_path = recording["cache_path"]
-        start_time = recording["start_time"]
+        cache_path: str = recording["cache_path"]
+        start_time: datetime.datetime = recording["start_time"]
         record_config = self.config.cameras[camera].record
 
         # Just delete files if recordings are turned off
@@ -231,8 +237,7 @@ class RecordingMaintainer(threading.Thread):
             camera not in self.config.cameras
             or not self.config.cameras[camera].record.enabled
         ):
-            Path(cache_path).unlink(missing_ok=True)
-            self.end_time_cache.pop(cache_path, None)
+            self.drop_segment(cache_path)
             return
 
         if cache_path in self.end_time_cache:
@@ -260,24 +265,34 @@ class RecordingMaintainer(threading.Thread):
                 return
 
         # if cached file's start_time is earlier than the retain days for the camera
+        # meaning continuous recording is not enabled
         if start_time <= (
             datetime.datetime.now().astimezone(datetime.timezone.utc)
             - datetime.timedelta(days=self.config.cameras[camera].record.retain.days)
         ):
-            # if the cached segment overlaps with the events:
+            # if the cached segment overlaps with the review items:
             overlaps = False
             for review in reviews:
-                # if the event starts in the future, stop checking events
+                severity = SeverityEnum[review.severity]
+
+                # if the review item starts in the future, stop checking review items
                 # and remove this segment
-                if review.start_time > end_time.timestamp():
+                if (
+                    review.start_time - record_config.get_review_pre_capture(severity)
+                ) > end_time.timestamp():
                     overlaps = False
-                    Path(cache_path).unlink(missing_ok=True)
-                    self.end_time_cache.pop(cache_path, None)
                     break
 
-                # if the event is in progress or ends after the recording starts, keep it
-                # and stop looking at events
-                if review.end_time is None or review.end_time >= start_time.timestamp():
+                # if the review item is in progress or ends after the recording starts, keep it
+                # and stop looking at review items
+                if (
+                    review.end_time is None
+                    or (
+                        review.end_time
+                        + record_config.get_review_post_capture(severity)
+                    )
+                    >= start_time.timestamp()
+                ):
                     overlaps = True
                     break
 
@@ -296,7 +311,7 @@ class RecordingMaintainer(threading.Thread):
                     cache_path,
                     record_mode,
                 )
-            # if it doesn't overlap with an event, go ahead and drop the segment
+            # if it doesn't overlap with an review item, go ahead and drop the segment
             # if it ends more than the configured pre_capture for the camera
             else:
                 camera_info = self.object_recordings_info[camera]
@@ -307,9 +322,9 @@ class RecordingMaintainer(threading.Thread):
                     most_recently_processed_frame_time - record_config.event_pre_capture
                 ).astimezone(datetime.timezone.utc)
                 if end_time < retain_cutoff:
-                    Path(cache_path).unlink(missing_ok=True)
-                    self.end_time_cache.pop(cache_path, None)
+                    self.drop_segment(cache_path)
         # else retain days includes this segment
+        # meaning continuous recording is enabled
         else:
             # assume that empty means the relevant recording info has not been received yet
             camera_info = self.object_recordings_info[camera]
@@ -390,8 +405,7 @@ class RecordingMaintainer(threading.Thread):
 
         # check if the segment shouldn't be stored
         if segment_info.should_discard_segment(store_mode):
-            Path(cache_path).unlink(missing_ok=True)
-            self.end_time_cache.pop(cache_path, None)
+            self.drop_segment(cache_path)
             return
 
         # directory will be in utc due to start_time being in utc

frigate/review/maintainer.py

@@ -7,7 +7,6 @@ import random
 import string
 import sys
 import threading
-from enum import Enum
 from multiprocessing.synchronize import Event as MpEvent
 from pathlib import Path
 from typing import Optional
@@ -27,6 +26,7 @@ from frigate.const import (
 from frigate.events.external import ManualEventState
 from frigate.models import ReviewSegment
 from frigate.object_processing import TrackedObject
+from frigate.review.types import SeverityEnum
 from frigate.util.image import SharedMemoryFrameManager, calculate_16_9_crop
 
 logger = logging.getLogger(__name__)
@@ -39,11 +39,6 @@ THRESHOLD_ALERT_ACTIVITY = 120
 THRESHOLD_DETECTION_ACTIVITY = 30
 
 
-class SeverityEnum(str, Enum):
-    alert = "alert"
-    detection = "detection"
-
-
 class PendingReviewSegment:
     def __init__(
         self,

frigate/review/types.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class SeverityEnum(str, Enum):
+    alert = "alert"
+    detection = "detection"

frigate/stats/util.py

@@ -293,7 +293,7 @@ def stats_snapshot(
     for path in [RECORD_DIR, CLIPS_DIR, CACHE_DIR, "/dev/shm"]:
         try:
             storage_stats = shutil.disk_usage(path)
-        except FileNotFoundError:
+        except (FileNotFoundError, OSError):
             stats["service"]["storage"][path] = {}
             continue
 

frigate/storage.py

@@ -17,6 +17,8 @@ bandwidth_equation = Recordings.segment_size / (
     Recordings.end_time - Recordings.start_time
 )
 
+MAX_CALCULATED_BANDWIDTH = 10000  # 10Gb/hr
+
 
 class StorageMaintainer(threading.Thread):
     """Maintain frigates recording storage."""
@@ -52,6 +54,12 @@ class StorageMaintainer(threading.Thread):
                         * 3600,
                         2,
                     )
+
+                    if bandwidth > MAX_CALCULATED_BANDWIDTH:
+                        logger.warning(
+                            f"{camera} has a bandwidth of {bandwidth} MB/hr which exceeds the expected maximum. This typically indicates an issue with the cameras recordings."
+                        )
+                        bandwidth = MAX_CALCULATED_BANDWIDTH
                 except TypeError:
                     bandwidth = 0
 

frigate/test/http_api/base_http_test.py

@@ -9,8 +9,8 @@ from playhouse.sqliteq import SqliteQueueDatabase
 
 from frigate.api.fastapi_app import create_fastapi_app
 from frigate.config import FrigateConfig
-from frigate.models import Event, ReviewSegment
-from frigate.review.maintainer import SeverityEnum
+from frigate.models import Event, Recordings, ReviewSegment
+from frigate.review.types import SeverityEnum
 from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS
 
 
@@ -146,17 +146,35 @@ class BaseTestHttp(unittest.TestCase):
     def insert_mock_review_segment(
         self,
         id: str,
-        start_time: datetime.datetime = datetime.datetime.now().timestamp(),
-        end_time: datetime.datetime = datetime.datetime.now().timestamp() + 20,
+        start_time: float = datetime.datetime.now().timestamp(),
+        end_time: float = datetime.datetime.now().timestamp() + 20,
+        severity: SeverityEnum = SeverityEnum.alert,
+        has_been_reviewed: bool = False,
     ) -> Event:
-        """Inserts a basic event model with a given id."""
+        """Inserts a review segment model with a given id."""
         return ReviewSegment.insert(
             id=id,
             camera="front_door",
             start_time=start_time,
             end_time=end_time,
-            has_been_reviewed=False,
-            severity=SeverityEnum.alert,
+            has_been_reviewed=has_been_reviewed,
+            severity=severity,
             thumb_path=False,
             data={},
         ).execute()
+
+    def insert_mock_recording(
+        self,
+        id: str,
+        start_time: float = datetime.datetime.now().timestamp(),
+        end_time: float = datetime.datetime.now().timestamp() + 20,
+    ) -> Event:
+        """Inserts a recording model with a given id."""
+        return Recordings.insert(
+            id=id,
+            path=id,
+            camera="front_door",
+            start_time=start_time,
+            end_time=end_time,
+            duration=end_time - start_time,
+        ).execute()

frigate/test/http_api/test_http_review.py

@@ -1,76 +1,89 @@
-import datetime
+from datetime import datetime, timedelta
 
 from fastapi.testclient import TestClient
 
-from frigate.models import Event, ReviewSegment
+from frigate.models import Event, Recordings, ReviewSegment
+from frigate.review.types import SeverityEnum
 from frigate.test.http_api.base_http_test import BaseTestHttp
 
 
 class TestHttpReview(BaseTestHttp):
     def setUp(self):
-        super().setUp([Event, ReviewSegment])
+        super().setUp([Event, Recordings, ReviewSegment])
+        self.app = super().create_app()
+
+    def _get_reviews(self, ids: list[str]):
+        return list(
+            ReviewSegment.select(ReviewSegment.id)
+            .where(ReviewSegment.id.in_(ids))
+            .execute()
+        )
+
+    def _get_recordings(self, ids: list[str]):
+        return list(
+            Recordings.select(Recordings.id).where(Recordings.id.in_(ids)).execute()
+        )
+
+    ####################################################################################################################
+    ###################################  GET /review Endpoint   ########################################################
+    ####################################################################################################################
 
     # Does not return any data point since the end time (before parameter) is not passed and the review segment end_time is 2 seconds from now
     def test_get_review_no_filters_no_matches(self):
-        app = super().create_app()
-        now = datetime.datetime.now().timestamp()
+        now = datetime.now().timestamp()
 
-        with TestClient(app) as client:
+        with TestClient(self.app) as client:
             super().insert_mock_review_segment("123456.random", now, now + 2)
-            reviews_response = client.get("/review")
-            assert reviews_response.status_code == 200
-            reviews_in_response = reviews_response.json()
-            assert len(reviews_in_response) == 0
+            response = client.get("/review")
+            assert response.status_code == 200
+            response_json = response.json()
+            assert len(response_json) == 0
 
     def test_get_review_no_filters(self):
-        app = super().create_app()
-        now = datetime.datetime.now().timestamp()
+        now = datetime.now().timestamp()
 
-        with TestClient(app) as client:
+        with TestClient(self.app) as client:
             super().insert_mock_review_segment("123456.random", now - 2, now - 1)
-            reviews_response = client.get("/review")
-            assert reviews_response.status_code == 200
-            reviews_in_response = reviews_response.json()
-            assert len(reviews_in_response) == 1
+            response = client.get("/review")
+            assert response.status_code == 200
+            response_json = response.json()
+            assert len(response_json) == 1
 
     def test_get_review_with_time_filter_no_matches(self):
-        app = super().create_app()
-        now = datetime.datetime.now().timestamp()
+        now = datetime.now().timestamp()
 
-        with TestClient(app) as client:
+        with TestClient(self.app) as client:
             id = "123456.random"
             super().insert_mock_review_segment(id, now, now + 2)
             params = {
                 "after": now,
                 "before": now + 3,
             }
-            reviews_response = client.get("/review", params=params)
-            assert reviews_response.status_code == 200
-            reviews_in_response = reviews_response.json()
-            assert len(reviews_in_response) == 0
+            response = client.get("/review", params=params)
+            assert response.status_code == 200
+            response_json = response.json()
+            assert len(response_json) == 0
 
     def test_get_review_with_time_filter(self):
-        app = super().create_app()
-        now = datetime.datetime.now().timestamp()
+        now = datetime.now().timestamp()
 
-        with TestClient(app) as client:
+        with TestClient(self.app) as client:
             id = "123456.random"
             super().insert_mock_review_segment(id, now, now + 2)
             params = {
                 "after": now - 1,
                 "before": now + 3,
             }
-            reviews_response = client.get("/review", params=params)
-            assert reviews_response.status_code == 200
-            reviews_in_response = reviews_response.json()
-            assert len(reviews_in_response) == 1
-            assert reviews_in_response[0]["id"] == id
+            response = client.get("/review", params=params)
+            assert response.status_code == 200
+            response_json = response.json()
+            assert len(response_json) == 1
+            assert response_json[0]["id"] == id
 
     def test_get_review_with_limit_filter(self):
-        app = super().create_app()
-        now = datetime.datetime.now().timestamp()
+        now = datetime.now().timestamp()
 
-        with TestClient(app) as client:
+        with TestClient(self.app) as client:
             id = "123456.random"
             id2 = "654321.random"
             super().insert_mock_review_segment(id, now, now + 2)
@@ -80,17 +93,49 @@ class TestHttpReview(BaseTestHttp):
                 "after": now,
                 "before": now + 3,
             }
-            reviews_response = client.get("/review", params=params)
-            assert reviews_response.status_code == 200
-            reviews_in_response = reviews_response.json()
-            assert len(reviews_in_response) == 1
-            assert reviews_in_response[0]["id"] == id2
+            response = client.get("/review", params=params)
+            assert response.status_code == 200
+            response_json = response.json()
+            assert len(response_json) == 1
+            assert response_json[0]["id"] == id2
+
+    def test_get_review_with_severity_filters_no_matches(self):
+        now = datetime.now().timestamp()
+
+        with TestClient(self.app) as client:
+            id = "123456.random"
+            super().insert_mock_review_segment(id, now, now + 2, SeverityEnum.detection)
+            params = {
+                "severity": "detection",
+                "after": now - 1,
+                "before": now + 3,
+            }
+            response = client.get("/review", params=params)
+            assert response.status_code == 200
+            response_json = response.json()
+            assert len(response_json) == 1
+            assert response_json[0]["id"] == id
+
+    def test_get_review_with_severity_filters(self):
+        now = datetime.now().timestamp()
+
+        with TestClient(self.app) as client:
+            id = "123456.random"
+            super().insert_mock_review_segment(id, now, now + 2, SeverityEnum.detection)
+            params = {
+                "severity": "alert",
+                "after": now - 1,
+                "before": now + 3,
+            }
+            response = client.get("/review", params=params)
+            assert response.status_code == 200
+            response_json = response.json()
+            assert len(response_json) == 0
 
     def test_get_review_with_all_filters(self):
-        app = super().create_app()
-        now = datetime.datetime.now().timestamp()
+        now = datetime.now().timestamp()
 
-        with TestClient(app) as client:
+        with TestClient(self.app) as client:
             id = "123456.random"
             super().insert_mock_review_segment(id, now, now + 2)
             params = {
@@ -103,8 +148,424 @@ class TestHttpReview(BaseTestHttp):
                 "after": now - 1,
                 "before": now + 3,
             }
-            reviews_response = client.get("/review", params=params)
-            assert reviews_response.status_code == 200
-            reviews_in_response = reviews_response.json()
-            assert len(reviews_in_response) == 1
-            assert reviews_in_response[0]["id"] == id
+            response = client.get("/review", params=params)
+            assert response.status_code == 200
+            response_json = response.json()
+            assert len(response_json) == 1
+            assert response_json[0]["id"] == id
+
+    ####################################################################################################################
+    ###################################  GET /review/summary Endpoint   #################################################
+    ####################################################################################################################
+    def test_get_review_summary_all_filters(self):
+        with TestClient(self.app) as client:
+            super().insert_mock_review_segment("123456.random")
+            params = {
+                "cameras": "front_door",
+                "labels": "all",
+                "zones": "all",
+                "timezone": "utc",
+            }
+            response = client.get("/review/summary", params=params)
+            assert response.status_code == 200
+            response_json = response.json()
+            # e.g. '2024-11-24'
+            today_formatted = datetime.today().strftime("%Y-%m-%d")
+            expected_response = {
+                "last24Hours": {
+                    "reviewed_alert": 0,
+                    "reviewed_detection": 0,
+                    "total_alert": 1,
+                    "total_detection": 0,
+                },
+                today_formatted: {
+                    "day": today_formatted,
+                    "reviewed_alert": 0,
+                    "reviewed_detection": 0,
+                    "total_alert": 1,
+                    "total_detection": 0,
+                },
+            }
+            self.assertEqual(response_json, expected_response)
+
+    def test_get_review_summary_no_filters(self):
+        with TestClient(self.app) as client:
+            super().insert_mock_review_segment("123456.random")
+            response = client.get("/review/summary")
+            assert response.status_code == 200
+            response_json = response.json()
+            # e.g. '2024-11-24'
+            today_formatted = datetime.today().strftime("%Y-%m-%d")
+            expected_response = {
+                "last24Hours": {
+                    "reviewed_alert": 0,
+                    "reviewed_detection": 0,
+                    "total_alert": 1,
+                    "total_detection": 0,
+                },
+                today_formatted: {
+                    "day": today_formatted,
+                    "reviewed_alert": 0,
+                    "reviewed_detection": 0,
+                    "total_alert": 1,
+                    "total_detection": 0,
+                },
+            }
+            self.assertEqual(response_json, expected_response)
+
+    def test_get_review_summary_multiple_days(self):
+        now = datetime.now()
+        five_days_ago = datetime.today() - timedelta(days=5)
+
+        with TestClient(self.app) as client:
+            super().insert_mock_review_segment(
+                "123456.random", now.timestamp() - 2, now.timestamp() - 1
+            )
+            super().insert_mock_review_segment(
+                "654321.random",
+                five_days_ago.timestamp(),
+                five_days_ago.timestamp() + 1,
+            )
+            response = client.get("/review/summary")
+            assert response.status_code == 200
+            response_json = response.json()
+            # e.g. '2024-11-24'
+            today_formatted = now.strftime("%Y-%m-%d")
+            # e.g. '2024-11-19'
+            five_days_ago_formatted = five_days_ago.strftime("%Y-%m-%d")
+            expected_response = {
+                "last24Hours": {
+                    "reviewed_alert": 0,
+                    "reviewed_detection": 0,
+                    "total_alert": 1,
+                    "total_detection": 0,
+                },
+                today_formatted: {
+                    "day": today_formatted,
+                    "reviewed_alert": 0,
+                    "reviewed_detection": 0,
+                    "total_alert": 1,
+                    "total_detection": 0,
+                },
+                five_days_ago_formatted: {
+                    "day": five_days_ago_formatted,
+                    "reviewed_alert": 0,
+                    "reviewed_detection": 0,
+                    "total_alert": 1,
+                    "total_detection": 0,
+                },
+            }
+            self.assertEqual(response_json, expected_response)
+
+    def test_get_review_summary_multiple_days_edge_cases(self):
+        now = datetime.now()
+        five_days_ago = datetime.today() - timedelta(days=5)
+        twenty_days_ago = datetime.today() - timedelta(days=20)
+        one_month_ago = datetime.today() - timedelta(days=30)
+        one_month_ago_ts = one_month_ago.timestamp()
+
+        with TestClient(self.app) as client:
+            super().insert_mock_review_segment("123456.random", now.timestamp())
+            super().insert_mock_review_segment(
+                "123457.random", five_days_ago.timestamp()
+            )
+            super().insert_mock_review_segment(
+                "123458.random",
+                twenty_days_ago.timestamp(),
+                None,
+                SeverityEnum.detection,
+            )
+            # One month ago plus 5 seconds fits within the condition (review.start_time > month_ago). Assuming that the endpoint does not take more than 5 seconds to be invoked
+            super().insert_mock_review_segment(
+                "123459.random",
+                one_month_ago_ts + 5,
+                None,
+                SeverityEnum.detection,
+            )
+            # This won't appear in the output since it's not within last month start_time clause (review.start_time > month_ago)
+            super().insert_mock_review_segment("123450.random", one_month_ago_ts)
+            response = client.get("/review/summary")
+            assert response.status_code == 200
+            response_json = response.json()
+            # e.g. '2024-11-24'
+            today_formatted = now.strftime("%Y-%m-%d")
+            # e.g. '2024-11-19'
+            five_days_ago_formatted = five_days_ago.strftime("%Y-%m-%d")
+            # e.g. '2024-11-04'
+            twenty_days_ago_formatted = twenty_days_ago.strftime("%Y-%m-%d")
+            # e.g. '2024-10-24'
+            one_month_ago_formatted = one_month_ago.strftime("%Y-%m-%d")
+            expected_response = {
+                "last24Hours": {
+                    "reviewed_alert": 0,
+                    "reviewed_detection": 0,
+                    "total_alert": 1,
+                    "total_detection": 0,
+                },
+                today_formatted: {
+                    "day": today_formatted,
+                    "reviewed_alert": 0,
+                    "reviewed_detection": 0,
+                    "total_alert": 1,
+                    "total_detection": 0,
+                },
+                five_days_ago_formatted: {
+                    "day": five_days_ago_formatted,
+                    "reviewed_alert": 0,
+                    "reviewed_detection": 0,
+                    "total_alert": 1,
+                    "total_detection": 0,
+                },
+                twenty_days_ago_formatted: {
+                    "day": twenty_days_ago_formatted,
+                    "reviewed_alert": 0,
+                    "reviewed_detection": 0,
+                    "total_alert": 0,
+                    "total_detection": 1,
+                },
+                one_month_ago_formatted: {
+                    "day": one_month_ago_formatted,
+                    "reviewed_alert": 0,
+                    "reviewed_detection": 0,
+                    "total_alert": 0,
+                    "total_detection": 1,
+                },
+            }
+            self.assertEqual(response_json, expected_response)
+
+    def test_get_review_summary_multiple_in_same_day(self):
+        now = datetime.now()
+        five_days_ago = datetime.today() - timedelta(days=5)
+
+        with TestClient(self.app) as client:
+            super().insert_mock_review_segment("123456.random", now.timestamp())
+            five_days_ago_ts = five_days_ago.timestamp()
+            for i in range(20):
+                super().insert_mock_review_segment(
+                    f"123456_{i}.random_alert",
+                    five_days_ago_ts,
+                    five_days_ago_ts,
+                    SeverityEnum.alert,
+                )
+            for i in range(15):
+                super().insert_mock_review_segment(
+                    f"123456_{i}.random_detection",
+                    five_days_ago_ts,
+                    five_days_ago_ts,
+                    SeverityEnum.detection,
+                )
+            response = client.get("/review/summary")
+            assert response.status_code == 200
+            response_json = response.json()
+            # e.g. '2024-11-24'
+            today_formatted = now.strftime("%Y-%m-%d")
+            # e.g. '2024-11-19'
+            five_days_ago_formatted = five_days_ago.strftime("%Y-%m-%d")
+            expected_response = {
+                "last24Hours": {
+                    "reviewed_alert": 0,
+                    "reviewed_detection": 0,
+                    "total_alert": 1,
+                    "total_detection": 0,
+                },
+                today_formatted: {
+                    "day": today_formatted,
+                    "reviewed_alert": 0,
+                    "reviewed_detection": 0,
+                    "total_alert": 1,
+                    "total_detection": 0,
+                },
+                five_days_ago_formatted: {
+                    "day": five_days_ago_formatted,
+                    "reviewed_alert": 0,
+                    "reviewed_detection": 0,
+                    "total_alert": 20,
+                    "total_detection": 15,
+                },
+            }
+            self.assertEqual(response_json, expected_response)
+
+    def test_get_review_summary_multiple_in_same_day_with_reviewed(self):
+        five_days_ago = datetime.today() - timedelta(days=5)
+
+        with TestClient(self.app) as client:
+            five_days_ago_ts = five_days_ago.timestamp()
+            for i in range(10):
+                super().insert_mock_review_segment(
+                    f"123456_{i}.random_alert_not_reviewed",
+                    five_days_ago_ts,
+                    five_days_ago_ts,
+                    SeverityEnum.alert,
+                    False,
+                )
+            for i in range(10):
+                super().insert_mock_review_segment(
+                    f"123456_{i}.random_alert_reviewed",
+                    five_days_ago_ts,
+                    five_days_ago_ts,
+                    SeverityEnum.alert,
+                    True,
+                )
+            for i in range(10):
+                super().insert_mock_review_segment(
+                    f"123456_{i}.random_detection_not_reviewed",
+                    five_days_ago_ts,
+                    five_days_ago_ts,
+                    SeverityEnum.detection,
+                    False,
+                )
+            for i in range(5):
+                super().insert_mock_review_segment(
+                    f"123456_{i}.random_detection_reviewed",
+                    five_days_ago_ts,
+                    five_days_ago_ts,
+                    SeverityEnum.detection,
+                    True,
+                )
+            response = client.get("/review/summary")
+            assert response.status_code == 200
+            response_json = response.json()
+            # e.g. '2024-11-19'
+            five_days_ago_formatted = five_days_ago.strftime("%Y-%m-%d")
+            expected_response = {
+                "last24Hours": {
+                    "reviewed_alert": None,
+                    "reviewed_detection": None,
+                    "total_alert": None,
+                    "total_detection": None,
+                },
+                five_days_ago_formatted: {
+                    "day": five_days_ago_formatted,
+                    "reviewed_alert": 10,
+                    "reviewed_detection": 5,
+                    "total_alert": 20,
+                    "total_detection": 15,
+                },
+            }
+            self.assertEqual(response_json, expected_response)
+
+    ####################################################################################################################
+    ###################################  POST reviews/viewed Endpoint   ################################################
+    ####################################################################################################################
+    def test_post_reviews_viewed_no_body(self):
+        with TestClient(self.app) as client:
+            super().insert_mock_review_segment("123456.random")
+            response = client.post("/reviews/viewed")
+            # Missing ids
+            assert response.status_code == 422
+
+    def test_post_reviews_viewed_no_body_ids(self):
+        with TestClient(self.app) as client:
+            super().insert_mock_review_segment("123456.random")
+            body = {"ids": [""]}
+            response = client.post("/reviews/viewed", json=body)
+            # Missing ids
+            assert response.status_code == 422
+
+    def test_post_reviews_viewed_non_existent_id(self):
+        with TestClient(self.app) as client:
+            id = "123456.random"
+            super().insert_mock_review_segment(id)
+            body = {"ids": ["1"]}
+            response = client.post("/reviews/viewed", json=body)
+            assert response.status_code == 200
+            response = response.json()
+            assert response["success"] == True
+            assert response["message"] == "Reviewed multiple items"
+            # Verify that in DB the review segment was not changed
+            review_segment_in_db = (
+                ReviewSegment.select(ReviewSegment.has_been_reviewed)
+                .where(ReviewSegment.id == id)
+                .get()
+            )
+            assert review_segment_in_db.has_been_reviewed == False
+
+    def test_post_reviews_viewed(self):
+        with TestClient(self.app) as client:
+            id = "123456.random"
+            super().insert_mock_review_segment(id)
+            body = {"ids": [id]}
+            response = client.post("/reviews/viewed", json=body)
+            assert response.status_code == 200
+            response = response.json()
+            assert response["success"] == True
+            assert response["message"] == "Reviewed multiple items"
+            # Verify that in DB the review segment was changed
+            review_segment_in_db = (
+                ReviewSegment.select(ReviewSegment.has_been_reviewed)
+                .where(ReviewSegment.id == id)
+                .get()
+            )
+            assert review_segment_in_db.has_been_reviewed == True
+
+    ####################################################################################################################
+    ###################################  POST reviews/delete Endpoint   ################################################
+    ####################################################################################################################
+    def test_post_reviews_delete_no_body(self):
+        with TestClient(self.app) as client:
+            super().insert_mock_review_segment("123456.random")
+            response = client.post("/reviews/delete")
+            # Missing ids
+            assert response.status_code == 422
+
+    def test_post_reviews_delete_no_body_ids(self):
+        with TestClient(self.app) as client:
+            super().insert_mock_review_segment("123456.random")
+            body = {"ids": [""]}
+            response = client.post("/reviews/delete", json=body)
+            # Missing ids
+            assert response.status_code == 422
+
+    def test_post_reviews_delete_non_existent_id(self):
+        with TestClient(self.app) as client:
+            id = "123456.random"
+            super().insert_mock_review_segment(id)
+            body = {"ids": ["1"]}
+            response = client.post("/reviews/delete", json=body)
+            assert response.status_code == 200
+            response_json = response.json()
+            assert response_json["success"] == True
+            assert response_json["message"] == "Deleted review items."
+            # Verify that in DB the review segment was not deleted
+            review_ids_in_db_after = self._get_reviews([id])
+            assert len(review_ids_in_db_after) == 1
+            assert review_ids_in_db_after[0].id == id
+
+    def test_post_reviews_delete(self):
+        with TestClient(self.app) as client:
+            id = "123456.random"
+            super().insert_mock_review_segment(id)
+            body = {"ids": [id]}
+            response = client.post("/reviews/delete", json=body)
+            assert response.status_code == 200
+            response_json = response.json()
+            assert response_json["success"] == True
+            assert response_json["message"] == "Deleted review items."
+            # Verify that in DB the review segment was deleted
+            review_ids_in_db_after = self._get_reviews([id])
+            assert len(review_ids_in_db_after) == 0
+
+    def test_post_reviews_delete_many(self):
+        with TestClient(self.app) as client:
+            ids = ["123456.random", "654321.random"]
+            for id in ids:
+                super().insert_mock_review_segment(id)
+                super().insert_mock_recording(id)
+
+            review_ids_in_db_before = self._get_reviews(ids)
+            recordings_ids_in_db_before = self._get_recordings(ids)
+            assert len(review_ids_in_db_before) == 2
+            assert len(recordings_ids_in_db_before) == 2
+
+            body = {"ids": ids}
+            response = client.post("/reviews/delete", json=body)
+            assert response.status_code == 200
+            response_json = response.json()
+            assert response_json["success"] == True
+            assert response_json["message"] == "Deleted review items."
+
+            # Verify that in DB all review segments and recordings that were passed were deleted
+            review_ids_in_db_after = self._get_reviews(ids)
+            recording_ids_in_db_after = self._get_recordings(ids)
+            assert len(review_ids_in_db_after) == 0
+            assert len(recording_ids_in_db_after) == 0

frigate/test/test_http.py

@@ -168,7 +168,7 @@ class TestHttp(unittest.TestCase):
 
         assert event
         assert event["id"] == id
-        assert event == model_to_dict(Event.get(Event.id == id))
+        assert event["id"] == model_to_dict(Event.get(Event.id == id))["id"]
 
     def test_get_bad_event(self):
         app = create_fastapi_app(

frigate/track/tracked_object.py

@@ -13,6 +13,7 @@ from frigate.config import (
     CameraConfig,
     ModelConfig,
 )
+from frigate.review.types import SeverityEnum
 from frigate.util.image import (
     area,
     calculate_region,
@@ -59,6 +60,27 @@ class TrackedObject:
         self.pending_loitering = False
         self.previous = self.to_dict()
 
+    @property
+    def max_severity(self) -> Optional[str]:
+        review_config = self.camera_config.review
+
+        if self.obj_data["label"] in review_config.alerts.labels and (
+            not review_config.alerts.required_zones
+            or set(self.entered_zones) & set(review_config.alerts.required_zones)
+        ):
+            return SeverityEnum.alert
+
+        if (
+            not review_config.detections.labels
+            or self.obj_data["label"] in review_config.detections.labels
+        ) and (
+            not review_config.detections.required_zones
+            or set(self.entered_zones) & set(review_config.detections.required_zones)
+        ):
+            return SeverityEnum.detection
+
+        return None
+
     def _is_false_positive(self):
         # once a true positive, always a true positive
         if not self.false_positive:
@@ -232,6 +254,7 @@ class TrackedObject:
             "attributes": self.attributes,
             "current_attributes": self.obj_data["attributes"],
             "pending_loitering": self.pending_loitering,
+            "max_severity": self.max_severity,
         }
 
         if include_thumbnail:

frigate/util/config.py

@@ -14,6 +14,16 @@ from frigate.util.services import get_video_properties
 logger = logging.getLogger(__name__)
 
 CURRENT_CONFIG_VERSION = "0.15-0"
+DEFAULT_CONFIG_FILE = "/config/config.yml"
+
+
+def find_config_file() -> str:
+    config_path = os.environ.get("CONFIG_FILE", DEFAULT_CONFIG_FILE)
+
+    if not os.path.isfile(config_path):
+        config_path = config_path.replace("yml", "yaml")
+
+    return config_path
 
 
 def migrate_frigate_config(config_file: str):

frigate/util/downloader.py

@@ -101,7 +101,7 @@ class ModelDownloader:
         self.download_complete.set()
 
     @staticmethod
-    def download_from_url(url: str, save_path: str, silent: bool = False):
+    def download_from_url(url: str, save_path: str, silent: bool = False) -> Path:
         temporary_filename = Path(save_path).with_name(
             os.path.basename(save_path) + ".part"
         )
@@ -125,6 +125,8 @@ class ModelDownloader:
         if not silent:
             logger.info(f"Downloading complete: {url}")
 
+        return Path(save_path)
+
     @staticmethod
     def mark_files_state(
         requestor: InterProcessRequestor,

frigate/util/image.py

@@ -219,6 +219,8 @@ def draw_box_with_label(
     text_width = size[0][0]
     text_height = size[0][1]
     line_height = text_height + size[1]
+    # get frame height
+    frame_height = frame.shape[0]
     # set the text start position
     if position == "ul":
         text_offset_x = x_min
@@ -228,18 +230,23 @@ def draw_box_with_label(
         text_offset_y = max(0, y_min - (line_height + 8))
     elif position == "bl":
         text_offset_x = x_min
-        text_offset_y = y_max
+        text_offset_y = min(frame_height - line_height, y_max)
     elif position == "br":
         text_offset_x = max(0, x_max - (text_width + 8))
-        text_offset_y = y_max
-
-    # Adjust position if it overlaps with the box
-    if position in {"ul", "ur"} and text_offset_y < y_min + thickness:
-        # Move the text below the box
-        text_offset_y = y_max
-    elif position in {"bl", "br"} and text_offset_y + line_height > y_max:
-        # Move the text above the box
-        text_offset_y = max(0, y_min - (line_height + 8))
+        text_offset_y = min(frame_height - line_height, y_max)
+    # Adjust position if it overlaps with the box or goes out of frame
+    if position in {"ul", "ur"}:
+        if text_offset_y < y_min + thickness:  # Label overlaps with the box
+            if y_min - (line_height + 8) < 0 and y_max + line_height <= frame_height:
+                # Not enough space above, and there is space below
+                text_offset_y = y_max
+            elif y_min - (line_height + 8) >= 0:
+                # Enough space above, keep the label at the top
+                text_offset_y = max(0, y_min - (line_height + 8))
+    elif position in {"bl", "br"}:
+        if text_offset_y + line_height > frame_height:
+            # If there's not enough space below, try above the box
+            text_offset_y = max(0, y_min - (line_height + 8))
 
     # make the coords of the box with a small padding of two pixels
     textbox_coords = (

frigate/util/model.py

@@ -2,9 +2,14 @@
 
 import logging
 import os
-from typing import Any
+from typing import Any, Optional
 
+import cv2
+import numpy as np
 import onnxruntime as ort
+from playhouse.sqliteq import SqliteQueueDatabase
+
+from frigate.config.semantic_search import FaceRecognitionConfig
 
 try:
     import openvino as ov
@@ -15,6 +20,9 @@ except ImportError:
 logger = logging.getLogger(__name__)
 
 
+MIN_MATCHING_FACES = 2
+
+
 def get_ort_providers(
     force_cpu: bool = False, device: str = "AUTO", requires_fp16: bool = False
 ) -> tuple[list[str], list[dict[str, any]]]:
@@ -148,3 +156,114 @@ class ONNXModelRunner:
             return [infer_request.get_output_tensor().data]
         elif self.type == "ort":
             return self.ort.run(None, input)
+
+
+class FaceClassificationModel:
+    def __init__(self, config: FaceRecognitionConfig, db: SqliteQueueDatabase):
+        self.config = config
+        self.db = db
+        self.landmark_detector = cv2.face.createFacemarkLBF()
+        self.landmark_detector.loadModel("/config/model_cache/facedet/landmarkdet.yaml")
+        self.recognizer: cv2.face.LBPHFaceRecognizer = (
+            cv2.face.LBPHFaceRecognizer_create(
+                radius=2, threshold=(1 - config.min_score) * 1000
+            )
+        )
+        self.label_map: dict[int, str] = {}
+        self.__build_classifier()
+
+    def __build_classifier(self) -> None:
+        labels = []
+        faces = []
+
+        dir = "/media/frigate/clips/faces"
+        for idx, name in enumerate(os.listdir(dir)):
+            if name == "debug":
+                continue
+
+            self.label_map[idx] = name
+            face_folder = os.path.join(dir, name)
+            for image in os.listdir(face_folder):
+                img = cv2.imread(os.path.join(face_folder, image))
+                img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+                img = self.__align_face(img, img.shape[1], img.shape[0])
+                faces.append(img)
+                labels.append(idx)
+
+        self.recognizer.train(faces, np.array(labels))
+
+    def __align_face(
+        self,
+        image: np.ndarray,
+        output_width: int,
+        output_height: int,
+    ) -> np.ndarray:
+        _, lands = self.landmark_detector.fit(
+            image, np.array([(0, 0, image.shape[1], image.shape[0])])
+        )
+        landmarks = lands[0][0]
+
+        # get landmarks for eyes
+        leftEyePts = landmarks[42:48]
+        rightEyePts = landmarks[36:42]
+
+        # compute the center of mass for each eye
+        leftEyeCenter = leftEyePts.mean(axis=0).astype("int")
+        rightEyeCenter = rightEyePts.mean(axis=0).astype("int")
+
+        # compute the angle between the eye centroids
+        dY = rightEyeCenter[1] - leftEyeCenter[1]
+        dX = rightEyeCenter[0] - leftEyeCenter[0]
+        angle = np.degrees(np.arctan2(dY, dX)) - 180
+
+        # compute the desired right eye x-coordinate based on the
+        # desired x-coordinate of the left eye
+        desiredRightEyeX = 1.0 - 0.35
+
+        # determine the scale of the new resulting image by taking
+        # the ratio of the distance between eyes in the *current*
+        # image to the ratio of distance between eyes in the
+        # *desired* image
+        dist = np.sqrt((dX**2) + (dY**2))
+        desiredDist = desiredRightEyeX - 0.35
+        desiredDist *= output_width
+        scale = desiredDist / dist
+
+        # compute center (x, y)-coordinates (i.e., the median point)
+        # between the two eyes in the input image
+        # grab the rotation matrix for rotating and scaling the face
+        eyesCenter = (
+            int((leftEyeCenter[0] + rightEyeCenter[0]) // 2),
+            int((leftEyeCenter[1] + rightEyeCenter[1]) // 2),
+        )
+        M = cv2.getRotationMatrix2D(eyesCenter, angle, scale)
+
+        # update the translation component of the matrix
+        tX = output_width * 0.5
+        tY = output_height * 0.35
+        M[0, 2] += tX - eyesCenter[0]
+        M[1, 2] += tY - eyesCenter[1]
+
+        # apply the affine transformation
+        return cv2.warpAffine(
+            image, M, (output_width, output_height), flags=cv2.INTER_CUBIC
+        )
+
+    def clear_classifier(self) -> None:
+        self.classifier = None
+        self.labeler = None
+        self.label_map = {}
+
+    def classify_face(self, face_image: np.ndarray) -> Optional[tuple[str, float]]:
+        if not self.label_map:
+            self.__build_classifier()
+
+        img = cv2.cvtColor(face_image, cv2.COLOR_BGR2GRAY)
+        img = self.__align_face(img, img.shape[1], img.shape[0])
+        index, distance = self.recognizer.predict(img)
+
+        if index == -1:
+            return None
+
+        score = 1.0 - (distance / 1000)
+        return self.label_map[index], round(score, 2)