Use sqlite-vec extension instead of chromadb for embeddings (#14163)

* swap sqlite_vec for chroma in requirements

* load sqlite_vec in embeddings manager

* remove chroma and revamp Embeddings class for sqlite_vec

* manual minilm onnx inference

* remove chroma in clip model

* migrate api from chroma to sqlite_vec

* migrate event cleanup from chroma to sqlite_vec

* migrate embedding maintainer from chroma to sqlite_vec

* genai description for sqlite_vec

* load sqlite_vec in main thread db

* extend the SqliteQueueDatabase class and use peewee db.execute_sql

* search with Event type for similarity

* fix similarity search

* install and add comment about transformers

* fix normalization

* add id filter

* clean up

* clean up

* fully remove chroma and add transformers env var

* readd uvicorn for fastapi

* readd tokenizer parallelism env var

* remove chroma from docs

* remove chroma from UI

* try removing custom pysqlite3 build

* hard code limit

* optimize queries

* revert explore query

* fix query

* keep building pysqlite3

* single pass fetch and process

* remove unnecessary re-embed

* update deps

* move SqliteVecQueueDatabase to db directory

* make search thumbnail take up full size of results box

* improve typing

* improve model downloading and add status screen

* daemon downloading thread

* catch case when semantic search is disabled

* fix typing

* build sqlite_vec from source

* resolve conflict

* file permissions

* try build deps

* remove sources

* sources

* fix thread start

* include git in build

* reorder embeddings after detectors are started

* build with sqlite amalgamation

* non-platform specific

* use wget instead of curl

* remove unzip -d

* remove sqlite_vec from requirements and load the compiled version

* fix build

* avoid race in db connection

* add scale_factor and bias to description zscore normalization

web/src/api/ws.tsx

@@ -5,6 +5,7 @@ import {
   FrigateCameraState,
   FrigateEvent,
   FrigateReview,
+  ModelState,
   ToggleableSetting,
 } from "@/types/ws";
 import { FrigateStats } from "@/types/stats";
@@ -266,6 +267,41 @@ export function useInitialCameraState(
   return { payload: data ? data[camera] : undefined };
 }
 
+export function useModelState(
+  model: string,
+  revalidateOnFocus: boolean = true,
+): { payload: ModelState } {
+  const {
+    value: { payload },
+    send: sendCommand,
+  } = useWs("model_state", "modelState");
+
+  const data = useDeepMemo(JSON.parse(payload as string));
+
+  useEffect(() => {
+    let listener = undefined;
+    if (revalidateOnFocus) {
+      sendCommand("modelState");
+      listener = () => {
+        if (document.visibilityState == "visible") {
+          sendCommand("modelState");
+        }
+      };
+      addEventListener("visibilitychange", listener);
+    }
+
+    return () => {
+      if (listener) {
+        removeEventListener("visibilitychange", listener);
+      }
+    };
+    // we know that these deps are correct
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [revalidateOnFocus]);
+
+  return { payload: data ? data[model] : undefined };
+}
+
 export function useMotionActivity(camera: string): { payload: string } {
   const {
     value: { payload },

web/src/components/card/SearchThumbnail.tsx

@@ -52,12 +52,11 @@ export default function SearchThumbnail({
         className="absolute inset-0"
         imgLoaded={imgLoaded}
       />
-      <div className={`${imgLoaded ? "visible" : "invisible"}`}>
+      <div className={`size-full ${imgLoaded ? "visible" : "invisible"}`}>
         <img
           ref={imgRef}
           className={cn(
-            "size-full select-none opacity-100 transition-opacity",
-            searchResult.search_source == "thumbnail" && "object-contain",
+            "size-full select-none object-cover object-center opacity-100 transition-opacity",
           )}
           style={
             isIOS

web/src/pages/Explore.tsx

@@ -1,11 +1,15 @@
-import { useEventUpdate } from "@/api/ws";
+import { useEventUpdate, useModelState } from "@/api/ws";
+import ActivityIndicator from "@/components/indicators/activity-indicator";
 import { useApiFilterArgs } from "@/hooks/use-api-filter";
 import { useTimezone } from "@/hooks/use-date-utils";
 import { FrigateConfig } from "@/types/frigateConfig";
 import { SearchFilter, SearchQuery, SearchResult } from "@/types/search";
+import { ModelState } from "@/types/ws";
 import SearchView from "@/views/search/SearchView";
 import { useCallback, useEffect, useMemo, useState } from "react";
+import { LuCheck, LuExternalLink, LuX } from "react-icons/lu";
 import { TbExclamationCircle } from "react-icons/tb";
+import { Link } from "react-router-dom";
 import useSWR from "swr";
 import useSWRInfinite from "swr/infinite";
 
@@ -111,14 +115,10 @@ export default function Explore() {
 
   // paging
 
-  // usually slow only on first run while downloading models
-  const [isSlowLoading, setIsSlowLoading] = useState(false);
-
   const getKey = (
     pageIndex: number,
     previousPageData: SearchResult[] | null,
   ): SearchQuery => {
-    if (isSlowLoading && !similaritySearch) return null;
     if (previousPageData && !previousPageData.length) return null; // reached the end
     if (!searchQuery) return null;
 
@@ -143,12 +143,6 @@ export default function Explore() {
     revalidateFirstPage: true,
     revalidateOnFocus: true,
     revalidateAll: false,
-    onLoadingSlow: () => {
-      if (!similaritySearch) {
-        setIsSlowLoading(true);
-      }
-    },
-    loadingTimeout: 15000,
   });
 
   const searchResults = useMemo(
@@ -168,7 +162,7 @@ export default function Explore() {
       if (searchQuery) {
         const [url] = searchQuery;
 
-        // for chroma, only load 100 results for description and similarity
+        // for embeddings, only load 100 results for description and similarity
         if (url === "events/search" && searchResults.length >= 100) {
           return;
         }
@@ -188,17 +182,113 @@ export default function Explore() {
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [eventUpdate]);
 
+  // model states
+
+  const { payload: minilmModelState } = useModelState(
+    "sentence-transformers/all-MiniLM-L6-v2-model.onnx",
+  );
+  const { payload: minilmTokenizerState } = useModelState(
+    "sentence-transformers/all-MiniLM-L6-v2-tokenizer",
+  );
+  const { payload: clipImageModelState } = useModelState(
+    "clip-clip_image_model_vitb32.onnx",
+  );
+  const { payload: clipTextModelState } = useModelState(
+    "clip-clip_text_model_vitb32.onnx",
+  );
+
+  const allModelsLoaded = useMemo(() => {
+    return (
+      minilmModelState === "downloaded" &&
+      minilmTokenizerState === "downloaded" &&
+      clipImageModelState === "downloaded" &&
+      clipTextModelState === "downloaded"
+    );
+  }, [
+    minilmModelState,
+    minilmTokenizerState,
+    clipImageModelState,
+    clipTextModelState,
+  ]);
+
+  const renderModelStateIcon = (modelState: ModelState) => {
+    if (modelState === "downloading") {
+      return <ActivityIndicator className="size-5" />;
+    }
+    if (modelState === "downloaded") {
+      return <LuCheck className="size-5 text-success" />;
+    }
+    if (modelState === "not_downloaded" || modelState === "error") {
+      return <LuX className="size-5 text-danger" />;
+    }
+    return null;
+  };
+
+  if (
+    !minilmModelState ||
+    !minilmTokenizerState ||
+    !clipImageModelState ||
+    !clipTextModelState
+  ) {
+    return (
+      <ActivityIndicator className="absolute left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2" />
+    );
+  }
+
   return (
     <>
-      {isSlowLoading && !similaritySearch ? (
+      {!allModelsLoaded ? (
         <div className="absolute inset-0 left-1/2 top-1/2 flex h-96 w-96 -translate-x-1/2 -translate-y-1/2">
-          <div className="flex flex-col items-center justify-center rounded-lg bg-background/50 p-5">
-            <p className="my-5 text-lg">Search Unavailable</p>
-            <TbExclamationCircle className="mb-3 size-10" />
-            <p className="max-w-96 text-center">
-              If this is your first time using Search, be patient while Frigate
-              downloads the necessary embeddings models. Check Frigate logs.
-            </p>
+          <div className="flex flex-col items-center justify-center space-y-3 rounded-lg bg-background/50 p-5">
+            <div className="my-5 flex flex-col items-center gap-2 text-xl">
+              <TbExclamationCircle className="mb-3 size-10" />
+              <div>Search Unavailable</div>
+            </div>
+            <div className="max-w-96 text-center">
+              Frigate is downloading the necessary embeddings models to support
+              semantic searching. This may take several minutes depending on the
+              speed of your network connection.
+            </div>
+            <div className="flex w-96 flex-col gap-2 py-5">
+              <div className="flex flex-row items-center justify-center gap-2">
+                {renderModelStateIcon(clipImageModelState)}
+                CLIP image model
+              </div>
+              <div className="flex flex-row items-center justify-center gap-2">
+                {renderModelStateIcon(clipTextModelState)}
+                CLIP text model
+              </div>
+              <div className="flex flex-row items-center justify-center gap-2">
+                {renderModelStateIcon(minilmModelState)}
+                MiniLM sentence model
+              </div>
+              <div className="flex flex-row items-center justify-center gap-2">
+                {renderModelStateIcon(minilmTokenizerState)}
+                MiniLM tokenizer
+              </div>
+            </div>
+            {(minilmModelState === "error" ||
+              clipImageModelState === "error" ||
+              clipTextModelState === "error") && (
+              <div className="my-3 max-w-96 text-center text-danger">
+                An error has occurred. Check Frigate logs.
+              </div>
+            )}
+            <div className="max-w-96 text-center">
+              You may want to reindex the embeddings of your tracked objects
+              once the models are downloaded.
+            </div>
+            <div className="flex max-w-96 items-center text-primary-variant">
+              <Link
+                to="https://docs.frigate.video/configuration/semantic_search"
+                target="_blank"
+                rel="noopener noreferrer"
+                className="inline"
+              >
+                Read the documentation{" "}
+                <LuExternalLink className="ml-2 inline-flex size-3" />
+              </Link>
+            </div>
           </div>
         </div>
       ) : (

web/src/types/log.ts

@@ -12,5 +12,5 @@ export type LogLine = {
   content: string;
 };
 
-export const logTypes = ["frigate", "go2rtc", "nginx", "chroma"] as const;
+export const logTypes = ["frigate", "go2rtc", "nginx"] as const;
 export type LogType = (typeof logTypes)[number];

web/src/types/ws.ts

@@ -56,4 +56,10 @@ export interface FrigateCameraState {
   objects: ObjectType[];
 }
 
+export type ModelState =
+  | "not_downloaded"
+  | "downloading"
+  | "downloaded"
+  | "error";
+
 export type ToggleableSetting = "ON" | "OFF";

web/src/utils/logUtil.ts

@@ -128,46 +128,6 @@ export function parseLogLines(logService: LogType, logs: string[]) {
         };
       })
       .filter((value) => value != null) as LogLine[];
-  } else if (logService == "chroma") {
-    return logs
-      .map((line) => {
-        const match = frigateDateStamp.exec(line);
-
-        if (!match) {
-          const infoIndex = line.indexOf("[INFO]");
-
-          if (infoIndex != -1) {
-            return {
-              dateStamp: line.substring(0, 19),
-              severity: "info",
-              section: "startup",
-              content: line.substring(infoIndex + 6).trim(),
-            };
-          }
-
-          return null;
-        }
-
-        const startup =
-          line.indexOf("Starting component") !== -1 ||
-          line.indexOf("startup") !== -1 ||
-          line.indexOf("Started") !== -1 ||
-          line.indexOf("Uvicorn") !== -1;
-        const api = !!httpMethods.exec(line);
-        const tag = startup ? "startup" : api ? "API" : "server";
-
-        return {
-          dateStamp: match.toString().slice(1, -1),
-          severity: pythonSeverity
-            .exec(line)
-            ?.at(0)
-            ?.toString()
-            ?.toLowerCase() as LogSeverity,
-          section: tag,
-          content: line.substring(match.index + match[0].length).trim(),
-        };
-      })
-      .filter((value) => value != null) as LogLine[];
   }
 
   return [];

web/src/views/search/SearchView.tsx

@@ -189,19 +189,9 @@ export default function SearchView({
 
   // confidence score - probably needs tweaking
 
-  const zScoreToConfidence = (score: number, source: string) => {
-    let midpoint, scale;
-
-    if (source === "thumbnail") {
-      midpoint = 2;
-      scale = 0.5;
-    } else {
-      midpoint = 0.5;
-      scale = 1.5;
-    }
-
+  const zScoreToConfidence = (score: number) => {
     // Sigmoid function: 1 / (1 + e^x)
-    const confidence = 1 / (1 + Math.exp((score - midpoint) * scale));
+    const confidence = 1 / (1 + Math.exp(score));
 
     return Math.round(confidence * 100);
   };
@@ -412,21 +402,13 @@ export default function SearchView({
                                 ) : (
                                   <LuText className="mr-1 size-3" />
                                 )}
-                                {zScoreToConfidence(
-                                  value.search_distance,
-                                  value.search_source,
-                                )}
-                                %
+                                {zScoreToConfidence(value.search_distance)}%
                               </Chip>
                             </TooltipTrigger>
                             <TooltipPortal>
                               <TooltipContent>
                                 Matched {value.search_source} at{" "}
-                                {zScoreToConfidence(
-                                  value.search_distance,
-                                  value.search_source,
-                                )}
-                                %
+                                {zScoreToConfidence(value.search_distance)}%
                               </TooltipContent>
                             </TooltipPortal>
                           </Tooltip>