Semantic Search for Detections (#11899)

* Initial re-implementation of semantic search

* put docker-compose back and make reindex match docs

* remove debug code and fix import

* fix docs

* manually build pysqlite3 as binaries are only available for x86-64

* update comment in build_pysqlite3.sh

* only embed objects

* better error handling when genai fails

* ask ollama to pull requested model at startup

* update ollama docs

* address some PR review comments

* fix lint

* use IPC to write description, update docs for reindex

* remove gemini-pro-vision from docs as it will be unavailable soon

* fix OpenAI doc available models

* fix api error in gemini and metadata for embeddings

frigate/app.py

@@ -22,11 +22,11 @@ from pydantic import ValidationError
 from frigate.api.app import create_app
 from frigate.api.auth import hash_password
 from frigate.comms.config_updater import ConfigPublisher
-from frigate.comms.detections_updater import DetectionProxy
 from frigate.comms.dispatcher import Communicator, Dispatcher
 from frigate.comms.inter_process import InterProcessCommunicator
 from frigate.comms.mqtt import MqttClient
 from frigate.comms.ws import WebSocketClient
+from frigate.comms.zmq_proxy import ZmqProxy
 from frigate.config import FrigateConfig
 from frigate.const import (
     CACHE_DIR,
@@ -37,6 +37,8 @@ from frigate.const import (
     MODEL_CACHE_DIR,
     RECORD_DIR,
 )
+from frigate.embeddings import manage_embeddings
+from frigate.embeddings.embeddings import Embeddings
 from frigate.events.audio import listen_to_audio
 from frigate.events.cleanup import EventCleanup
 from frigate.events.external import ExternalEventProcessor
@@ -316,7 +318,21 @@ class FrigateApp:
         self.review_segment_process = review_segment_process
         review_segment_process.start()
         self.processes["review_segment"] = review_segment_process.pid or 0
-        logger.info(f"Recording process started: {review_segment_process.pid}")
+        logger.info(f"Review process started: {review_segment_process.pid}")
+
+    def init_embeddings_manager(self) -> None:
+        # Create a client for other processes to use
+        self.embeddings = Embeddings()
+        embedding_process = mp.Process(
+            target=manage_embeddings,
+            name="embeddings_manager",
+            args=(self.config,),
+        )
+        embedding_process.daemon = True
+        self.embedding_process = embedding_process
+        embedding_process.start()
+        self.processes["embeddings"] = embedding_process.pid or 0
+        logger.info(f"Embedding process started: {embedding_process.pid}")
 
     def bind_database(self) -> None:
         """Bind db to the main process."""
@@ -362,7 +378,7 @@ class FrigateApp:
     def init_inter_process_communicator(self) -> None:
         self.inter_process_communicator = InterProcessCommunicator()
         self.inter_config_updater = ConfigPublisher()
-        self.inter_detection_proxy = DetectionProxy()
+        self.inter_zmq_proxy = ZmqProxy()
 
     def init_web_server(self) -> None:
         self.flask_app = create_app(
@@ -678,6 +694,7 @@ class FrigateApp:
             self.init_onvif()
             self.init_recording_manager()
             self.init_review_segment_manager()
+            self.init_embeddings_manager()
             self.init_go2rtc()
             self.bind_database()
             self.check_db_data_migrations()
@@ -797,7 +814,7 @@ class FrigateApp:
         # Stop Communicators
         self.inter_process_communicator.stop()
         self.inter_config_updater.stop()
-        self.inter_detection_proxy.stop()
+        self.inter_zmq_proxy.stop()
 
         while len(self.detection_shms) > 0:
             shm = self.detection_shms.pop()