Semantic Search for Detections (#11899)

* Initial re-implementation of semantic search

* put docker-compose back and make reindex match docs

* remove debug code and fix import

* fix docs

* manually build pysqlite3 as binaries are only available for x86-64

* update comment in build_pysqlite3.sh

* only embed objects

* better error handling when genai fails

* ask ollama to pull requested model at startup

* update ollama docs

* address some PR review comments

* fix lint

* use IPC to write description, update docs for reindex

* remove gemini-pro-vision from docs as it will be unavailable soon

* fix OpenAI doc available models

* fix api error in gemini and metadata for embeddings

frigate/genai/__init__.py

@@ -0,0 +1,63 @@
+"""Generative AI module for Frigate."""
+
+import importlib
+import os
+from typing import Optional
+
+from frigate.config import GenAIConfig, GenAIProviderEnum
+
+PROVIDERS = {}
+
+
+def register_genai_provider(key: GenAIProviderEnum):
+    """Register a GenAI provider."""
+
+    def decorator(cls):
+        PROVIDERS[key] = cls
+        return cls
+
+    return decorator
+
+
+class GenAIClient:
+    """Generative AI client for Frigate."""
+
+    def __init__(self, genai_config: GenAIConfig, timeout: int = 60) -> None:
+        self.genai_config: GenAIConfig = genai_config
+        self.timeout = timeout
+        self.provider = self._init_provider()
+
+    def generate_description(
+        self, thumbnails: list[bytes], metadata: dict[str, any]
+    ) -> Optional[str]:
+        """Generate a description for the frame."""
+        prompt = self.genai_config.object_prompts.get(
+            metadata["label"], self.genai_config.prompt
+        ).format(**metadata)
+        return self._send(prompt, thumbnails)
+
+    def _init_provider(self):
+        """Initialize the client."""
+        return None
+
+    def _send(self, prompt: str, images: list[bytes]) -> Optional[str]:
+        """Submit a request to the provider."""
+        return None
+
+
+def get_genai_client(genai_config: GenAIConfig) -> Optional[GenAIClient]:
+    """Get the GenAI client."""
+    if genai_config.enabled:
+        load_providers()
+        provider = PROVIDERS.get(genai_config.provider)
+        if provider:
+            return provider(genai_config)
+    return None
+
+
+def load_providers():
+    package_dir = os.path.dirname(__file__)
+    for filename in os.listdir(package_dir):
+        if filename.endswith(".py") and filename != "__init__.py":
+            module_name = f"frigate.genai.{filename[:-3]}"
+            importlib.import_module(module_name)

frigate/genai/gemini.py

@@ -0,0 +1,49 @@
+"""Gemini Provider for Frigate AI."""
+
+from typing import Optional
+
+import google.generativeai as genai
+from google.api_core.exceptions import GoogleAPICallError
+
+from frigate.config import GenAIProviderEnum
+from frigate.genai import GenAIClient, register_genai_provider
+
+
+@register_genai_provider(GenAIProviderEnum.gemini)
+class GeminiClient(GenAIClient):
+    """Generative AI client for Frigate using Gemini."""
+
+    provider: genai.GenerativeModel
+
+    def _init_provider(self):
+        """Initialize the client."""
+        genai.configure(api_key=self.genai_config.api_key)
+        return genai.GenerativeModel(self.genai_config.model)
+
+    def _send(self, prompt: str, images: list[bytes]) -> Optional[str]:
+        """Submit a request to Gemini."""
+        data = [
+            {
+                "mime_type": "image/jpeg",
+                "data": img,
+            }
+            for img in images
+        ] + [prompt]
+        try:
+            response = self.provider.generate_content(
+                data,
+                generation_config=genai.types.GenerationConfig(
+                    candidate_count=1,
+                ),
+                request_options=genai.types.RequestOptions(
+                    timeout=self.timeout,
+                ),
+            )
+        except GoogleAPICallError:
+            return None
+        try:
+            description = response.text.strip()
+        except ValueError:
+            # No description was generated
+            return None
+        return description

frigate/genai/ollama.py

@@ -0,0 +1,41 @@
+"""Ollama Provider for Frigate AI."""
+
+import logging
+from typing import Optional
+
+from httpx import TimeoutException
+from ollama import Client as ApiClient
+from ollama import ResponseError
+
+from frigate.config import GenAIProviderEnum
+from frigate.genai import GenAIClient, register_genai_provider
+
+logger = logging.getLogger(__name__)
+
+
+@register_genai_provider(GenAIProviderEnum.ollama)
+class OllamaClient(GenAIClient):
+    """Generative AI client for Frigate using Ollama."""
+
+    provider: ApiClient
+
+    def _init_provider(self):
+        """Initialize the client."""
+        client = ApiClient(host=self.genai_config.base_url, timeout=self.timeout)
+        response = client.pull(self.genai_config.model)
+        if response["status"] != "success":
+            logger.error("Failed to pull %s model from Ollama", self.genai_config.model)
+            return None
+        return client
+
+    def _send(self, prompt: str, images: list[bytes]) -> Optional[str]:
+        """Submit a request to Ollama"""
+        try:
+            result = self.provider.generate(
+                self.genai_config.model,
+                prompt,
+                images=images,
+            )
+            return result["response"].strip()
+        except (TimeoutException, ResponseError):
+            return None

frigate/genai/openai.py

@@ -0,0 +1,51 @@
+"""OpenAI Provider for Frigate AI."""
+
+import base64
+from typing import Optional
+
+from httpx import TimeoutException
+from openai import OpenAI
+
+from frigate.config import GenAIProviderEnum
+from frigate.genai import GenAIClient, register_genai_provider
+
+
+@register_genai_provider(GenAIProviderEnum.openai)
+class OpenAIClient(GenAIClient):
+    """Generative AI client for Frigate using OpenAI."""
+
+    provider: OpenAI
+
+    def _init_provider(self):
+        """Initialize the client."""
+        return OpenAI(api_key=self.genai_config.api_key)
+
+    def _send(self, prompt: str, images: list[bytes]) -> Optional[str]:
+        """Submit a request to OpenAI."""
+        encoded_images = [base64.b64encode(image).decode("utf-8") for image in images]
+        try:
+            result = self.provider.chat.completions.create(
+                model=self.genai_config.model,
+                messages=[
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": f"data:image/jpeg;base64,{image}",
+                                    "detail": "low",
+                                },
+                            }
+                            for image in encoded_images
+                        ]
+                        + [prompt],
+                    },
+                ],
+                timeout=self.timeout,
+            )
+        except TimeoutException:
+            return None
+        if len(result.choices) > 0:
+            return result.choices[0].message.content.strip()
+        return None