write prompts for genai at the camera level (#13767)

frigate/config.py

@@ -763,8 +763,14 @@ class GenAIConfig(FrigateBaseModel):
     object_prompts: Dict[str, str] = Field(default={}, title="Object specific prompts.")
 
 
-class GenAICameraConfig(FrigateBaseModel):
+# uses BaseModel because some global attributes are not available at the camera level
+class GenAICameraConfig(BaseModel):
     enabled: bool = Field(default=False, title="Enable GenAI for camera.")
+    prompt: str = Field(
+        default="Describe the {label} in the sequence of images with as much detail as possible. Do not describe the background.",
+        title="Default caption prompt.",
+    )
+    object_prompts: Dict[str, str] = Field(default={}, title="Object specific prompts.")
 
 
 class AudioConfig(FrigateBaseModel):
@@ -1520,7 +1526,7 @@ class FrigateConfig(FrigateBaseModel):
                 "live": ...,
                 "objects": ...,
                 "review": ...,
-                "genai": {"enabled"},
+                "genai": ...,
                 "motion": ...,
                 "detect": ...,
                 "ffmpeg": ...,

frigate/embeddings/maintainer.py

@@ -171,8 +171,11 @@ class EmbeddingMaintainer(threading.Thread):
         self, event: Event, thumbnails: list[bytes], metadata: dict
     ) -> None:
         """Embed the description for an event."""
+        camera_config = self.config.cameras[event.camera]
 
-        description = self.genai_client.generate_description(thumbnails, metadata)
+        description = self.genai_client.generate_description(
+            camera_config, thumbnails, metadata
+        )
 
         if description is None:
             logger.debug("Failed to generate description for %s", event.id)

frigate/genai/__init__.py

@@ -4,7 +4,7 @@ import importlib
 import os
 from typing import Optional
 
-from frigate.config import GenAIConfig, GenAIProviderEnum
+from frigate.config import CameraConfig, GenAIConfig, GenAIProviderEnum
 
 PROVIDERS = {}
 
@@ -28,11 +28,14 @@ class GenAIClient:
         self.provider = self._init_provider()
 
     def generate_description(
-        self, thumbnails: list[bytes], metadata: dict[str, any]
+        self,
+        camera_config: CameraConfig,
+        thumbnails: list[bytes],
+        metadata: dict[str, any],
     ) -> Optional[str]:
         """Generate a description for the frame."""
-        prompt = self.genai_config.object_prompts.get(
-            metadata["label"], self.genai_config.prompt
+        prompt = camera_config.genai.object_prompts.get(
+            metadata["label"], camera_config.genai.prompt
         ).format(**metadata)
         return self._send(prompt, thumbnails)