chris/frigate
1
0
Fork 0
forked from Github/frigate
Code Pull Requests Packages Activity

Regenerate genai tracked object descriptions (#13930)

Browse Source 
* add genai to frigateconfig

* add regenerate button if genai is enabled

* add endpoint and new zmq pub/sub model

* move publisher to app

* dont override

* logging

* debug timeouts

* clean up

* clean up

* allow saving of empty description

* ensure descriptions can be empty

* update search detail when results change

* revalidate explore page on focus

* global mutate hook

* description websocket hook and dispatcher

* revalidation and mutation

* fix merge conflicts

* update tests

* fix merge conflicts

* fix response message

* fix response message

* fix fastapi

* fix test

* remove log

* json content

* fix content response

* more json content fixes

* another one
This commit is contained in:
Josh Hawkins
2024-09-24 09:14:51 -05:00
committed by GitHub
parent cffc431bf0
commit ecbf0410eb
14 changed files with 274 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
frigate/api/event.py
View File

@@ -928,20 +928,62 @@ def set_description(
ids=[event_id],
)
response_message = (
f"Event {event_id} description is now blank"
if new_description is None or len(new_description) == 0
else f"Event {event_id} description set to {new_description}"
)
return JSONResponse(
content=(
{
"success": True,
"message": "Event "
+ event_id
+ " description set to "
+ new_description,
"message": response_message,
}
),
status_code=200,
)
@router.put("/events/<id>/description/regenerate")
def regenerate_description(request: Request, event_id: str):
try:
event: Event = Event.get(Event.id == event_id)
except DoesNotExist:
return JSONResponse(
content=({"success": False, "message": "Event " + event_id + " not found"}),
status_code=404,
)
if (
request.app.frigate_config.semantic_search.enabled
and request.app.frigate_config.genai.enabled
):
request.app.event_metadata_updater.publish(event.id)
return JSONResponse(
content=(
{
"success": True,
"message": "Event "
+ event_id
+ " description regeneration has been requested.",
}
),
status_code=200,
)
return JSONResponse(
content=(
{
"success": False,
"message": "Semantic search and generative AI are not enabled",
}
),
status_code=400,
)
@router.delete("/events/{event_id}")
def delete_event(request: Request, event_id: str):
try:

5
frigate/api/fastapi_app.py
View File

@@ -13,6 +13,9 @@ from starlette_context.plugins import Plugin
from frigate.api import app as main_app
from frigate.api import auth, event, export, media, notification, preview, review
from frigate.api.auth import get_jwt_secret, limiter
from frigate.comms.event_metadata_updater import (
EventMetadataPublisher,
)
from frigate.config import FrigateConfig
from frigate.embeddings import EmbeddingsContext
from frigate.events.external import ExternalEventProcessor
@@ -47,6 +50,7 @@ def create_fastapi_app(
onvif: OnvifController,
external_processor: ExternalEventProcessor,
stats_emitter: StatsEmitter,
event_metadata_updater: EventMetadataPublisher,
):
logger.info("Starting FastAPI app")
app = FastAPI(
@@ -102,6 +106,7 @@ def create_fastapi_app(
app.camera_error_image = None
app.onvif = onvif
app.stats_emitter = stats_emitter
app.event_metadata_updater = event_metadata_updater
app.external_processor = external_processor
app.jwt_token = get_jwt_secret() if frigate_config.auth.enabled else None

9
frigate/app.py
View File

@@ -18,6 +18,10 @@ from frigate.api.auth import hash_password
from frigate.api.fastapi_app import create_fastapi_app
from frigate.comms.config_updater import ConfigPublisher
from frigate.comms.dispatcher import Communicator, Dispatcher
from frigate.comms.event_metadata_updater import (
EventMetadataPublisher,
EventMetadataTypeEnum,
)
from frigate.comms.inter_process import InterProcessCommunicator
from frigate.comms.mqtt import MqttClient
from frigate.comms.webpush import WebPushClient
@@ -332,6 +336,9 @@ class FrigateApp:
def init_inter_process_communicator(self) -> None:
self.inter_process_communicator = InterProcessCommunicator()
self.inter_config_updater = ConfigPublisher()
self.event_metadata_updater = EventMetadataPublisher(
EventMetadataTypeEnum.regenerate_description
)
self.inter_zmq_proxy = ZmqProxy()
def init_onvif(self) -> None:
@@ -656,6 +663,7 @@ class FrigateApp:
self.onvif_controller,
self.external_event_processor,
self.stats_emitter,
self.event_metadata_updater,
),
host="127.0.0.1",
port=5001,
@@ -743,6 +751,7 @@ class FrigateApp:
# Stop Communicators
self.inter_process_communicator.stop()
self.inter_config_updater.stop()
self.event_metadata_updater.stop()
self.inter_zmq_proxy.stop()
while len(self.detection_shms) > 0:

4
frigate/comms/dispatcher.py
View File

@@ -140,6 +140,10 @@ class Dispatcher:
event: Event = Event.get(Event.id == payload["id"])
event.data["description"] = payload["description"]
event.save()
self.publish(
"event_update",
json.dumps({"id": event.id, "description": event.data["description"]}),
)
elif topic == "onConnect":
camera_status = self.camera_activity.copy()

44
frigate/comms/event_metadata_updater.py Normal file
View File

@@ -0,0 +1,44 @@
"""Facilitates communication between processes."""
import logging
from enum import Enum
from typing import Optional
from .zmq_proxy import Publisher, Subscriber
logger = logging.getLogger(__name__)
class EventMetadataTypeEnum(str, Enum):
all = ""
regenerate_description = "regenerate_description"
class EventMetadataPublisher(Publisher):
"""Simplifies receiving event metadata."""
topic_base = "event_metadata/"
def __init__(self, topic: EventMetadataTypeEnum) -> None:
topic = topic.value
super().__init__(topic)
class EventMetadataSubscriber(Subscriber):
"""Simplifies receiving event metadata."""
topic_base = "event_metadata/"
def __init__(self, topic: EventMetadataTypeEnum) -> None:
topic = topic.value
super().__init__(topic)
def check_for_update(
self, timeout: float = None
) -> Optional[tuple[EventMetadataTypeEnum, any]]:
return super().check_for_update(timeout)
def _return_object(self, topic: str, payload: any) -> any:
if payload is None:
return (None, None)
return (EventMetadataTypeEnum[topic[len(self.topic_base) :]], payload)

36
frigate/embeddings/maintainer.py
View File

@@ -12,6 +12,10 @@ import numpy as np
from peewee import DoesNotExist
from PIL import Image
from frigate.comms.event_metadata_updater import (
EventMetadataSubscriber,
EventMetadataTypeEnum,
)
from frigate.comms.events_updater import EventEndSubscriber, EventUpdateSubscriber
from frigate.comms.inter_process import InterProcessRequestor
from frigate.config import FrigateConfig
@@ -40,6 +44,9 @@ class EmbeddingMaintainer(threading.Thread):
self.embeddings = Embeddings()
self.event_subscriber = EventUpdateSubscriber()
self.event_end_subscriber = EventEndSubscriber()
self.event_metadata_subscriber = EventMetadataSubscriber(
EventMetadataTypeEnum.regenerate_description
)
self.frame_manager = SharedMemoryFrameManager()
# create communication for updating event descriptions
self.requestor = InterProcessRequestor()
@@ -52,9 +59,11 @@ class EmbeddingMaintainer(threading.Thread):
while not self.stop_event.is_set():
self._process_updates()
self._process_finalized()
self._process_event_metadata()
self.event_subscriber.stop()
self.event_end_subscriber.stop()
self.event_metadata_subscriber.stop()
self.requestor.stop()
logger.info("Exiting embeddings maintenance...")
@@ -140,6 +149,16 @@ class EmbeddingMaintainer(threading.Thread):
if event_id in self.tracked_events:
del self.tracked_events[event_id]
def _process_event_metadata(self):
# Check for regenerate description requests
(topic, event_id) = self.event_metadata_subscriber.check_for_update()
if topic is None:
return
if event_id:
self.handle_regenerate_description(event_id)
def _create_thumbnail(self, yuv_frame, box, height=500) -> Optional[bytes]:
"""Return jpg thumbnail of a region of the frame."""
frame = cv2.cvtColor(yuv_frame, cv2.COLOR_YUV2BGR_I420)
@@ -200,3 +219,20 @@ class EmbeddingMaintainer(threading.Thread):
len(thumbnails),
description,
)
def handle_regenerate_description(self, event_id: str) -> None:
try:
event: Event = Event.get(Event.id == event_id)
except DoesNotExist:
logger.error(f"Event {event_id} not found for description regeneration")
return
camera_config = self.config.cameras[event.camera]
if not camera_config.genai.enabled or self.genai_client is None:
logger.error(f"GenAI not enabled for camera {event.camera}")
return
metadata = get_metadata(event)
thumbnail = base64.b64decode(event.thumbnail)
self._embed_description(event, [thumbnail], metadata)

11
frigate/test/test_http.py
View File

@@ -121,6 +121,7 @@ class TestHttp(unittest.TestCase):
None,
None,
None,
None,
)
id = "123456.random"
id2 = "7890.random"
@@ -157,6 +158,7 @@ class TestHttp(unittest.TestCase):
None,
None,
None,
None,
)
id = "123456.random"
@@ -178,6 +180,7 @@ class TestHttp(unittest.TestCase):
None,
None,
None,
None,
)
id = "123456.random"
bad_id = "654321.other"
@@ -198,6 +201,7 @@ class TestHttp(unittest.TestCase):
None,
None,
None,
None,
)
id = "123456.random"
@@ -220,6 +224,7 @@ class TestHttp(unittest.TestCase):
None,
None,
None,
None,
)
id = "123456.random"
@@ -246,6 +251,7 @@ class TestHttp(unittest.TestCase):
None,
None,
None,
None,
)
morning_id = "123456.random"
evening_id = "654321.random"
@@ -284,6 +290,7 @@ class TestHttp(unittest.TestCase):
None,
None,
None,
None,
)
id = "123456.random"
sub_label = "sub"
@@ -319,6 +326,7 @@ class TestHttp(unittest.TestCase):
None,
None,
None,
None,
)
id = "123456.random"
sub_label = "sub"
@@ -343,6 +351,7 @@ class TestHttp(unittest.TestCase):
None,
None,
None,
None,
)
with TestClient(app) as client:
@@ -360,6 +369,7 @@ class TestHttp(unittest.TestCase):
None,
None,
None,
None,
)
id = "123456.random"
@@ -383,6 +393,7 @@ class TestHttp(unittest.TestCase):
None,
None,
stats,
None,
)
with TestClient(app) as client: