Regenerate genai tracked object descriptions (#13930)

* add genai to frigateconfig

* add regenerate button if genai is enabled

* add endpoint and new zmq pub/sub model

* move publisher to app

* dont override

* logging

* debug timeouts

* clean up

* clean up

* allow saving of empty description

* ensure descriptions can be empty

* update search detail when results change

* revalidate explore page on focus

* global mutate hook

* description websocket hook and dispatcher

* revalidation and mutation

* fix merge conflicts

* update tests

* fix merge conflicts

* fix response message

* fix response message

* fix fastapi

* fix test

* remove log

* json content

* fix content response

* more json content fixes

* another one

web/src/api/ws.tsx

@@ -321,3 +321,10 @@ export function useImproveContrast(camera: string): {
   );
   return { payload: payload as ToggleableSetting, send };
 }
+
+export function useEventUpdate(): { payload: string } {
+  const {
+    value: { payload },
+  } = useWs("event_update", "");
+  return useDeepMemo(JSON.parse(payload as string));
+}

web/src/components/overlay/detail/SearchDetailDialog.tsx

@@ -45,6 +45,8 @@ import {
 import { ReviewSegment } from "@/types/review";
 import { useNavigate } from "react-router-dom";
 import Chip from "@/components/indicators/Chip";
+import { capitalizeFirstLetter } from "@/utils/stringUtil";
+import useGlobalMutation from "@/hooks/use-global-mutate";
 
 const SEARCH_TABS = [
   "details",
@@ -232,6 +234,10 @@ function ObjectDetailsTab({
 }: ObjectDetailsTabProps) {
   const apiHost = useApiHost();
 
+  // mutation / revalidation
+
+  const mutate = useGlobalMutation();
+
   // data
 
   const [desc, setDesc] = useState(search?.data.description);
@@ -282,6 +288,13 @@ function ObjectDetailsTab({
             position: "top-center",
           });
         }
+        mutate(
+          (key) =>
+            typeof key === "string" &&
+            (key.includes("events") ||
+              key.includes("events/search") ||
+              key.includes("explore")),
+        );
       })
       .catch(() => {
         toast.error("Failed to update the description", {
@@ -289,7 +302,35 @@ function ObjectDetailsTab({
         });
         setDesc(search.data.description);
       });
-  }, [desc, search]);
+  }, [desc, search, mutate]);
+
+  const regenerateDescription = useCallback(() => {
+    if (!search) {
+      return;
+    }
+
+    axios
+      .put(`events/${search.id}/description/regenerate`)
+      .then((resp) => {
+        if (resp.status == 200) {
+          toast.success(
+            `A new description has been requested from ${capitalizeFirstLetter(config?.genai.provider ?? "Generative AI")}. Depending on the speed of your provider, the new description may take some time to regenerate.`,
+            {
+              position: "top-center",
+              duration: 7000,
+            },
+          );
+        }
+      })
+      .catch(() => {
+        toast.error(
+          `Failed to call ${capitalizeFirstLetter(config?.genai.provider ?? "Generative AI")} for a new description`,
+          {
+            position: "top-center",
+          },
+        );
+      });
+  }, [search, config]);
 
   return (
     <div className="flex flex-col gap-5">
@@ -355,7 +396,10 @@ function ObjectDetailsTab({
           value={desc}
           onChange={(e) => setDesc(e.target.value)}
         />
-        <div className="flex w-full flex-row justify-end">
+        <div className="flex w-full flex-row justify-end gap-2">
+          {config?.genai.enabled && (
+            <Button onClick={regenerateDescription}>Regenerate</Button>
+          )}
           <Button variant="select" onClick={updateDescription}>
             Save
           </Button>

web/src/hooks/use-global-mutate.ts

@@ -0,0 +1,16 @@
+// https://github.com/vercel/swr/issues/1670#issuecomment-1844114401
+import { useCallback } from "react";
+import { cache, mutate } from "swr/_internal";
+
+const useGlobalMutation = () => {
+  return useCallback((swrKey: string | ((key: string) => boolean), ...args) => {
+    if (typeof swrKey === "function") {
+      const keys = Array.from(cache.keys()).filter(swrKey);
+      keys.forEach((key) => mutate(key, ...args));
+    } else {
+      mutate(swrKey, ...args);
+    }
+  }, []) as typeof mutate;
+};
+
+export default useGlobalMutation;

web/src/pages/Explore.tsx

@@ -1,3 +1,4 @@
+import { useEventUpdate } from "@/api/ws";
 import { useApiFilterArgs } from "@/hooks/use-api-filter";
 import { SearchFilter, SearchQuery, SearchResult } from "@/types/search";
 import SearchView from "@/views/search/SearchView";
@@ -123,19 +124,19 @@ export default function Explore() {
     return [url, { ...params, limit: API_LIMIT }];
   };
 
-  const { data, size, setSize, isValidating } = useSWRInfinite<SearchResult[]>(
-    getKey,
-    {
-      revalidateFirstPage: true,
-      revalidateAll: false,
-      onLoadingSlow: () => {
-        if (!similaritySearch) {
-          setIsSlowLoading(true);
-        }
-      },
-      loadingTimeout: 10000,
+  const { data, size, setSize, isValidating, mutate } = useSWRInfinite<
+    SearchResult[]
+  >(getKey, {
+    revalidateFirstPage: true,
+    revalidateOnFocus: true,
+    revalidateAll: false,
+    onLoadingSlow: () => {
+      if (!similaritySearch) {
+        setIsSlowLoading(true);
+      }
     },
-  );
+    loadingTimeout: 10000,
+  });
 
   const searchResults = useMemo(
     () => (data ? ([] as SearchResult[]).concat(...data) : []),
@@ -164,6 +165,16 @@ export default function Explore() {
     }
   }, [isReachingEnd, isLoadingMore, setSize, size, searchResults, searchQuery]);
 
+  // mutation and revalidation
+
+  const eventUpdate = useEventUpdate();
+
+  useEffect(() => {
+    mutate();
+    // mutate / revalidate when event description updates come in
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [eventUpdate]);
+
   return (
     <>
       {isSlowLoading && !similaritySearch ? (

web/src/types/frigateConfig.ts

@@ -298,6 +298,16 @@ export interface FrigateConfig {
     retry_interval: number;
   };
 
+  genai: {
+    enabled: boolean;
+    provider: string;
+    base_url?: string;
+    api_key?: string;
+    model: string;
+    prompt: string;
+    object_prompts: { [key: string]: string };
+  };
+
   go2rtc: {
     streams: string[];
     webrtc: {

web/src/views/explore/ExploreView.tsx

@@ -37,7 +37,7 @@ export default function ExploreView({ onSelectSearch }: ExploreViewProps) {
       },
     ],
     {
-      revalidateOnFocus: false,
+      revalidateOnFocus: true,
     },
   );
 

web/src/views/search/SearchView.tsx

@@ -23,6 +23,7 @@ import useKeyboardListener, {
 import scrollIntoView from "scroll-into-view-if-needed";
 import InputWithTags from "@/components/input/InputWithTags";
 import { ScrollArea, ScrollBar } from "@/components/ui/scroll-area";
+import { isEqual } from "lodash";
 
 type SearchViewProps = {
   search: string;
@@ -140,6 +141,21 @@ export default function SearchView({
     setSelectedIndex(index);
   }, []);
 
+  // update search detail when results change
+
+  useEffect(() => {
+    if (searchDetail && searchResults) {
+      const flattenedResults = searchResults.flat();
+      const updatedSearchDetail = flattenedResults.find(
+        (result) => result.id === searchDetail.id,
+      );
+
+      if (updatedSearchDetail && !isEqual(updatedSearchDetail, searchDetail)) {
+        setSearchDetail(updatedSearchDetail);
+      }
+    }
+  }, [searchResults, searchDetail]);
+
   // confidence score - probably needs tweaking
 
   const zScoreToConfidence = (score: number, source: string) => {