Add metrics page for embeddings and face / license plate processing times (#15818)

* Get stats for embeddings inferences

* cleanup embeddings inferences

* Enable UI for feature metrics

* Change threshold

* Fix check

* Update python for actions

* Set python version

* Ignore type for now

web/src/pages/FaceLibrary.tsx

@@ -309,7 +309,7 @@ function FaceAttempt({
             <div className="capitalize">{data.name}</div>
             <div
               className={cn(
-                Number.parseFloat(data.score) > threshold
+                Number.parseFloat(data.score) >= threshold
                   ? "text-success"
                   : "text-danger",
               )}

web/src/pages/System.tsx

@@ -1,12 +1,12 @@
 import useSWR from "swr";
 import { FrigateStats } from "@/types/stats";
-import { useEffect, useState } from "react";
+import { useEffect, useMemo, useState } from "react";
 import TimeAgo from "@/components/dynamic/TimeAgo";
 import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group";
 import { isDesktop, isMobile } from "react-device-detect";
 import GeneralMetrics from "@/views/system/GeneralMetrics";
 import StorageMetrics from "@/views/system/StorageMetrics";
-import { LuActivity, LuHardDrive } from "react-icons/lu";
+import { LuActivity, LuHardDrive, LuSearchCode } from "react-icons/lu";
 import { FaVideo } from "react-icons/fa";
 import Logo from "@/components/Logo";
 import useOptimisticState from "@/hooks/use-optimistic-state";
@@ -14,11 +14,28 @@ import CameraMetrics from "@/views/system/CameraMetrics";
 import { useHashState } from "@/hooks/use-overlay-state";
 import { capitalizeFirstLetter } from "@/utils/stringUtil";
 import { Toaster } from "@/components/ui/sonner";
+import { FrigateConfig } from "@/types/frigateConfig";
+import FeatureMetrics from "@/views/system/FeatureMetrics";
 
-const metrics = ["general", "storage", "cameras"] as const;
-type SystemMetric = (typeof metrics)[number];
+const allMetrics = ["general", "features", "storage", "cameras"] as const;
+type SystemMetric = (typeof allMetrics)[number];
 
 function System() {
+  const { data: config } = useSWR<FrigateConfig>("config", {
+    revalidateOnFocus: false,
+  });
+
+  const metrics = useMemo(() => {
+    const metrics = [...allMetrics];
+
+    if (!config?.semantic_search.enabled) {
+      const index = metrics.indexOf("features");
+      metrics.splice(index, 1);
+    }
+
+    return metrics;
+  }, [config]);
+
   // stats page
 
   const [page, setPage] = useHashState<SystemMetric>();
@@ -67,6 +84,7 @@ function System() {
               aria-label={`Select ${item}`}
             >
               {item == "general" && <LuActivity className="size-4" />}
+              {item == "features" && <LuSearchCode className="size-4" />}
               {item == "storage" && <LuHardDrive className="size-4" />}
               {item == "cameras" && <FaVideo className="size-4" />}
               {isDesktop && <div className="capitalize">{item}</div>}
@@ -96,6 +114,12 @@ function System() {
           setLastUpdated={setLastUpdated}
         />
       )}
+      {page == "features" && (
+        <FeatureMetrics
+          lastUpdated={lastUpdated}
+          setLastUpdated={setLastUpdated}
+        />
+      )}
       {page == "storage" && <StorageMetrics setLastUpdated={setLastUpdated} />}
       {page == "cameras" && (
         <CameraMetrics

web/src/types/graph.ts

@@ -18,6 +18,11 @@ export const InferenceThreshold = {
   error: 100,
 } as Threshold;
 
+export const EmbeddingThreshold = {
+  warning: 500,
+  error: 1000,
+} as Threshold;
+
 export const DetectorTempThreshold = {
   warning: 72,
   error: 80,

web/src/types/stats.ts

@@ -2,6 +2,7 @@ export interface FrigateStats {
   cameras: { [camera_name: string]: CameraStats };
   cpu_usages: { [pid: string]: CpuStats };
   detectors: { [detectorKey: string]: DetectorStats };
+  embeddings?: EmbeddingsStats;
   gpu_usages?: { [gpuKey: string]: GpuStats };
   processes: { [processKey: string]: ExtraProcessStats };
   service: ServiceStats;
@@ -34,6 +35,13 @@ export type DetectorStats = {
   pid: number;
 };
 
+export type EmbeddingsStats = {
+  image_embedding_speed: number;
+  face_embedding_speed: number;
+  plate_recognition_speed: number;
+  text_embedding_speed: number;
+};
+
 export type ExtraProcessStats = {
   pid: number;
 };

web/src/views/system/FeatureMetrics.tsx

@@ -0,0 +1,122 @@
+import useSWR from "swr";
+import { FrigateStats } from "@/types/stats";
+import { useEffect, useMemo, useState } from "react";
+import { useFrigateStats } from "@/api/ws";
+import { EmbeddingThreshold } from "@/types/graph";
+import { Skeleton } from "@/components/ui/skeleton";
+import { ThresholdBarGraph } from "@/components/graph/SystemGraph";
+import { cn } from "@/lib/utils";
+
+type FeatureMetricsProps = {
+  lastUpdated: number;
+  setLastUpdated: (last: number) => void;
+};
+export default function FeatureMetrics({
+  lastUpdated,
+  setLastUpdated,
+}: FeatureMetricsProps) {
+  // stats
+
+  const { data: initialStats } = useSWR<FrigateStats[]>(
+    ["stats/history", { keys: "embeddings,service" }],
+    {
+      revalidateOnFocus: false,
+    },
+  );
+
+  const [statsHistory, setStatsHistory] = useState<FrigateStats[]>([]);
+  const updatedStats = useFrigateStats();
+
+  useEffect(() => {
+    if (initialStats == undefined || initialStats.length == 0) {
+      return;
+    }
+
+    if (statsHistory.length == 0) {
+      setStatsHistory(initialStats);
+      return;
+    }
+
+    if (!updatedStats) {
+      return;
+    }
+
+    if (updatedStats.service.last_updated > lastUpdated) {
+      setStatsHistory([...statsHistory.slice(1), updatedStats]);
+      setLastUpdated(Date.now() / 1000);
+    }
+  }, [initialStats, updatedStats, statsHistory, lastUpdated, setLastUpdated]);
+
+  // timestamps
+
+  const updateTimes = useMemo(
+    () => statsHistory.map((stats) => stats.service.last_updated),
+    [statsHistory],
+  );
+
+  // features stats
+
+  const embeddingInferenceTimeSeries = useMemo(() => {
+    if (!statsHistory) {
+      return [];
+    }
+
+    const series: {
+      [key: string]: { name: string; data: { x: number; y: number }[] };
+    } = {};
+
+    statsHistory.forEach((stats, statsIdx) => {
+      if (!stats?.embeddings) {
+        return;
+      }
+
+      Object.entries(stats.embeddings).forEach(([rawKey, stat]) => {
+        const key = rawKey.replaceAll("_", " ");
+
+        if (!(key in series)) {
+          series[key] = { name: key, data: [] };
+        }
+
+        series[key].data.push({ x: statsIdx + 1, y: stat });
+      });
+    });
+    return Object.values(series);
+  }, [statsHistory]);
+
+  return (
+    <>
+      <div className="scrollbar-container mt-4 flex size-full flex-col overflow-y-auto">
+        <div className="text-sm font-medium text-muted-foreground">
+          Features
+        </div>
+        <div
+          className={cn(
+            "mt-4 grid w-full grid-cols-1 gap-2 sm:grid-cols-3",
+            embeddingInferenceTimeSeries && "sm:grid-cols-4",
+          )}
+        >
+          {statsHistory.length != 0 ? (
+            <>
+              {embeddingInferenceTimeSeries.map((series) => (
+                <div className="rounded-lg bg-background_alt p-2.5 md:rounded-2xl">
+                  <div className="mb-5 capitalize">{series.name}</div>
+                  <ThresholdBarGraph
+                    key={series.name}
+                    graphId={`${series.name}-inference`}
+                    name={series.name}
+                    unit="ms"
+                    threshold={EmbeddingThreshold}
+                    updateTimes={updateTimes}
+                    data={[series]}
+                  />
+                </div>
+              ))}
+            </>
+          ) : (
+            <Skeleton className="aspect-video w-full rounded-lg md:rounded-2xl" />
+          )}
+        </div>
+      </div>
+    </>
+  );
+}