Improve face recognition (#15205)

* Validate faces using cosine distance and SVC

* Formatting

* Use opencv instead of face embedding

* Update docs for training data

* Adjust to score system

* Set bounds

* remove face embeddings

* Update writing images

* Add face library page

* Add ability to select file

* Install opencv deps

* Cleanup

* Use different deps

* Move deps

* Cleanup

* Only show face library for desktop

* Implement deleting

* Add ability to upload image

* Add support for uploading images

web/src/App.tsx

@@ -19,6 +19,7 @@ const ConfigEditor = lazy(() => import("@/pages/ConfigEditor"));
 const System = lazy(() => import("@/pages/System"));
 const Settings = lazy(() => import("@/pages/Settings"));
 const UIPlayground = lazy(() => import("@/pages/UIPlayground"));
+const FaceLibrary = lazy(() => import("@/pages/FaceLibrary"));
 const Logs = lazy(() => import("@/pages/Logs"));
 
 function App() {
@@ -51,6 +52,7 @@ function App() {
                   <Route path="/config" element={<ConfigEditor />} />
                   <Route path="/logs" element={<Logs />} />
                   <Route path="/playground" element={<UIPlayground />} />
+                  <Route path="/faces" element={<FaceLibrary />} />
                   <Route path="*" element={<Redirect to="/" />} />
                 </Routes>
               </Suspense>

web/src/components/overlay/dialog/UploadImageDialog.tsx

@@ -0,0 +1,88 @@
+import { Button } from "@/components/ui/button";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTitle,
+} from "@/components/ui/dialog";
+import { Form, FormControl, FormField, FormItem } from "@/components/ui/form";
+import { Input } from "@/components/ui/input";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { useCallback } from "react";
+import { useForm } from "react-hook-form";
+import { z } from "zod";
+
+type UploadImageDialogProps = {
+  open: boolean;
+  title: string;
+  description?: string;
+  setOpen: (open: boolean) => void;
+  onSave: (file: File) => void;
+};
+export default function UploadImageDialog({
+  open,
+  title,
+  description,
+  setOpen,
+  onSave,
+}: UploadImageDialogProps) {
+  const formSchema = z.object({
+    file: z.instanceof(FileList, { message: "Please select an image file." }),
+  });
+
+  const form = useForm<z.infer<typeof formSchema>>({
+    resolver: zodResolver(formSchema),
+  });
+  const fileRef = form.register("file");
+
+  // upload handler
+
+  const onSubmit = useCallback(
+    (data: z.infer<typeof formSchema>) => {
+      if (!data["file"]) {
+        return;
+      }
+
+      onSave(data["file"]["0"]);
+    },
+    [onSave],
+  );
+
+  return (
+    <Dialog open={open} defaultOpen={false} onOpenChange={setOpen}>
+      <DialogContent>
+        <DialogHeader>
+          <DialogTitle>{title}</DialogTitle>
+          {description && <DialogDescription>{description}</DialogDescription>}
+        </DialogHeader>
+        <Form {...form}>
+          <form onSubmit={form.handleSubmit(onSubmit)}>
+            <FormField
+              control={form.control}
+              name="file"
+              render={() => (
+                <FormItem>
+                  <FormControl>
+                    <Input
+                      className="aspect-video h-40 w-full"
+                      type="file"
+                      {...fileRef}
+                    />
+                  </FormControl>
+                </FormItem>
+              )}
+            />
+            <DialogFooter className="pt-4">
+              <Button onClick={() => setOpen(false)}>Cancel</Button>
+              <Button variant="select" type="submit">
+                Save
+              </Button>
+            </DialogFooter>
+          </form>
+        </Form>
+      </DialogContent>
+    </Dialog>
+  );
+}

web/src/hooks/use-navigation.ts

@@ -1,20 +1,29 @@
 import { ENV } from "@/env";
+import { FrigateConfig } from "@/types/frigateConfig";
 import { NavData } from "@/types/navigation";
 import { useMemo } from "react";
+import { isDesktop } from "react-device-detect";
 import { FaCompactDisc, FaVideo } from "react-icons/fa";
 import { IoSearch } from "react-icons/io5";
 import { LuConstruction } from "react-icons/lu";
 import { MdVideoLibrary } from "react-icons/md";
+import { TbFaceId } from "react-icons/tb";
+import useSWR from "swr";
 
 export const ID_LIVE = 1;
 export const ID_REVIEW = 2;
 export const ID_EXPLORE = 3;
 export const ID_EXPORT = 4;
 export const ID_PLAYGROUND = 5;
+export const ID_FACE_LIBRARY = 6;
 
 export default function useNavigation(
   variant: "primary" | "secondary" = "primary",
 ) {
+  const { data: config } = useSWR<FrigateConfig>("config", {
+    revalidateOnFocus: false,
+  });
+
   return useMemo(
     () =>
       [
@@ -54,7 +63,15 @@ export default function useNavigation(
           url: "/playground",
           enabled: ENV !== "production",
         },
+        {
+          id: ID_FACE_LIBRARY,
+          variant,
+          icon: TbFaceId,
+          title: "Face Library",
+          url: "/faces",
+          enabled: isDesktop && config?.face_recognition.enabled,
+        },
       ] as NavData[],
-    [variant],
+    [config?.face_recognition.enabled, variant],
   );
 }

web/src/pages/FaceLibrary.tsx

@@ -0,0 +1,170 @@
+import { baseUrl } from "@/api/baseUrl";
+import Chip from "@/components/indicators/Chip";
+import UploadImageDialog from "@/components/overlay/dialog/UploadImageDialog";
+import { Button } from "@/components/ui/button";
+import { ScrollArea, ScrollBar } from "@/components/ui/scroll-area";
+import { Toaster } from "@/components/ui/sonner";
+import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group";
+import useOptimisticState from "@/hooks/use-optimistic-state";
+import axios from "axios";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { isDesktop } from "react-device-detect";
+import { LuImagePlus, LuTrash } from "react-icons/lu";
+import { toast } from "sonner";
+import useSWR from "swr";
+
+export default function FaceLibrary() {
+  const [page, setPage] = useState<string>();
+  const [pageToggle, setPageToggle] = useOptimisticState(page, setPage, 100);
+  const tabsRef = useRef<HTMLDivElement | null>(null);
+
+  // face data
+
+  const { data: faceData } = useSWR("faces");
+
+  const faces = useMemo<string[]>(
+    () => (faceData ? Object.keys(faceData) : []),
+    [faceData],
+  );
+  const faceImages = useMemo<string[]>(
+    () => (pageToggle && faceData ? faceData[pageToggle] : []),
+    [pageToggle, faceData],
+  );
+
+  useEffect(() => {
+    if (!pageToggle && faces) {
+      setPageToggle(faces[0]);
+    }
+    // we need to listen on the value of the faces list
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [faces]);
+
+  // upload
+
+  const [upload, setUpload] = useState(false);
+
+  const onUploadImage = useCallback(
+    (file: File) => {
+      const formData = new FormData();
+      formData.append("file", file);
+      axios.post(`faces/${pageToggle}`, formData, {
+        headers: {
+          "Content-Type": "multipart/form-data",
+        },
+      });
+    },
+    [pageToggle],
+  );
+
+  return (
+    <div className="flex size-full flex-col p-2">
+      <Toaster />
+
+      <UploadImageDialog
+        open={upload}
+        title="Upload Face Image"
+        description={`Upload an image to scan for faces and include for ${pageToggle}`}
+        setOpen={setUpload}
+        onSave={onUploadImage}
+      />
+
+      <div className="relative flex h-11 w-full items-center justify-between">
+        <ScrollArea className="w-full whitespace-nowrap">
+          <div ref={tabsRef} className="flex flex-row">
+            <ToggleGroup
+              className="*:rounded-md *:px-3 *:py-4"
+              type="single"
+              size="sm"
+              value={pageToggle}
+              onValueChange={(value: string) => {
+                if (value) {
+                  setPageToggle(value);
+                }
+              }}
+            >
+              {Object.values(faces).map((item) => (
+                <ToggleGroupItem
+                  key={item}
+                  className={`flex scroll-mx-10 items-center justify-between gap-2 ${page == "UI settings" ? "last:mr-20" : ""} ${pageToggle == item ? "" : "*:text-muted-foreground"}`}
+                  value={item}
+                  data-nav-item={item}
+                  aria-label={`Select ${item}`}
+                >
+                  <div className="capitalize">{item}</div>
+                </ToggleGroupItem>
+              ))}
+            </ToggleGroup>
+            <ScrollBar orientation="horizontal" className="h-0" />
+          </div>
+        </ScrollArea>
+      </div>
+      {pageToggle && (
+        <div className="flex flex-wrap gap-2">
+          {faceImages.map((image: string) => (
+            <FaceImage key={image} name={pageToggle} image={image} />
+          ))}
+          <Button
+            key="upload"
+            className="size-40"
+            onClick={() => setUpload(true)}
+          >
+            <LuImagePlus className="size-10" />
+          </Button>
+        </div>
+      )}
+    </div>
+  );
+}
+
+type FaceImageProps = {
+  name: string;
+  image: string;
+};
+function FaceImage({ name, image }: FaceImageProps) {
+  const [hovered, setHovered] = useState(false);
+
+  const onDelete = useCallback(() => {
+    axios
+      .post(`/faces/${name}/delete`, { ids: [image] })
+      .then((resp) => {
+        if (resp.status == 200) {
+          toast.error(`Successfully deleted face.`, { position: "top-center" });
+        }
+      })
+      .catch((error) => {
+        if (error.response?.data?.message) {
+          toast.error(`Failed to delete: ${error.response.data.message}`, {
+            position: "top-center",
+          });
+        } else {
+          toast.error(`Failed to delete: ${error.message}`, {
+            position: "top-center",
+          });
+        }
+      });
+  }, [name, image]);
+
+  return (
+    <div
+      className="relative h-40"
+      onMouseEnter={isDesktop ? () => setHovered(true) : undefined}
+      onMouseLeave={isDesktop ? () => setHovered(false) : undefined}
+      onClick={isDesktop ? undefined : () => setHovered(!hovered)}
+    >
+      {hovered && (
+        <div className="absolute right-1 top-1">
+          <Chip
+            className="cursor-pointer rounded-md bg-gray-500 bg-gradient-to-br from-gray-400 to-gray-500"
+            onClick={() => onDelete()}
+          >
+            <LuTrash className="size-4 fill-destructive text-destructive" />
+          </Chip>
+        </div>
+      )}
+      <img
+        className="h-40 rounded-md"
+        src={`${baseUrl}clips/faces/${name}/${image}`}
+      />
+    </div>
+  );
+}

web/src/types/frigateConfig.ts

@@ -288,6 +288,10 @@ export interface FrigateConfig {
 
   environment_vars: Record<string, unknown>;
 
+  face_recognition: {
+    enabled: boolean;
+  };
+
   ffmpeg: {
     global_args: string[];
     hwaccel_args: string;