Implement object lifecycle pane (#13550)

* Object lifecycle pane

* fix thumbnails and annotation offset math

* snapshot endpoint height and format, yaml types, bugfixes

* clean up for new type

* use get_image_from_recording in recordings snapshot api

* make height optional

web/src/components/overlay/detail/AnnotationSettingsPane.tsx

@@ -0,0 +1,235 @@
+import Heading from "@/components/ui/heading";
+import { Label } from "@/components/ui/label";
+import { Switch } from "@/components/ui/switch";
+import { Event } from "@/types/event";
+import { FrigateConfig } from "@/types/frigateConfig";
+import { zodResolver } from "@hookform/resolvers/zod";
+import axios from "axios";
+import { useCallback, useState } from "react";
+import { useForm } from "react-hook-form";
+import { LuExternalLink } from "react-icons/lu";
+import { PiWarningCircle } from "react-icons/pi";
+import { Link } from "react-router-dom";
+import { toast } from "sonner";
+import useSWR from "swr";
+import {
+  Form,
+  FormControl,
+  FormDescription,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from "@/components/ui/form";
+import { z } from "zod";
+import { Button } from "@/components/ui/button";
+import ActivityIndicator from "@/components/indicators/activity-indicator";
+import { Input } from "@/components/ui/input";
+import { Separator } from "@/components/ui/separator";
+
+type AnnotationSettingsPaneProps = {
+  event: Event;
+  showZones: boolean;
+  setShowZones: React.Dispatch<React.SetStateAction<boolean>>;
+  annotationOffset: number;
+  setAnnotationOffset: React.Dispatch<React.SetStateAction<number>>;
+};
+export function AnnotationSettingsPane({
+  event,
+  showZones,
+  setShowZones,
+  annotationOffset,
+  setAnnotationOffset,
+}: AnnotationSettingsPaneProps) {
+  const { data: config, mutate: updateConfig } =
+    useSWR<FrigateConfig>("config");
+
+  const [isLoading, setIsLoading] = useState(false);
+
+  const formSchema = z.object({
+    annotationOffset: z.coerce.number().optional().or(z.literal("")),
+  });
+
+  const form = useForm<z.infer<typeof formSchema>>({
+    resolver: zodResolver(formSchema),
+    mode: "onChange",
+    defaultValues: {
+      annotationOffset: annotationOffset,
+    },
+  });
+
+  const saveToConfig = useCallback(
+    async (annotation_offset: number | string) => {
+      if (!config || !event) {
+        return;
+      }
+
+      axios
+        .put(
+          `config/set?cameras.${event?.camera}.detect.annotation_offset=${annotation_offset}`,
+          {
+            requires_restart: 0,
+          },
+        )
+        .then((res) => {
+          if (res.status === 200) {
+            toast.success(
+              `Annotation offset for ${event?.camera} has been saved to the config file. Restart Frigate to apply your changes.`,
+              {
+                position: "top-center",
+              },
+            );
+            updateConfig();
+          } else {
+            toast.error(`Failed to save config changes: ${res.statusText}`, {
+              position: "top-center",
+            });
+          }
+        })
+        .catch((error) => {
+          toast.error(
+            `Failed to save config changes: ${error.response.data.message}`,
+            { position: "top-center" },
+          );
+        })
+        .finally(() => {
+          setIsLoading(false);
+        });
+    },
+    [updateConfig, config, event],
+  );
+
+  function onSubmit(values: z.infer<typeof formSchema>) {
+    if (!values || values.annotationOffset == null || !config) {
+      return;
+    }
+    setIsLoading(true);
+
+    saveToConfig(values.annotationOffset);
+  }
+
+  function onApply(values: z.infer<typeof formSchema>) {
+    if (
+      !values ||
+      values.annotationOffset == null ||
+      values.annotationOffset == "" ||
+      !config
+    ) {
+      return;
+    }
+    setAnnotationOffset(values.annotationOffset);
+  }
+
+  return (
+    <div className="space-y-3 rounded-lg border border-secondary-foreground bg-background_alt p-2">
+      <Heading as="h4" className="my-2">
+        Annotation Settings
+      </Heading>
+      <div className="flex flex-col">
+        <div className="flex flex-row items-center justify-start gap-2 p-3">
+          <Switch
+            id="show-zones"
+            checked={showZones}
+            onCheckedChange={setShowZones}
+          />
+          <Label className="cursor-pointer" htmlFor="show-zones">
+            Show All Zones
+          </Label>
+        </div>
+        <div className="text-sm text-muted-foreground">
+          Always show zones on frames where objects have entered a zone.
+        </div>
+      </div>
+      <Separator className="my-2 flex bg-secondary" />
+      <Form {...form}>
+        <form
+          onSubmit={form.handleSubmit(onSubmit)}
+          className="flex flex-1 flex-col space-y-6"
+        >
+          <FormField
+            control={form.control}
+            name="annotationOffset"
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>Annotation Offset</FormLabel>
+                <div className="flex flex-col gap-8 md:flex-row-reverse">
+                  <div className="my-5 flex flex-row items-center gap-3 rounded-lg bg-destructive/50 p-3 text-sm text-primary-variant md:my-0">
+                    <PiWarningCircle className="size-24" />
+                    <div>
+                      This data comes from your camera's detect feed but is
+                      overlayed on images from the the record feed. It is
+                      unlikely that the two streams are perfectly in sync. As a
+                      result, the bounding box and the footage will not line up
+                      perfectly. However, the <code>annotation_offset</code>{" "}
+                      field in your config can be used to adjust this.
+                      <div className="mt-2 flex items-center text-primary">
+                        <Link
+                          to="https://docs.frigate.video/configuration/reference"
+                          target="_blank"
+                          rel="noopener noreferrer"
+                          className="inline"
+                        >
+                          Read the documentation{" "}
+                          <LuExternalLink className="ml-2 inline-flex size-3" />
+                        </Link>
+                      </div>
+                    </div>
+                  </div>
+                  <div className="flex flex-col">
+                    <FormControl>
+                      <Input
+                        className="w-full border border-input bg-background p-2 hover:bg-accent hover:text-accent-foreground dark:[color-scheme:dark]"
+                        placeholder="0"
+                        {...field}
+                      />
+                    </FormControl>
+                    <FormDescription>
+                      Milliseconds to offset detect annotations by.{" "}
+                      <em>Default: 0</em>
+                      <div className="mt-2">
+                        TIP: Imagine there is an event clip with a person
+                        walking from left to right. If the event timeline
+                        bounding box is consistently to the left of the person
+                        then the value should be decreased. Similarly, if a
+                        person is walking from left to right and the bounding
+                        box is consistently ahead of the person then the value
+                        should be increased.
+                      </div>
+                    </FormDescription>
+                  </div>
+                </div>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+
+          <div className="flex flex-1 flex-col justify-end">
+            <div className="flex flex-row gap-2 pt-5">
+              <Button
+                className="flex flex-1"
+                onClick={form.handleSubmit(onApply)}
+              >
+                Apply
+              </Button>
+              <Button
+                variant="select"
+                disabled={isLoading}
+                className="flex flex-1"
+                type="submit"
+              >
+                {isLoading ? (
+                  <div className="flex flex-row items-center gap-2">
+                    <ActivityIndicator />
+                    <span>Saving...</span>
+                  </div>
+                ) : (
+                  "Save"
+                )}
+              </Button>
+            </div>
+          </div>
+        </form>
+      </Form>
+    </div>
+  );
+}

web/src/components/overlay/detail/ObjectLifecycle.tsx

@@ -0,0 +1,592 @@
+import useSWR from "swr";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { Event } from "@/types/event";
+import ActivityIndicator from "@/components/indicators/activity-indicator";
+import {
+  Carousel,
+  CarouselApi,
+  CarouselContent,
+  CarouselItem,
+  CarouselNext,
+  CarouselPrevious,
+} from "@/components/ui/carousel";
+import { Button } from "@/components/ui/button";
+import { ObjectLifecycleSequence } from "@/types/timeline";
+import Heading from "@/components/ui/heading";
+import { ReviewDetailPaneType, ReviewSegment } from "@/types/review";
+import { FrigateConfig } from "@/types/frigateConfig";
+import { formatUnixTimestampToDateTime } from "@/utils/dateUtil";
+import { getIconForLabel } from "@/utils/iconUtil";
+import {
+  LuCircle,
+  LuCircleDot,
+  LuEar,
+  LuFolderX,
+  LuPlay,
+  LuPlayCircle,
+  LuSettings,
+  LuTruck,
+} from "react-icons/lu";
+import { IoMdArrowRoundBack, IoMdExit } from "react-icons/io";
+import {
+  MdFaceUnlock,
+  MdOutlineLocationOn,
+  MdOutlinePictureInPictureAlt,
+} from "react-icons/md";
+import { cn } from "@/lib/utils";
+import { Card, CardContent } from "@/components/ui/card";
+import { useApiHost } from "@/api";
+import { isDesktop, isIOS, isSafari } from "react-device-detect";
+import ImageLoadingIndicator from "@/components/indicators/ImageLoadingIndicator";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
+import { AnnotationSettingsPane } from "./AnnotationSettingsPane";
+
+type ObjectLifecycleProps = {
+  review: ReviewSegment;
+  event: Event;
+  setPane: React.Dispatch<React.SetStateAction<ReviewDetailPaneType>>;
+};
+
+export default function ObjectLifecycle({
+  review,
+  event,
+  setPane,
+}: ObjectLifecycleProps) {
+  const { data: eventSequence } = useSWR<ObjectLifecycleSequence[]>([
+    "timeline",
+    {
+      source_id: event.id,
+    },
+  ]);
+
+  const { data: config } = useSWR<FrigateConfig>("config");
+  const apiHost = useApiHost();
+
+  const [imgLoaded, setImgLoaded] = useState(false);
+  const imgRef = useRef<HTMLImageElement>(null);
+
+  const [selectedZone, setSelectedZone] = useState("");
+  const [lifecycleZones, setLifecycleZones] = useState<string[]>([]);
+  const [showControls, setShowControls] = useState(false);
+  const [showZones, setShowZones] = useState(true);
+
+  const getZoneColor = useCallback(
+    (zoneName: string) => {
+      const zoneColor =
+        config?.cameras?.[review.camera]?.zones?.[zoneName]?.color;
+      if (zoneColor) {
+        const reversed = [...zoneColor].reverse();
+        return reversed;
+      }
+    },
+    [config, review],
+  );
+
+  const getZonePolygon = useCallback(
+    (zoneName: string) => {
+      if (!imgRef.current || !config) {
+        return;
+      }
+      const zonePoints =
+        config?.cameras[review.camera].zones[zoneName].coordinates;
+      const imgElement = imgRef.current;
+      const imgRect = imgElement.getBoundingClientRect();
+
+      return zonePoints
+        .split(",")
+        .map(parseFloat)
+        .reduce((acc, value, index) => {
+          const isXCoordinate = index % 2 === 0;
+          const coordinate = isXCoordinate
+            ? value * imgRect.width
+            : value * imgRect.height;
+          acc.push(coordinate);
+          return acc;
+        }, [] as number[])
+        .join(",");
+    },
+    [config, imgRef, review],
+  );
+
+  const [boxStyle, setBoxStyle] = useState<React.CSSProperties | null>(null);
+
+  const configAnnotationOffset = useMemo(() => {
+    if (!config) {
+      return 0;
+    }
+
+    return config.cameras[event.camera]?.detect?.annotation_offset || 0;
+  }, [config, event]);
+
+  const [annotationOffset, setAnnotationOffset] = useState<number>(
+    configAnnotationOffset,
+  );
+
+  const detectArea = useMemo(() => {
+    if (!config) {
+      return 0;
+    }
+    return (
+      config.cameras[event.camera]?.detect?.width *
+      config.cameras[event.camera]?.detect?.height
+    );
+  }, [config, event.camera]);
+
+  const [timeIndex, setTimeIndex] = useState(0);
+
+  const handleSetBox = useCallback(
+    (box: number[]) => {
+      if (imgRef.current && Array.isArray(box) && box.length === 4) {
+        const imgElement = imgRef.current;
+        const imgRect = imgElement.getBoundingClientRect();
+
+        const style = {
+          left: `${box[0] * imgRect.width}px`,
+          top: `${box[1] * imgRect.height}px`,
+          width: `${box[2] * imgRect.width}px`,
+          height: `${box[3] * imgRect.height}px`,
+        };
+
+        setBoxStyle(style);
+      }
+    },
+    [imgRef],
+  );
+
+  // image
+
+  const [src, setSrc] = useState(
+    `${apiHost}api/${event.camera}/recordings/${event.start_time + annotationOffset / 1000}/snapshot.jpg?height=500`,
+  );
+  const [hasError, setHasError] = useState(false);
+
+  useEffect(() => {
+    if (timeIndex) {
+      const newSrc = `${apiHost}api/${event.camera}/recordings/${timeIndex + annotationOffset / 1000}/snapshot.jpg?height=500`;
+      setSrc(newSrc);
+    }
+    setImgLoaded(false);
+    setHasError(false);
+    // we know that these deps are correct
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [timeIndex, annotationOffset]);
+
+  // carousels
+
+  const [mainApi, setMainApi] = useState<CarouselApi>();
+  const [thumbnailApi, setThumbnailApi] = useState<CarouselApi>();
+  const [current, setCurrent] = useState(0);
+
+  const handleThumbnailClick = (index: number) => {
+    if (!mainApi || !thumbnailApi) {
+      return;
+    }
+    thumbnailApi.scrollTo(index);
+    mainApi.scrollTo(index);
+    setCurrent(index);
+  };
+
+  useEffect(() => {
+    if (eventSequence) {
+      setTimeIndex(eventSequence?.[current].timestamp);
+      handleSetBox(eventSequence?.[current].data.box ?? []);
+      setLifecycleZones(eventSequence?.[current].data.zones);
+      setSelectedZone("");
+    }
+  }, [current, imgLoaded, handleSetBox, eventSequence]);
+
+  useEffect(() => {
+    if (!mainApi || !thumbnailApi || !eventSequence || !event) {
+      return;
+    }
+
+    const handleTopSelect = () => {
+      const selected = mainApi.selectedScrollSnap();
+      setCurrent(selected);
+      thumbnailApi.scrollTo(selected);
+    };
+
+    const handleBottomSelect = () => {
+      const selected = thumbnailApi.selectedScrollSnap();
+      setCurrent(selected);
+      mainApi.scrollTo(selected);
+    };
+
+    mainApi.on("select", handleTopSelect);
+    thumbnailApi.on("select", handleBottomSelect);
+
+    return () => {
+      mainApi.off("select", handleTopSelect);
+      thumbnailApi.off("select", handleBottomSelect);
+    };
+    // we know that these deps are correct
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [mainApi, thumbnailApi]);
+
+  if (!event.id || !eventSequence || !config || !timeIndex) {
+    return <ActivityIndicator />;
+  }
+
+  return (
+    <>
+      <div className={cn("flex items-center gap-2")}>
+        <Button
+          className="flex items-center gap-2.5 rounded-lg"
+          size="sm"
+          onClick={() => setPane("overview")}
+        >
+          <IoMdArrowRoundBack className="size-5 text-secondary-foreground" />
+          {isDesktop && <div className="text-primary">Back</div>}
+        </Button>
+      </div>
+
+      <div className="relative mx-auto">
+        <ImageLoadingIndicator
+          className="absolute inset-0"
+          imgLoaded={imgLoaded}
+        />
+        {hasError && (
+          <div className="relative aspect-video">
+            <div className="flex flex-col items-center justify-center p-20 text-center">
+              <LuFolderX className="size-16" />
+              No image found for this timestamp.
+            </div>
+          </div>
+        )}
+        <div className={cn(imgLoaded ? "visible" : "invisible")}>
+          <img
+            key={event.id}
+            ref={imgRef}
+            className={cn(
+              "max-h-[50dvh] max-w-full select-none rounded-lg object-contain transition-opacity",
+            )}
+            loading={isSafari ? "eager" : "lazy"}
+            style={
+              isIOS
+                ? {
+                    WebkitUserSelect: "none",
+                    WebkitTouchCallout: "none",
+                  }
+                : undefined
+            }
+            draggable={false}
+            src={src}
+            onLoad={() => setImgLoaded(true)}
+            onError={() => setHasError(true)}
+          />
+
+          {showZones &&
+            lifecycleZones?.map((zone) => (
+              <div
+                className="absolute left-0 top-0"
+                style={{
+                  width: imgRef.current?.clientWidth,
+                  height: imgRef.current?.clientHeight,
+                }}
+                key={zone}
+              >
+                <svg
+                  viewBox={`0 0 ${imgRef.current?.width} ${imgRef.current?.height}`}
+                >
+                  <polygon
+                    points={getZonePolygon(zone)}
+                    className="fill-none stroke-2"
+                    style={{
+                      stroke: `rgb(${getZoneColor(zone)?.join(",")})`,
+                      fill:
+                        selectedZone == zone
+                          ? `rgba(${getZoneColor(zone)?.join(",")}, 0.5)`
+                          : `rgba(${getZoneColor(zone)?.join(",")}, 0.3)`,
+                      strokeWidth: selectedZone == zone ? 4 : 2,
+                    }}
+                  />
+                </svg>
+              </div>
+            ))}
+
+          {boxStyle && (
+            <div className="absolute border-2 border-red-600" style={boxStyle}>
+              <div className="absolute bottom-[-3px] left-1/2 h-[5px] w-[5px] -translate-x-1/2 transform bg-yellow-500" />
+            </div>
+          )}
+        </div>
+      </div>
+
+      <div className="mt-3 flex flex-row items-center justify-between">
+        <Heading as="h4">Object Lifecycle</Heading>
+
+        <div className="flex flex-row gap-2">
+          <Tooltip>
+            <TooltipTrigger asChild>
+              <Button
+                variant={showControls ? "select" : "default"}
+                className="size-7 p-1.5"
+              >
+                <LuSettings
+                  className="size-5"
+                  onClick={() => setShowControls(!showControls)}
+                />
+              </Button>
+            </TooltipTrigger>
+            <TooltipContent>Adjust annotation settings</TooltipContent>
+          </Tooltip>
+        </div>
+      </div>
+      <div className="flex flex-row items-center justify-between">
+        <div className="mb-2 text-sm text-muted-foreground">
+          Scroll to view the significant moments of this object's lifecycle.
+        </div>
+        <div className="min-w-20 text-right text-sm text-muted-foreground">
+          {current + 1} of {eventSequence.length}
+        </div>
+      </div>
+      {showControls && (
+        <AnnotationSettingsPane
+          event={event}
+          showZones={showZones}
+          setShowZones={setShowZones}
+          annotationOffset={annotationOffset}
+          setAnnotationOffset={setAnnotationOffset}
+        />
+      )}
+
+      <div className="relative flex flex-col items-center justify-center">
+        <Carousel className="m-0 w-full" setApi={setMainApi}>
+          <CarouselContent>
+            {eventSequence.map((item, index) => (
+              <CarouselItem key={index}>
+                <Card className="p-1 text-sm md:p-2" key={index}>
+                  <CardContent className="flex flex-row items-center gap-3 p-1 md:p-6">
+                    <div className="flex flex-1 flex-row items-center justify-start p-3 pl-1">
+                      <div
+                        className="rounded-lg p-2"
+                        style={{
+                          backgroundColor: "rgb(110,110,110)",
+                        }}
+                      >
+                        <div
+                          key={item.data.label}
+                          className="relative flex aspect-square size-4 flex-row items-center md:size-8"
+                        >
+                          {getIconForLabel(
+                            item.data.label,
+                            "size-4 md:size-6 absolute left-0 top-0",
+                          )}
+                          <LifecycleIcon
+                            className="absolute bottom-0 right-0 size-2 md:size-4"
+                            lifecycleItem={item}
+                          />
+                        </div>
+                      </div>
+                      <div className="mx-3 text-lg">
+                        <div className="flex flex-row items-center capitalize text-primary">
+                          {getLifecycleItemDescription(item)}
+                        </div>
+                        <div className="text-sm text-primary-variant">
+                          {formatUnixTimestampToDateTime(item.timestamp, {
+                            strftime_fmt:
+                              config.ui.time_format == "24hour"
+                                ? "%d %b %H:%M:%S"
+                                : "%m/%d %I:%M:%S%P",
+                            time_style: "medium",
+                            date_style: "medium",
+                          })}
+                        </div>
+                      </div>
+                    </div>
+                    <div className="flex w-5/12 flex-row items-start justify-start">
+                      <div className="text-md mr-2 w-1/3">
+                        <div className="flex flex-col items-end justify-start">
+                          <p className="mb-1.5 text-sm text-primary-variant">
+                            Zones
+                          </p>
+                          {item.class_type === "entered_zone"
+                            ? item.data.zones.map((zone, index) => (
+                                <div
+                                  key={index}
+                                  className="flex flex-row items-center gap-1"
+                                >
+                                  {true && (
+                                    <div
+                                      className="size-3 rounded-lg"
+                                      style={{
+                                        backgroundColor: `rgb(${getZoneColor(zone)})`,
+                                      }}
+                                    />
+                                  )}
+                                  <div
+                                    key={index}
+                                    className="cursor-pointer capitalize"
+                                    onClick={() => setSelectedZone(zone)}
+                                  >
+                                    {zone.replaceAll("_", " ")}
+                                  </div>
+                                </div>
+                              ))
+                            : "-"}
+                        </div>
+                      </div>
+                      <div className="text-md mr-2 w-1/3">
+                        <div className="flex flex-col items-end justify-start">
+                          <p className="mb-1.5 text-sm text-primary-variant">
+                            Ratio
+                          </p>
+                          {Array.isArray(item.data.box) &&
+                          item.data.box.length >= 4
+                            ? (item.data.box[2] / item.data.box[3]).toFixed(2)
+                            : "N/A"}
+                        </div>
+                      </div>
+                      <div className="text-md mr-2 w-1/3">
+                        <div className="flex flex-col items-end justify-start">
+                          <p className="mb-1.5 text-sm text-primary-variant">
+                            Area
+                          </p>
+                          {Array.isArray(item.data.box) &&
+                          item.data.box.length >= 4
+                            ? Math.round(
+                                detectArea *
+                                  (item.data.box[2] * item.data.box[3]),
+                              )
+                            : "N/A"}
+                        </div>
+                      </div>
+                    </div>
+                  </CardContent>
+                </Card>
+              </CarouselItem>
+            ))}
+          </CarouselContent>
+        </Carousel>
+      </div>
+      <div className="relative flex flex-col items-center justify-center">
+        <Carousel
+          opts={{
+            align: "center",
+          }}
+          className="w-full max-w-[72%] md:max-w-[85%]"
+          setApi={setThumbnailApi}
+        >
+          <CarouselContent className="flex flex-row justify-center">
+            {eventSequence.map((item, index) => (
+              <CarouselItem
+                key={index}
+                className={cn("basis-1/4 cursor-pointer md:basis-[10%]")}
+                onClick={() => handleThumbnailClick(index)}
+              >
+                <div className="p-1">
+                  <Card>
+                    <CardContent
+                      className={cn(
+                        "flex aspect-square items-center justify-center rounded-md p-2",
+                        index === current && "bg-selected",
+                      )}
+                    >
+                      <LifecycleIcon
+                        className={cn(
+                          "size-8",
+                          index === current
+                            ? "bg-selected text-white"
+                            : "text-muted-foreground",
+                        )}
+                        lifecycleItem={item}
+                      />
+                    </CardContent>
+                  </Card>
+                </div>
+              </CarouselItem>
+            ))}
+          </CarouselContent>
+          <CarouselPrevious />
+          <CarouselNext />
+        </Carousel>
+      </div>
+    </>
+  );
+}
+
+type GetTimelineIconParams = {
+  lifecycleItem: ObjectLifecycleSequence;
+  className?: string;
+};
+
+export function LifecycleIcon({
+  lifecycleItem,
+  className,
+}: GetTimelineIconParams) {
+  switch (lifecycleItem.class_type) {
+    case "visible":
+      return <LuPlay className={cn(className)} />;
+    case "gone":
+      return <IoMdExit className={cn(className)} />;
+    case "active":
+      return <LuPlayCircle className={cn(className)} />;
+    case "stationary":
+      return <LuCircle className={cn(className)} />;
+    case "entered_zone":
+      return <MdOutlineLocationOn className={cn(className)} />;
+    case "attribute":
+      switch (lifecycleItem.data?.attribute) {
+        case "face":
+          return <MdFaceUnlock className={cn(className)} />;
+        case "license_plate":
+          return <MdOutlinePictureInPictureAlt className={cn(className)} />;
+        default:
+          return <LuTruck className={cn(className)} />;
+      }
+    case "heard":
+      return <LuEar className={cn(className)} />;
+    case "external":
+      return <LuCircleDot className={cn(className)} />;
+    default:
+      return null;
+  }
+}
+
+function getLifecycleItemDescription(lifecycleItem: ObjectLifecycleSequence) {
+  const label = (
+    (Array.isArray(lifecycleItem.data.sub_label)
+      ? lifecycleItem.data.sub_label[0]
+      : lifecycleItem.data.sub_label) || lifecycleItem.data.label
+  ).replaceAll("_", " ");
+
+  switch (lifecycleItem.class_type) {
+    case "visible":
+      return `${label} detected`;
+    case "entered_zone":
+      return `${label} entered ${lifecycleItem.data.zones
+        .join(" and ")
+        .replaceAll("_", " ")}`;
+    case "active":
+      return `${label} became active`;
+    case "stationary":
+      return `${label} became stationary`;
+    case "attribute": {
+      let title = "";
+      if (
+        lifecycleItem.data.attribute == "face" ||
+        lifecycleItem.data.attribute == "license_plate"
+      ) {
+        title = `${lifecycleItem.data.attribute.replaceAll(
+          "_",
+          " ",
+        )} detected for ${label}`;
+      } else {
+        title = `${
+          lifecycleItem.data.sub_label
+        } recognized as ${lifecycleItem.data.attribute.replaceAll("_", " ")}`;
+      }
+      return title;
+    }
+    case "gone":
+      return `${label} left`;
+    case "heard":
+      return `${label} heard`;
+    case "external":
+      return `${label} detected`;
+  }
+}

web/src/components/overlay/detail/ReviewDetailDialog.tsx

@@ -1,4 +1,4 @@
-import { isDesktop, isIOS } from "react-device-detect";
+import { isDesktop, isIOS, isMobile } from "react-device-detect";
 import { Sheet, SheetContent } from "../../ui/sheet";
 import { Drawer, DrawerContent } from "../../ui/drawer";
 import useSWR from "swr";
@@ -6,11 +6,21 @@ import { FrigateConfig } from "@/types/frigateConfig";
 import { useFormattedTimestamp } from "@/hooks/use-date-utils";
 import { getIconForLabel } from "@/utils/iconUtil";
 import { useApiHost } from "@/api";
-import { ReviewSegment } from "@/types/review";
+import { ReviewDetailPaneType, ReviewSegment } from "@/types/review";
 import { Event } from "@/types/event";
-import { useMemo, useState } from "react";
+import { useMemo, useRef, useState } from "react";
 import { cn } from "@/lib/utils";
 import { FrigatePlusDialog } from "../dialog/FrigatePlusDialog";
+import ObjectLifecycle from "./ObjectLifecycle";
+import Chip from "@/components/indicators/Chip";
+import { FaDownload } from "react-icons/fa";
+import FrigatePlusIcon from "@/components/icons/FrigatePlusIcon";
+import { FaArrowsRotate } from "react-icons/fa6";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
 
 type ReviewDetailDialogProps = {
   review?: ReviewSegment;
@@ -24,8 +34,6 @@ export default function ReviewDetailDialog({
     revalidateOnFocus: false,
   });
 
-  const apiHost = useApiHost();
-
   // upload
 
   const [upload, setUpload] = useState<Event>();
@@ -53,133 +61,258 @@ export default function ReviewDetailDialog({
 
   // content
 
+  const [selectedEvent, setSelectedEvent] = useState<Event>();
+  const [pane, setPane] = useState<ReviewDetailPaneType>("overview");
+
   const Overlay = isDesktop ? Sheet : Drawer;
   const Content = isDesktop ? SheetContent : DrawerContent;
 
+  if (!review) {
+    return;
+  }
+
   return (
-    <Overlay
-      open={review != undefined}
-      onOpenChange={(open) => {
-        if (!open) {
-          setReview(undefined);
-        }
-      }}
-    >
-      <FrigatePlusDialog
-        upload={upload}
-        onClose={() => setUpload(undefined)}
-        onEventUploaded={() => {
-          if (upload) {
-            upload.plus_id = "new_upload";
+    <>
+      <Overlay
+        open={review != undefined}
+        onOpenChange={(open) => {
+          if (!open) {
+            setReview(undefined);
+            setSelectedEvent(undefined);
+            setPane("overview");
           }
         }}
-      />
-
-      <Content
-        className={
-          isDesktop ? "sm:max-w-xl" : "max-h-[75dvh] overflow-hidden p-2 pb-4"
-        }
       >
-        {review && (
-          <div className="scrollbar-container mt-3 flex size-full flex-col gap-5 overflow-y-auto md:mt-0">
-            <div className="flex w-full flex-row">
-              <div className="flex w-full flex-col gap-3">
-                <div className="flex flex-col gap-1.5">
-                  <div className="text-sm text-primary/40">Camera</div>
-                  <div className="text-sm capitalize">
-                    {review.camera.replaceAll("_", " ")}
-                  </div>
-                </div>
-                <div className="flex flex-col gap-1.5">
-                  <div className="text-sm text-primary/40">Timestamp</div>
-                  <div className="text-sm">{formattedDate}</div>
-                </div>
-              </div>
-              <div className="flex w-full flex-col gap-2 px-6">
-                <div className="flex flex-col gap-1.5">
-                  <div className="text-sm text-primary/40">Objects</div>
-                  <div className="flex flex-col items-start gap-2 text-sm capitalize">
-                    {events?.map((event) => {
-                      return (
-                        <div
-                          key={event.id}
-                          className="flex flex-row items-center gap-2 text-sm capitalize"
-                        >
-                          {getIconForLabel(event.label, "size-3 text-white")}
-                          {event.sub_label ?? event.label} (
-                          {Math.round(event.data.top_score * 100)}%)
-                        </div>
-                      );
-                    })}
-                  </div>
-                </div>
-                {review.data.zones.length > 0 && (
+        <FrigatePlusDialog
+          upload={upload}
+          onClose={() => setUpload(undefined)}
+          onEventUploaded={() => {
+            if (upload) {
+              upload.plus_id = "new_upload";
+            }
+          }}
+        />
+
+        <Content
+          className={cn(
+            isDesktop
+              ? pane == "overview"
+                ? "sm:max-w-xl"
+                : "pt-2 sm:max-w-4xl"
+              : "max-h-[80dvh] overflow-hidden p-2 pb-4",
+          )}
+        >
+          {pane == "overview" && (
+            <div className="scrollbar-container mt-3 flex size-full flex-col gap-5 overflow-y-auto">
+              <div className="flex w-full flex-row">
+                <div className="flex w-full flex-col gap-3">
                   <div className="flex flex-col gap-1.5">
-                    <div className="text-sm text-primary/40">Zones</div>
+                    <div className="text-sm text-primary/40">Camera</div>
+                    <div className="text-sm capitalize">
+                      {review.camera.replaceAll("_", " ")}
+                    </div>
+                  </div>
+                  <div className="flex flex-col gap-1.5">
+                    <div className="text-sm text-primary/40">Timestamp</div>
+                    <div className="text-sm">{formattedDate}</div>
+                  </div>
+                </div>
+                <div className="flex w-full flex-col gap-2">
+                  <div className="flex flex-col gap-1.5">
+                    <div className="text-sm text-primary/40">Objects</div>
                     <div className="flex flex-col items-start gap-2 text-sm capitalize">
-                      {review.data.zones.map((zone) => {
+                      {events?.map((event) => {
                         return (
                           <div
-                            key={zone}
-                            className="flex flex-row items-center gap-2 text-sm capitalize"
+                            key={event.id}
+                            className="flex flex-row items-center gap-2 capitalize"
                           >
-                            {zone.replaceAll("_", " ")}
+                            {getIconForLabel(
+                              event.label,
+                              "size-3 text-primary",
+                            )}
+                            {event.sub_label ?? event.label} (
+                            {Math.round(event.data.top_score * 100)}%)
                           </div>
                         );
                       })}
                     </div>
                   </div>
-                )}
+                  {review.data.zones.length > 0 && (
+                    <div className="flex flex-col gap-1.5">
+                      <div className="text-sm text-primary/40">Zones</div>
+                      <div className="flex flex-col items-start gap-2 text-sm capitalize">
+                        {review.data.zones.map((zone) => {
+                          return (
+                            <div
+                              key={zone}
+                              className="flex flex-row items-center gap-2 capitalize"
+                            >
+                              {zone.replaceAll("_", " ")}
+                            </div>
+                          );
+                        })}
+                      </div>
+                    </div>
+                  )}
+                </div>
+              </div>
+              {hasMismatch && (
+                <div className="p-4 text-center text-sm">
+                  Some objects that were detected are not included in this list
+                  because the object does not have a snapshot
+                </div>
+              )}
+              <div className="relative flex size-full flex-col gap-2">
+                {events?.map((event) => (
+                  <EventItem
+                    key={event.id}
+                    event={event}
+                    setPane={setPane}
+                    setSelectedEvent={setSelectedEvent}
+                    setUpload={setUpload}
+                  />
+                ))}
               </div>
             </div>
-            {hasMismatch && (
-              <div className="p-4 text-center text-sm">
-                Some objects that were detected are not included in this list
-                because the object does not have a snapshot
-              </div>
-            )}
-            <div className="flex size-full flex-col gap-2 px-6">
-              {events?.map((event) => {
-                return (
-                  <img
-                    key={event.id}
-                    className={cn(
-                      "aspect-video select-none rounded-lg object-contain transition-opacity",
-                      event.has_snapshot &&
-                        event.plus_id == undefined &&
-                        config?.plus.enabled &&
-                        "cursor-pointer",
-                    )}
-                    style={
-                      isIOS
-                        ? {
-                            WebkitUserSelect: "none",
-                            WebkitTouchCallout: "none",
-                          }
-                        : undefined
-                    }
-                    draggable={false}
-                    src={
+          )}
+
+          {pane == "details" && selectedEvent && (
+            <div className="scrollbar-container overflow-x-none mt-0 flex size-full flex-col gap-2 overflow-y-auto overflow-x-hidden">
+              <ObjectLifecycle
+                review={review}
+                event={selectedEvent}
+                setPane={setPane}
+              />
+            </div>
+          )}
+        </Content>
+      </Overlay>
+    </>
+  );
+}
+
+type EventItemProps = {
+  event: Event;
+  setPane: React.Dispatch<React.SetStateAction<ReviewDetailPaneType>>;
+  setSelectedEvent: React.Dispatch<React.SetStateAction<Event | undefined>>;
+  setUpload?: React.Dispatch<React.SetStateAction<Event | undefined>>;
+};
+
+function EventItem({
+  event,
+  setPane,
+  setSelectedEvent,
+  setUpload,
+}: EventItemProps) {
+  const { data: config } = useSWR<FrigateConfig>("config", {
+    revalidateOnFocus: false,
+  });
+
+  const apiHost = useApiHost();
+
+  const imgRef = useRef(null);
+
+  const [hovered, setHovered] = useState(isMobile);
+
+  return (
+    <>
+      <div
+        className={cn(
+          "relative",
+          !event.has_snapshot && "flex flex-row items-center justify-center",
+        )}
+        onMouseEnter={isDesktop ? () => setHovered(true) : undefined}
+        onMouseLeave={isDesktop ? () => setHovered(false) : undefined}
+        key={event.id}
+      >
+        {event.has_snapshot && (
+          <>
+            <div className="pointer-events-none absolute inset-x-0 top-0 h-[30%] w-full rounded-lg bg-gradient-to-b from-black/20 to-transparent md:rounded-2xl"></div>
+            <div className="pointer-events-none absolute inset-x-0 bottom-0 z-10 h-[10%] w-full rounded-lg bg-gradient-to-t from-black/20 to-transparent md:rounded-2xl"></div>
+          </>
+        )}
+        <img
+          ref={imgRef}
+          className={cn(
+            "select-none rounded-lg object-contain transition-opacity",
+          )}
+          style={
+            isIOS
+              ? {
+                  WebkitUserSelect: "none",
+                  WebkitTouchCallout: "none",
+                }
+              : undefined
+          }
+          draggable={false}
+          src={
+            event.has_snapshot
+              ? `${apiHost}api/events/${event.id}/snapshot.jpg`
+              : `${apiHost}api/events/${event.id}/thumbnail.jpg`
+          }
+        />
+        {hovered && (
+          <div>
+            <div
+              className={cn("absolute right-1 top-1 flex items-center gap-2")}
+            >
+              <Tooltip>
+                <TooltipTrigger asChild>
+                  <a
+                    download
+                    href={
                       event.has_snapshot
                         ? `${apiHost}api/events/${event.id}/snapshot.jpg`
                         : `${apiHost}api/events/${event.id}/thumbnail.jpg`
                     }
-                    onClick={() => {
-                      if (
-                        event.has_snapshot &&
-                        event.plus_id == undefined &&
-                        config?.plus.enabled
-                      ) {
-                        setUpload(event);
-                      }
-                    }}
-                  />
-                );
-              })}
+                  >
+                    <Chip className="cursor-pointer rounded-md bg-gray-500 bg-gradient-to-br from-gray-400 to-gray-500">
+                      <FaDownload className="size-4 text-white" />
+                    </Chip>
+                  </a>
+                </TooltipTrigger>
+                <TooltipContent>Download</TooltipContent>
+              </Tooltip>
+
+              {event.has_snapshot &&
+                event.plus_id == undefined &&
+                config?.plus.enabled && (
+                  <Tooltip>
+                    <TooltipTrigger>
+                      <Chip
+                        className="cursor-pointer rounded-md bg-gray-500 bg-gradient-to-br from-gray-400 to-gray-500"
+                        onClick={() => {
+                          setUpload?.(event);
+                        }}
+                      >
+                        <FrigatePlusIcon className="size-4 text-white" />
+                      </Chip>
+                    </TooltipTrigger>
+                    <TooltipContent>Submit to Frigate+</TooltipContent>
+                  </Tooltip>
+                )}
+
+              {event.has_clip && (
+                <Tooltip>
+                  <TooltipTrigger>
+                    <Chip
+                      className="cursor-pointer rounded-md bg-gray-500 bg-gradient-to-br from-gray-400 to-gray-500"
+                      onClick={() => {
+                        setPane("details");
+                        setSelectedEvent(event);
+                      }}
+                    >
+                      <FaArrowsRotate className="size-4 text-white" />
+                    </Chip>
+                  </TooltipTrigger>
+                  <TooltipContent>View Object Lifecycle</TooltipContent>
+                </Tooltip>
+              )}
             </div>
           </div>
         )}
-      </Content>
-    </Overlay>
+      </div>
+    </>
   );
 }