forked from Github/frigate
Implement object lifecycle pane (#13550)
* Object lifecycle pane * fix thumbnails and annotation offset math * snapshot endpoint height and format, yaml types, bugfixes * clean up for new type * use get_image_from_recording in recordings snapshot api * make height optional
This commit is contained in:
Josh Hawkins
592
web/src/components/overlay/detail/ObjectLifecycle.tsx
Normal file
592
web/src/components/overlay/detail/ObjectLifecycle.tsx
Normal file
@@ -0,0 +1,592 @@
|
||||
import useSWR from "swr";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { Event } from "@/types/event";
|
||||
import ActivityIndicator from "@/components/indicators/activity-indicator";
|
||||
import {
|
||||
Carousel,
|
||||
CarouselApi,
|
||||
CarouselContent,
|
||||
CarouselItem,
|
||||
CarouselNext,
|
||||
CarouselPrevious,
|
||||
} from "@/components/ui/carousel";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { ObjectLifecycleSequence } from "@/types/timeline";
|
||||
import Heading from "@/components/ui/heading";
|
||||
import { ReviewDetailPaneType, ReviewSegment } from "@/types/review";
|
||||
import { FrigateConfig } from "@/types/frigateConfig";
|
||||
import { formatUnixTimestampToDateTime } from "@/utils/dateUtil";
|
||||
import { getIconForLabel } from "@/utils/iconUtil";
|
||||
import {
|
||||
LuCircle,
|
||||
LuCircleDot,
|
||||
LuEar,
|
||||
LuFolderX,
|
||||
LuPlay,
|
||||
LuPlayCircle,
|
||||
LuSettings,
|
||||
LuTruck,
|
||||
} from "react-icons/lu";
|
||||
import { IoMdArrowRoundBack, IoMdExit } from "react-icons/io";
|
||||
import {
|
||||
MdFaceUnlock,
|
||||
MdOutlineLocationOn,
|
||||
MdOutlinePictureInPictureAlt,
|
||||
} from "react-icons/md";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { Card, CardContent } from "@/components/ui/card";
|
||||
import { useApiHost } from "@/api";
|
||||
import { isDesktop, isIOS, isSafari } from "react-device-detect";
|
||||
import ImageLoadingIndicator from "@/components/indicators/ImageLoadingIndicator";
|
||||
import {
|
||||
Tooltip,
|
||||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip";
|
||||
import { AnnotationSettingsPane } from "./AnnotationSettingsPane";
|
||||
|
||||
type ObjectLifecycleProps = {
|
||||
review: ReviewSegment;
|
||||
event: Event;
|
||||
setPane: React.Dispatch<React.SetStateAction<ReviewDetailPaneType>>;
|
||||
};
|
||||
|
||||
export default function ObjectLifecycle({
|
||||
review,
|
||||
event,
|
||||
setPane,
|
||||
}: ObjectLifecycleProps) {
|
||||
const { data: eventSequence } = useSWR<ObjectLifecycleSequence[]>([
|
||||
"timeline",
|
||||
{
|
||||
source_id: event.id,
|
||||
},
|
||||
]);
|
||||
|
||||
const { data: config } = useSWR<FrigateConfig>("config");
|
||||
const apiHost = useApiHost();
|
||||
|
||||
const [imgLoaded, setImgLoaded] = useState(false);
|
||||
const imgRef = useRef<HTMLImageElement>(null);
|
||||
|
||||
const [selectedZone, setSelectedZone] = useState("");
|
||||
const [lifecycleZones, setLifecycleZones] = useState<string[]>([]);
|
||||
const [showControls, setShowControls] = useState(false);
|
||||
const [showZones, setShowZones] = useState(true);
|
||||
|
||||
const getZoneColor = useCallback(
|
||||
(zoneName: string) => {
|
||||
const zoneColor =
|
||||
config?.cameras?.[review.camera]?.zones?.[zoneName]?.color;
|
||||
if (zoneColor) {
|
||||
const reversed = [...zoneColor].reverse();
|
||||
return reversed;
|
||||
}
|
||||
},
|
||||
[config, review],
|
||||
);
|
||||
|
||||
const getZonePolygon = useCallback(
|
||||
(zoneName: string) => {
|
||||
if (!imgRef.current || !config) {
|
||||
return;
|
||||
}
|
||||
const zonePoints =
|
||||
config?.cameras[review.camera].zones[zoneName].coordinates;
|
||||
const imgElement = imgRef.current;
|
||||
const imgRect = imgElement.getBoundingClientRect();
|
||||
|
||||
return zonePoints
|
||||
.split(",")
|
||||
.map(parseFloat)
|
||||
.reduce((acc, value, index) => {
|
||||
const isXCoordinate = index % 2 === 0;
|
||||
const coordinate = isXCoordinate
|
||||
? value * imgRect.width
|
||||
: value * imgRect.height;
|
||||
acc.push(coordinate);
|
||||
return acc;
|
||||
}, [] as number[])
|
||||
.join(",");
|
||||
},
|
||||
[config, imgRef, review],
|
||||
);
|
||||
|
||||
const [boxStyle, setBoxStyle] = useState<React.CSSProperties | null>(null);
|
||||
|
||||
const configAnnotationOffset = useMemo(() => {
|
||||
if (!config) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return config.cameras[event.camera]?.detect?.annotation_offset || 0;
|
||||
}, [config, event]);
|
||||
|
||||
const [annotationOffset, setAnnotationOffset] = useState<number>(
|
||||
configAnnotationOffset,
|
||||
);
|
||||
|
||||
const detectArea = useMemo(() => {
|
||||
if (!config) {
|
||||
return 0;
|
||||
}
|
||||
return (
|
||||
config.cameras[event.camera]?.detect?.width *
|
||||
config.cameras[event.camera]?.detect?.height
|
||||
);
|
||||
}, [config, event.camera]);
|
||||
|
||||
const [timeIndex, setTimeIndex] = useState(0);
|
||||
|
||||
const handleSetBox = useCallback(
|
||||
(box: number[]) => {
|
||||
if (imgRef.current && Array.isArray(box) && box.length === 4) {
|
||||
const imgElement = imgRef.current;
|
||||
const imgRect = imgElement.getBoundingClientRect();
|
||||
|
||||
const style = {
|
||||
left: `${box[0] * imgRect.width}px`,
|
||||
top: `${box[1] * imgRect.height}px`,
|
||||
width: `${box[2] * imgRect.width}px`,
|
||||
height: `${box[3] * imgRect.height}px`,
|
||||
};
|
||||
|
||||
setBoxStyle(style);
|
||||
}
|
||||
},
|
||||
[imgRef],
|
||||
);
|
||||
|
||||
// image
|
||||
|
||||
const [src, setSrc] = useState(
|
||||
`${apiHost}api/${event.camera}/recordings/${event.start_time + annotationOffset / 1000}/snapshot.jpg?height=500`,
|
||||
);
|
||||
const [hasError, setHasError] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (timeIndex) {
|
||||
const newSrc = `${apiHost}api/${event.camera}/recordings/${timeIndex + annotationOffset / 1000}/snapshot.jpg?height=500`;
|
||||
setSrc(newSrc);
|
||||
}
|
||||
setImgLoaded(false);
|
||||
setHasError(false);
|
||||
// we know that these deps are correct
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [timeIndex, annotationOffset]);
|
||||
|
||||
// carousels
|
||||
|
||||
const [mainApi, setMainApi] = useState<CarouselApi>();
|
||||
const [thumbnailApi, setThumbnailApi] = useState<CarouselApi>();
|
||||
const [current, setCurrent] = useState(0);
|
||||
|
||||
const handleThumbnailClick = (index: number) => {
|
||||
if (!mainApi || !thumbnailApi) {
|
||||
return;
|
||||
}
|
||||
thumbnailApi.scrollTo(index);
|
||||
mainApi.scrollTo(index);
|
||||
setCurrent(index);
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (eventSequence) {
|
||||
setTimeIndex(eventSequence?.[current].timestamp);
|
||||
handleSetBox(eventSequence?.[current].data.box ?? []);
|
||||
setLifecycleZones(eventSequence?.[current].data.zones);
|
||||
setSelectedZone("");
|
||||
}
|
||||
}, [current, imgLoaded, handleSetBox, eventSequence]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!mainApi || !thumbnailApi || !eventSequence || !event) {
|
||||
return;
|
||||
}
|
||||
|
||||
const handleTopSelect = () => {
|
||||
const selected = mainApi.selectedScrollSnap();
|
||||
setCurrent(selected);
|
||||
thumbnailApi.scrollTo(selected);
|
||||
};
|
||||
|
||||
const handleBottomSelect = () => {
|
||||
const selected = thumbnailApi.selectedScrollSnap();
|
||||
setCurrent(selected);
|
||||
mainApi.scrollTo(selected);
|
||||
};
|
||||
|
||||
mainApi.on("select", handleTopSelect);
|
||||
thumbnailApi.on("select", handleBottomSelect);
|
||||
|
||||
return () => {
|
||||
mainApi.off("select", handleTopSelect);
|
||||
thumbnailApi.off("select", handleBottomSelect);
|
||||
};
|
||||
// we know that these deps are correct
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [mainApi, thumbnailApi]);
|
||||
|
||||
if (!event.id || !eventSequence || !config || !timeIndex) {
|
||||
return <ActivityIndicator />;
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className={cn("flex items-center gap-2")}>
|
||||
<Button
|
||||
className="flex items-center gap-2.5 rounded-lg"
|
||||
size="sm"
|
||||
onClick={() => setPane("overview")}
|
||||
>
|
||||
<IoMdArrowRoundBack className="size-5 text-secondary-foreground" />
|
||||
{isDesktop && <div className="text-primary">Back</div>}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<div className="relative mx-auto">
|
||||
<ImageLoadingIndicator
|
||||
className="absolute inset-0"
|
||||
imgLoaded={imgLoaded}
|
||||
/>
|
||||
{hasError && (
|
||||
<div className="relative aspect-video">
|
||||
<div className="flex flex-col items-center justify-center p-20 text-center">
|
||||
<LuFolderX className="size-16" />
|
||||
No image found for this timestamp.
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<div className={cn(imgLoaded ? "visible" : "invisible")}>
|
||||
<img
|
||||
key={event.id}
|
||||
ref={imgRef}
|
||||
className={cn(
|
||||
"max-h-[50dvh] max-w-full select-none rounded-lg object-contain transition-opacity",
|
||||
)}
|
||||
loading={isSafari ? "eager" : "lazy"}
|
||||
style={
|
||||
isIOS
|
||||
? {
|
||||
WebkitUserSelect: "none",
|
||||
WebkitTouchCallout: "none",
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
draggable={false}
|
||||
src={src}
|
||||
onLoad={() => setImgLoaded(true)}
|
||||
onError={() => setHasError(true)}
|
||||
/>
|
||||
|
||||
{showZones &&
|
||||
lifecycleZones?.map((zone) => (
|
||||
<div
|
||||
className="absolute left-0 top-0"
|
||||
style={{
|
||||
width: imgRef.current?.clientWidth,
|
||||
height: imgRef.current?.clientHeight,
|
||||
}}
|
||||
key={zone}
|
||||
>
|
||||
<svg
|
||||
viewBox={`0 0 ${imgRef.current?.width} ${imgRef.current?.height}`}
|
||||
>
|
||||
<polygon
|
||||
points={getZonePolygon(zone)}
|
||||
className="fill-none stroke-2"
|
||||
style={{
|
||||
stroke: `rgb(${getZoneColor(zone)?.join(",")})`,
|
||||
fill:
|
||||
selectedZone == zone
|
||||
? `rgba(${getZoneColor(zone)?.join(",")}, 0.5)`
|
||||
: `rgba(${getZoneColor(zone)?.join(",")}, 0.3)`,
|
||||
strokeWidth: selectedZone == zone ? 4 : 2,
|
||||
}}
|
||||
/>
|
||||
</svg>
|
||||
</div>
|
||||
))}
|
||||
|
||||
{boxStyle && (
|
||||
<div className="absolute border-2 border-red-600" style={boxStyle}>
|
||||
<div className="absolute bottom-[-3px] left-1/2 h-[5px] w-[5px] -translate-x-1/2 transform bg-yellow-500" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="mt-3 flex flex-row items-center justify-between">
|
||||
<Heading as="h4">Object Lifecycle</Heading>
|
||||
|
||||
<div className="flex flex-row gap-2">
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
variant={showControls ? "select" : "default"}
|
||||
className="size-7 p-1.5"
|
||||
>
|
||||
<LuSettings
|
||||
className="size-5"
|
||||
onClick={() => setShowControls(!showControls)}
|
||||
/>
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>Adjust annotation settings</TooltipContent>
|
||||
</Tooltip>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex flex-row items-center justify-between">
|
||||
<div className="mb-2 text-sm text-muted-foreground">
|
||||
Scroll to view the significant moments of this object's lifecycle.
|
||||
</div>
|
||||
<div className="min-w-20 text-right text-sm text-muted-foreground">
|
||||
{current + 1} of {eventSequence.length}
|
||||
</div>
|
||||
</div>
|
||||
{showControls && (
|
||||
<AnnotationSettingsPane
|
||||
event={event}
|
||||
showZones={showZones}
|
||||
setShowZones={setShowZones}
|
||||
annotationOffset={annotationOffset}
|
||||
setAnnotationOffset={setAnnotationOffset}
|
||||
/>
|
||||
)}
|
||||
|
||||
<div className="relative flex flex-col items-center justify-center">
|
||||
<Carousel className="m-0 w-full" setApi={setMainApi}>
|
||||
<CarouselContent>
|
||||
{eventSequence.map((item, index) => (
|
||||
<CarouselItem key={index}>
|
||||
<Card className="p-1 text-sm md:p-2" key={index}>
|
||||
<CardContent className="flex flex-row items-center gap-3 p-1 md:p-6">
|
||||
<div className="flex flex-1 flex-row items-center justify-start p-3 pl-1">
|
||||
<div
|
||||
className="rounded-lg p-2"
|
||||
style={{
|
||||
backgroundColor: "rgb(110,110,110)",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
key={item.data.label}
|
||||
className="relative flex aspect-square size-4 flex-row items-center md:size-8"
|
||||
>
|
||||
{getIconForLabel(
|
||||
item.data.label,
|
||||
"size-4 md:size-6 absolute left-0 top-0",
|
||||
)}
|
||||
<LifecycleIcon
|
||||
className="absolute bottom-0 right-0 size-2 md:size-4"
|
||||
lifecycleItem={item}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="mx-3 text-lg">
|
||||
<div className="flex flex-row items-center capitalize text-primary">
|
||||
{getLifecycleItemDescription(item)}
|
||||
</div>
|
||||
<div className="text-sm text-primary-variant">
|
||||
{formatUnixTimestampToDateTime(item.timestamp, {
|
||||
strftime_fmt:
|
||||
config.ui.time_format == "24hour"
|
||||
? "%d %b %H:%M:%S"
|
||||
: "%m/%d %I:%M:%S%P",
|
||||
time_style: "medium",
|
||||
date_style: "medium",
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex w-5/12 flex-row items-start justify-start">
|
||||
<div className="text-md mr-2 w-1/3">
|
||||
<div className="flex flex-col items-end justify-start">
|
||||
<p className="mb-1.5 text-sm text-primary-variant">
|
||||
Zones
|
||||
</p>
|
||||
{item.class_type === "entered_zone"
|
||||
? item.data.zones.map((zone, index) => (
|
||||
<div
|
||||
key={index}
|
||||
className="flex flex-row items-center gap-1"
|
||||
>
|
||||
{true && (
|
||||
<div
|
||||
className="size-3 rounded-lg"
|
||||
style={{
|
||||
backgroundColor: `rgb(${getZoneColor(zone)})`,
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
<div
|
||||
key={index}
|
||||
className="cursor-pointer capitalize"
|
||||
onClick={() => setSelectedZone(zone)}
|
||||
>
|
||||
{zone.replaceAll("_", " ")}
|
||||
</div>
|
||||
</div>
|
||||
))
|
||||
: "-"}
|
||||
</div>
|
||||
</div>
|
||||
<div className="text-md mr-2 w-1/3">
|
||||
<div className="flex flex-col items-end justify-start">
|
||||
<p className="mb-1.5 text-sm text-primary-variant">
|
||||
Ratio
|
||||
</p>
|
||||
{Array.isArray(item.data.box) &&
|
||||
item.data.box.length >= 4
|
||||
? (item.data.box[2] / item.data.box[3]).toFixed(2)
|
||||
: "N/A"}
|
||||
</div>
|
||||
</div>
|
||||
<div className="text-md mr-2 w-1/3">
|
||||
<div className="flex flex-col items-end justify-start">
|
||||
<p className="mb-1.5 text-sm text-primary-variant">
|
||||
Area
|
||||
</p>
|
||||
{Array.isArray(item.data.box) &&
|
||||
item.data.box.length >= 4
|
||||
? Math.round(
|
||||
detectArea *
|
||||
(item.data.box[2] * item.data.box[3]),
|
||||
)
|
||||
: "N/A"}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</CarouselItem>
|
||||
))}
|
||||
</CarouselContent>
|
||||
</Carousel>
|
||||
</div>
|
||||
<div className="relative flex flex-col items-center justify-center">
|
||||
<Carousel
|
||||
opts={{
|
||||
align: "center",
|
||||
}}
|
||||
className="w-full max-w-[72%] md:max-w-[85%]"
|
||||
setApi={setThumbnailApi}
|
||||
>
|
||||
<CarouselContent className="flex flex-row justify-center">
|
||||
{eventSequence.map((item, index) => (
|
||||
<CarouselItem
|
||||
key={index}
|
||||
className={cn("basis-1/4 cursor-pointer md:basis-[10%]")}
|
||||
onClick={() => handleThumbnailClick(index)}
|
||||
>
|
||||
<div className="p-1">
|
||||
<Card>
|
||||
<CardContent
|
||||
className={cn(
|
||||
"flex aspect-square items-center justify-center rounded-md p-2",
|
||||
index === current && "bg-selected",
|
||||
)}
|
||||
>
|
||||
<LifecycleIcon
|
||||
className={cn(
|
||||
"size-8",
|
||||
index === current
|
||||
? "bg-selected text-white"
|
||||
: "text-muted-foreground",
|
||||
)}
|
||||
lifecycleItem={item}
|
||||
/>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
</CarouselItem>
|
||||
))}
|
||||
</CarouselContent>
|
||||
<CarouselPrevious />
|
||||
<CarouselNext />
|
||||
</Carousel>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
type GetTimelineIconParams = {
|
||||
lifecycleItem: ObjectLifecycleSequence;
|
||||
className?: string;
|
||||
};
|
||||
|
||||
export function LifecycleIcon({
|
||||
lifecycleItem,
|
||||
className,
|
||||
}: GetTimelineIconParams) {
|
||||
switch (lifecycleItem.class_type) {
|
||||
case "visible":
|
||||
return <LuPlay className={cn(className)} />;
|
||||
case "gone":
|
||||
return <IoMdExit className={cn(className)} />;
|
||||
case "active":
|
||||
return <LuPlayCircle className={cn(className)} />;
|
||||
case "stationary":
|
||||
return <LuCircle className={cn(className)} />;
|
||||
case "entered_zone":
|
||||
return <MdOutlineLocationOn className={cn(className)} />;
|
||||
case "attribute":
|
||||
switch (lifecycleItem.data?.attribute) {
|
||||
case "face":
|
||||
return <MdFaceUnlock className={cn(className)} />;
|
||||
case "license_plate":
|
||||
return <MdOutlinePictureInPictureAlt className={cn(className)} />;
|
||||
default:
|
||||
return <LuTruck className={cn(className)} />;
|
||||
}
|
||||
case "heard":
|
||||
return <LuEar className={cn(className)} />;
|
||||
case "external":
|
||||
return <LuCircleDot className={cn(className)} />;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function getLifecycleItemDescription(lifecycleItem: ObjectLifecycleSequence) {
|
||||
const label = (
|
||||
(Array.isArray(lifecycleItem.data.sub_label)
|
||||
? lifecycleItem.data.sub_label[0]
|
||||
: lifecycleItem.data.sub_label) || lifecycleItem.data.label
|
||||
).replaceAll("_", " ");
|
||||
|
||||
switch (lifecycleItem.class_type) {
|
||||
case "visible":
|
||||
return `${label} detected`;
|
||||
case "entered_zone":
|
||||
return `${label} entered ${lifecycleItem.data.zones
|
||||
.join(" and ")
|
||||
.replaceAll("_", " ")}`;
|
||||
case "active":
|
||||
return `${label} became active`;
|
||||
case "stationary":
|
||||
return `${label} became stationary`;
|
||||
case "attribute": {
|
||||
let title = "";
|
||||
if (
|
||||
lifecycleItem.data.attribute == "face" ||
|
||||
lifecycleItem.data.attribute == "license_plate"
|
||||
) {
|
||||
title = `${lifecycleItem.data.attribute.replaceAll(
|
||||
"_",
|
||||
" ",
|
||||
)} detected for ${label}`;
|
||||
} else {
|
||||
title = `${
|
||||
lifecycleItem.data.sub_label
|
||||
} recognized as ${lifecycleItem.data.attribute.replaceAll("_", " ")}`;
|
||||
}
|
||||
return title;
|
||||
}
|
||||
case "gone":
|
||||
return `${label} left`;
|
||||
case "heard":
|
||||
return `${label} heard`;
|
||||
case "external":
|
||||
return `${label} detected`;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user