Review segment UI (#9945)

* Add ui for events

* Display data for review items

* Use preview thumbnails

* Implement paging

* Show icons for what was detected

* Show progress bar on preview thumbnail

* Hide the overlays on hover and update reviewed status

* Dim previews that have been reviewed

* Show audio icons

* Cleanup preview thumb player

* initial implementation of review timeline

* Use timeout for hover playback

* Break apart mobile and desktop views

* Show icons for sub labels

* autoplay first video on mobile

* Only show the last 24 hours by default

* Rework scrolling to fix nested scrolling

* Final scroll cleanups

---------

Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
This commit is contained in:
Nicolas Mowen
2024-02-21 13:07:32 -07:00
committed by GitHub
parent be4b570346
commit 509e46adc8
21 changed files with 1262 additions and 277 deletions

View File

@@ -5,7 +5,7 @@ type TWrapperProps = {
};
const Wrapper = ({ children }: TWrapperProps) => {
return <main className="flex flex-col h-screen">{children}</main>;
return <main className="w-screen h-screen overflow-hidden">{children}</main>;
};
export default Wrapper;

View File

@@ -1,5 +1,4 @@
import useSWR from "swr";
import PreviewThumbnailPlayer from "../player/PreviewThumbnailPlayer";
import { Card } from "../ui/card";
import { FrigateConfig } from "@/types/frigateConfig";
import ActivityIndicator from "../ui/activity-indicator";
@@ -21,8 +20,10 @@ type HistoryCardProps = {
};
export default function HistoryCard({
// @ts-ignore
relevantPreview,
timeline,
// @ts-ignore
isMobile,
onClick,
onDelete,
@@ -38,14 +39,6 @@ export default function HistoryCard({
className="cursor-pointer my-2 xs:mr-2 w-full xs:w-[48%] sm:w-[284px]"
onClick={onClick}
>
<PreviewThumbnailPlayer
camera={timeline.camera}
relevantPreview={relevantPreview}
startTs={Object.values(timeline.entries)[0].timestamp}
eventId={Object.values(timeline.entries)[0].source_id}
isMobile={isMobile}
onClick={onClick}
/>
<>
<div className="text-sm flex justify-between items-center">
<div className="pl-1 pt-1">

View File

@@ -2,23 +2,26 @@ import VideoPlayer from "./VideoPlayer";
import React, {
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from "react";
import { useApiHost } from "@/api";
import Player from "video.js/dist/types/player";
import { AspectRatio } from "../ui/aspect-ratio";
import { LuPlayCircle } from "react-icons/lu";
import { isCurrentHour } from "@/utils/dateUtil";
import { isSafari } from "@/utils/browserUtil";
import { formatUnixTimestampToDateTime } from "@/utils/dateUtil";
import { ReviewSegment } from "@/types/review";
import { Slider } from "../ui/slider";
import { getIconForLabel, getIconForSubLabel } from "@/utils/iconUtil";
import TimeAgo from "../dynamic/TimeAgo";
import useSWR from "swr";
import { FrigateConfig } from "@/types/frigateConfig";
import { isMobile, isSafari } from "react-device-detect";
type PreviewPlayerProps = {
camera: string;
review: ReviewSegment;
relevantPreview?: Preview;
startTs: number;
eventId: string;
isMobile: boolean;
onClick?: () => void;
autoPlayback?: boolean;
setReviewed?: () => void;
};
type Preview = {
@@ -30,17 +33,43 @@ type Preview = {
};
export default function PreviewThumbnailPlayer({
camera,
review,
relevantPreview,
startTs,
eventId,
isMobile,
onClick,
autoPlayback = false,
setReviewed,
}: PreviewPlayerProps) {
const apiHost = useApiHost();
const { data: config } = useSWR<FrigateConfig>("config");
const playerRef = useRef<Player | null>(null);
const [visible, setVisible] = useState(false);
const [isInitiallyVisible, setIsInitiallyVisible] = useState(false);
const [hoverTimeout, setHoverTimeout] = useState<NodeJS.Timeout | null>();
const [playback, setPlayback] = useState(false);
const [progress, setProgress] = useState(0);
const playingBack = useMemo(
() => relevantPreview && playback,
[playback, autoPlayback, relevantPreview]
);
useEffect(() => {
if (!autoPlayback) {
setPlayback(false);
if (hoverTimeout) {
clearTimeout(hoverTimeout);
}
return;
}
const timeout = setTimeout(() => {
setPlayback(true);
setHoverTimeout(null);
}, 500);
return () => {
clearTimeout(timeout);
};
}, [autoPlayback]);
const onPlayback = useCallback(
(isHovered: Boolean) => {
@@ -48,205 +77,181 @@ export default function PreviewThumbnailPlayer({
return;
}
if (!playerRef.current) {
if (isHovered) {
setIsInitiallyVisible(true);
}
return;
}
if (isHovered) {
playerRef.current.play();
setHoverTimeout(
setTimeout(() => {
setPlayback(true);
setHoverTimeout(null);
}, 500)
);
} else {
playerRef.current.pause();
playerRef.current.currentTime(startTs - relevantPreview.start);
}
},
[relevantPreview, startTs, playerRef]
);
const autoPlayObserver = useRef<IntersectionObserver | null>();
const preloadObserver = useRef<IntersectionObserver | null>();
const inViewRef = useCallback(
(node: HTMLElement | null) => {
if (!preloadObserver.current) {
try {
preloadObserver.current = new IntersectionObserver(
(entries) => {
const [{ isIntersecting }] = entries;
setVisible(isIntersecting);
},
{
threshold: 0,
root: document.getElementById("pageRoot"),
rootMargin: "10% 0px 25% 0px",
}
);
if (node) preloadObserver.current.observe(node);
} catch (e) {
// no op
if (hoverTimeout) {
clearTimeout(hoverTimeout);
}
}
if (isMobile && !autoPlayObserver.current) {
try {
autoPlayObserver.current = new IntersectionObserver(
(entries) => {
const [{ isIntersecting }] = entries;
if (isIntersecting) {
onPlayback(true);
} else {
onPlayback(false);
}
},
{
threshold: 1.0,
root: document.getElementById("pageRoot"),
rootMargin: "-10% 0px -25% 0px",
}
setPlayback(false);
setProgress(0);
if (playerRef.current) {
playerRef.current.pause();
playerRef.current.currentTime(
review.start_time - relevantPreview.start
);
if (node) autoPlayObserver.current.observe(node);
} catch (e) {
// no op
}
}
},
[preloadObserver, autoPlayObserver, onPlayback]
[hoverTimeout, relevantPreview, review, playerRef]
);
return (
<AspectRatio
ref={relevantPreview ? inViewRef : null}
ratio={16 / 9}
className="bg-black flex justify-center items-center"
onMouseEnter={() => onPlayback(true)}
onMouseLeave={() => onPlayback(false)}
<div
className="relative w-full h-full cursor-pointer"
onMouseEnter={isMobile ? undefined : () => onPlayback(true)}
onMouseLeave={isMobile ? undefined : () => onPlayback(false)}
>
<PreviewContent
playerRef={playerRef}
relevantPreview={relevantPreview}
isVisible={visible}
isInitiallyVisible={isInitiallyVisible}
startTs={startTs}
camera={camera}
eventId={eventId}
isMobile={isMobile}
onClick={onClick}
/>
</AspectRatio>
{playingBack ? (
<PreviewContent
playerRef={playerRef}
review={review}
relevantPreview={relevantPreview}
playback={playingBack}
setProgress={setProgress}
setReviewed={setReviewed}
/>
) : (
<img
className="h-full w-full"
src={`${apiHost}${review.thumb_path.replace("/media/frigate/", "")}`}
/>
)}
{!playingBack &&
(review.severity == "alert" || review.severity == "detection") && (
<div className="absolute top-1 left-[6px] flex gap-1">
{review.data.objects.map((object) => {
return getIconForLabel(object, "w-3 h-3 text-white");
})}
{review.data.audio.map((audio) => {
return getIconForLabel(audio, "w-3 h-3 text-white");
})}
{review.data.sub_labels?.map((sub) => {
return getIconForSubLabel(sub, "w-3 h-3 text-white");
})}
</div>
)}
{!playingBack && (
<div className="absolute left-[6px] right-[6px] bottom-1 flex justify-between text-white">
<TimeAgo time={review.start_time * 1000} />
{config &&
formatUnixTimestampToDateTime(review.start_time, {
strftime_fmt:
config.ui.time_format == "24hour"
? "%b %-d, %H:%M"
: "%b %-d, %I:%M %p",
})}
</div>
)}
<div className="absolute top-0 left-0 right-0 rounded-2xl z-10 w-full h-[30%] bg-gradient-to-b from-black/20 to-transparent pointer-events-none" />
<div className="absolute bottom-0 left-0 right-0 rounded-2xl z-10 w-full h-[10%] bg-gradient-to-t from-black/20 to-transparent pointer-events-none" />
{playingBack && (
<Slider
className="absolute left-0 right-0 bottom-0 z-10"
value={[progress]}
min={0}
step={1}
max={100}
/>
)}
{!playingBack && review.has_been_reviewed && (
<div className="absolute left-0 top-0 bottom-0 right-0 bg-black bg-opacity-60" />
)}
</div>
);
}
type PreviewContentProps = {
playerRef: React.MutableRefObject<Player | null>;
camera: string;
review: ReviewSegment;
relevantPreview: Preview | undefined;
eventId: string;
isVisible: boolean;
isInitiallyVisible: boolean;
startTs: number;
isMobile: boolean;
onClick?: () => void;
playback: boolean;
setProgress?: (progress: number) => void;
setReviewed?: () => void;
};
function PreviewContent({
playerRef,
camera,
review,
relevantPreview,
eventId,
isVisible,
isInitiallyVisible,
startTs,
isMobile,
onClick,
playback,
setProgress,
setReviewed,
}: PreviewContentProps) {
const apiHost = useApiHost();
const slowPlayack = isSafari();
// handle touchstart -> touchend as click
const [touchStart, setTouchStart] = useState(0);
const handleTouchStart = useCallback(() => {
setTouchStart(new Date().getTime());
}, []);
useEffect(() => {
if (!isMobile || !playerRef.current || !onClick) {
return;
}
playerRef.current.on("touchend", () => {
if (!onClick) {
return;
}
const touchEnd = new Date().getTime();
// consider tap less than 100 ms
if (touchEnd - touchStart < 100) {
onClick();
}
});
}, [playerRef, touchStart]);
if (relevantPreview && !isVisible) {
return <div />;
} else if (!relevantPreview && !isCurrentHour(startTs)) {
if (relevantPreview && playback) {
return (
<img
className="w-[160px]"
src={`${apiHost}api/events/${eventId}/thumbnail.jpg`}
<VideoPlayer
options={{
preload: "auto",
autoplay: true,
controls: false,
muted: true,
fluid: true,
aspectRatio: "16:9",
loadingSpinner: false,
sources: relevantPreview
? [
{
src: `${relevantPreview.src}`,
type: "video/mp4",
},
]
: [],
}}
seekOptions={{}}
onReady={(player) => {
playerRef.current = player;
if (!relevantPreview) {
return;
}
// start with a bit of padding
const playerStartTime = Math.max(
0,
review.start_time - relevantPreview.start - 8
);
player.playbackRate(isSafari ? 2 : 8);
player.currentTime(playerStartTime);
player.on("timeupdate", () => {
if (!setProgress || playerRef.current?.paused()) {
return;
}
const playerProgress =
(player.currentTime() || 0) - playerStartTime;
// end with a bit of padding
const playerDuration = review.end_time - review.start_time + 8;
const playerPercent = (playerProgress / playerDuration) * 100;
if (
setReviewed &&
!review.has_been_reviewed &&
playerPercent > 50
) {
setReviewed();
}
if (playerPercent > 100) {
playerRef.current?.pause();
setProgress(100.0);
} else {
setProgress(playerPercent);
}
});
}}
onDispose={() => {
playerRef.current = null;
}}
/>
);
} else {
return (
<>
<div className="w-full">
<VideoPlayer
options={{
preload: "auto",
aspectRatio: "16:9",
autoplay: true,
controls: false,
muted: true,
loadingSpinner: false,
poster: relevantPreview
? ""
: `${apiHost}api/preview/${camera}/${startTs}/thumbnail.jpg`,
sources: relevantPreview
? [
{
src: `${relevantPreview.src}`,
type: "video/mp4",
},
]
: [],
}}
seekOptions={{}}
onReady={(player) => {
playerRef.current = player;
if (!relevantPreview) {
return;
}
if (!isInitiallyVisible) {
player.pause(); // autoplay + pause is required for iOS
}
player.playbackRate(slowPlayack ? 2 : 8);
player.currentTime(startTs - relevantPreview.start);
if (isMobile && onClick) {
player.on("touchstart", handleTouchStart);
}
}}
onDispose={() => {
playerRef.current = null;
}}
/>
</div>
{relevantPreview && (
<LuPlayCircle className="absolute z-10 left-1 bottom-1 w-4 h-4 text-white text-opacity-60" />
)}
</>
);
}
}

View File

@@ -47,7 +47,7 @@ export function EventReviewTimeline({
const currentTimeRef = useRef<HTMLDivElement>(null);
const observer = useRef<ResizeObserver | null>(null);
const timelineDuration = useMemo(
() => timelineEnd - timelineStart,
() => timelineStart - timelineEnd,
[timelineEnd, timelineStart]
);
@@ -208,7 +208,7 @@ export function EventReviewTimeline({
return (
<div
ref={timelineRef}
className={`relative w-[120px] md:w-[100px] h-[100dvh] overflow-y-scroll no-scrollbar bg-secondary ${
className={`relative w-[120px] md:w-[100px] h-full overflow-y-scroll no-scrollbar bg-secondary ${
isDragging && showHandlebar ? "cursor-grabbing" : "cursor-auto"
}`}
>

View File

@@ -1,7 +1,7 @@
import * as React from "react"
import * as SliderPrimitive from "@radix-ui/react-slider"
import * as React from "react";
import * as SliderPrimitive from "@radix-ui/react-slider";
import { cn } from "@/lib/utils"
import { cn } from "@/lib/utils";
const Slider = React.forwardRef<
React.ElementRef<typeof SliderPrimitive.Root>,
@@ -15,12 +15,11 @@ const Slider = React.forwardRef<
)}
{...props}
>
<SliderPrimitive.Track className="relative h-2 w-full grow overflow-hidden rounded-full bg-secondary">
<SliderPrimitive.Range className="absolute h-full bg-primary" />
<SliderPrimitive.Track className="relative h-1 w-full grow overflow-hidden rounded-full">
<SliderPrimitive.Range className="absolute h-full bg-blue-500" />
</SliderPrimitive.Track>
<SliderPrimitive.Thumb className="block h-5 w-5 rounded-full border-2 border-primary bg-background ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50" />
</SliderPrimitive.Root>
))
Slider.displayName = SliderPrimitive.Root.displayName
));
Slider.displayName = SliderPrimitive.Root.displayName;
export { Slider }
export { Slider };

View File

@@ -0,0 +1,59 @@
import * as React from "react"
import * as ToggleGroupPrimitive from "@radix-ui/react-toggle-group"
import { VariantProps } from "class-variance-authority"
import { cn } from "@/lib/utils"
import { toggleVariants } from "@/components/ui/toggle"
const ToggleGroupContext = React.createContext<
VariantProps<typeof toggleVariants>
>({
size: "default",
variant: "default",
})
const ToggleGroup = React.forwardRef<
React.ElementRef<typeof ToggleGroupPrimitive.Root>,
React.ComponentPropsWithoutRef<typeof ToggleGroupPrimitive.Root> &
VariantProps<typeof toggleVariants>
>(({ className, variant, size, children, ...props }, ref) => (
<ToggleGroupPrimitive.Root
ref={ref}
className={cn("flex items-center justify-center gap-1", className)}
{...props}
>
<ToggleGroupContext.Provider value={{ variant, size }}>
{children}
</ToggleGroupContext.Provider>
</ToggleGroupPrimitive.Root>
))
ToggleGroup.displayName = ToggleGroupPrimitive.Root.displayName
const ToggleGroupItem = React.forwardRef<
React.ElementRef<typeof ToggleGroupPrimitive.Item>,
React.ComponentPropsWithoutRef<typeof ToggleGroupPrimitive.Item> &
VariantProps<typeof toggleVariants>
>(({ className, children, variant, size, ...props }, ref) => {
const context = React.useContext(ToggleGroupContext)
return (
<ToggleGroupPrimitive.Item
ref={ref}
className={cn(
toggleVariants({
variant: context.variant || variant,
size: context.size || size,
}),
className
)}
{...props}
>
{children}
</ToggleGroupPrimitive.Item>
)
})
ToggleGroupItem.displayName = ToggleGroupPrimitive.Item.displayName
export { ToggleGroup, ToggleGroupItem }

View File

@@ -0,0 +1,43 @@
import * as React from "react"
import * as TogglePrimitive from "@radix-ui/react-toggle"
import { cva, type VariantProps } from "class-variance-authority"
import { cn } from "@/lib/utils"
const toggleVariants = cva(
"inline-flex items-center justify-center rounded-md text-sm font-medium ring-offset-background transition-colors hover:bg-muted hover:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 data-[state=on]:bg-accent data-[state=on]:text-accent-foreground",
{
variants: {
variant: {
default: "bg-transparent",
outline:
"border border-input bg-transparent hover:bg-accent hover:text-accent-foreground",
},
size: {
default: "h-10 px-3",
sm: "h-9 px-2.5",
lg: "h-11 px-5",
},
},
defaultVariants: {
variant: "default",
size: "default",
},
}
)
const Toggle = React.forwardRef<
React.ElementRef<typeof TogglePrimitive.Root>,
React.ComponentPropsWithoutRef<typeof TogglePrimitive.Root> &
VariantProps<typeof toggleVariants>
>(({ className, variant, size, ...props }, ref) => (
<TogglePrimitive.Root
ref={ref}
className={cn(toggleVariants({ variant, size, className }))}
{...props}
/>
))
Toggle.displayName = TogglePrimitive.Root.displayName
export { Toggle, toggleVariants }