Motion timeline data (#10245)

* Refactor activity api to send motion and audio data * Prepare for using motion data timeline * Get working * reduce to 0 * fix * Formatting * fix typing * add motion data to timelines and allow motion cameas to be selectable * Fix tests * cleanup * Fix not loading preview when changing hours
2024-03-05 12:55:44 -07:00
parent a174d82eb9
commit 9e8a42ca0e
11 changed files with 269 additions and 197 deletions
--- a/web/src/components/player/DynamicVideoPlayer.tsx
+++ b/web/src/components/player/DynamicVideoPlayer.tsx
@@ -85,6 +85,16 @@ export default function DynamicVideoPlayer({
    );
  }, [camera, config, previewOnly]);

+  useEffect(() => {
+    if (!controller) {
+      return;
+    }
+
+    if (onControllerReady) {
+      onControllerReady(controller);
+    }
+  }, [controller, onControllerReady]);
+
  const [hasRecordingAtTime, setHasRecordingAtTime] = useState(true);

  // keyboard control
@@ -215,6 +225,10 @@ export default function DynamicVideoPlayer({
    );
    setCurrentPreview(preview);

+    if (preview && previewRef.current) {
+      previewRef.current.load();
+    }
+
    controller.newPlayback({
      recordings: recordings ?? [],
      playbackUri,
@@ -283,27 +297,20 @@ export default function DynamicVideoPlayer({
      )}
      <video
        ref={previewRef}
-        className={`size-full rounded-2xl ${currentPreview != undefined && isScrubbing ? "visible" : "hidden"} ${tallVideo ? "aspect-tall" : ""} bg-black`}
+        className={`size-full rounded-2xl ${currentPreview != undefined && (previewOnly || isScrubbing) ? "visible" : "hidden"} ${tallVideo ? "aspect-tall" : ""} bg-black`}
        preload="auto"
        autoPlay
        playsInline
        muted
        onSeeked={onPreviewSeeked}
        onLoadedData={() => controller.previewReady()}
-        onLoadStart={
-          previewOnly && onControllerReady
-            ? () => {
-                onControllerReady(controller);
-              }
-            : undefined
-        }
      >
        {currentPreview != undefined && (
          <source src={currentPreview.src} type={currentPreview.type} />
        )}
      </video>
      {onClick && !hasRecordingAtTime && (
-        <div className="absolute inset-0 z-10 bg-black bg-opacity-60" />
+        <div className="absolute inset-0 z-10 bg-black bg-opacity-60 rounded-2xl" />
      )}
    </div>
  );
--- a/web/src/components/timeline/MotionReviewTimeline.tsx
+++ b/web/src/components/timeline/MotionReviewTimeline.tsx
@@ -9,9 +9,8 @@ import {
 } from "react";
 import MotionSegment from "./MotionSegment";
 import { useEventUtils } from "@/hooks/use-event-utils";
-import { ReviewSegment, ReviewSeverity } from "@/types/review";
+import { MotionData, ReviewSegment, ReviewSeverity } from "@/types/review";
 import ReviewTimeline from "./ReviewTimeline";
-import { MockMotionData } from "@/pages/UIPlayground";

 export type MotionReviewTimelineProps = {
  segmentDuration: number;
@@ -25,7 +24,7 @@ export type MotionReviewTimelineProps = {
  minimapStartTime?: number;
  minimapEndTime?: number;
  events: ReviewSegment[];
-  motion_events: MockMotionData[];
+  motion_events: MotionData[];
  severityType: ReviewSeverity;
  contentRef: RefObject<HTMLDivElement>;
  onHandlebarDraggingChange?: (isDragging: boolean) => void;
--- a/web/src/components/timeline/MotionSegment.tsx
+++ b/web/src/components/timeline/MotionSegment.tsx
@@ -1,6 +1,6 @@
 import { useEventUtils } from "@/hooks/use-event-utils";
 import { useEventSegmentUtils } from "@/hooks/use-event-segment-utils";
-import { ReviewSegment } from "@/types/review";
+import { MotionData, ReviewSegment } from "@/types/review";
 import React, {
  RefObject,
  useCallback,
@@ -10,13 +10,12 @@ import React, {
 } from "react";
 import scrollIntoView from "scroll-into-view-if-needed";
 import { MinimapBounds, Tick, Timestamp } from "./segment-metadata";
-import { MockMotionData } from "@/pages/UIPlayground";
 import { useMotionSegmentUtils } from "@/hooks/use-motion-segment-utils";
 import { isMobile } from "react-device-detect";

 type MotionSegmentProps = {
  events: ReviewSegment[];
-  motion_events: MockMotionData[];
+  motion_events: MotionData[];
  segmentTime: number;
  segmentDuration: number;
  timestampSpread: number;
--- a/web/src/hooks/use-motion-segment-utils.ts
+++ b/web/src/hooks/use-motion-segment-utils.ts
@@ -1,9 +1,9 @@
 import { useCallback, useMemo } from "react";
-import { MockMotionData } from "@/pages/UIPlayground";
+import { MotionData } from "@/types/review";

 export const useMotionSegmentUtils = (
  segmentDuration: number,
-  motion_events: MockMotionData[],
+  motion_events: MotionData[],
 ) => {
  const halfSegmentDuration = useMemo(
    () => segmentDuration / 2,
@@ -47,7 +47,7 @@ export const useMotionSegmentUtils = (
        );
      });

-      return matchingEvent?.motionValue ?? 0;
+      return matchingEvent?.motion ?? 0;
    },
    [motion_events, getSegmentStart, getSegmentEnd],
  );
@@ -61,7 +61,7 @@ export const useMotionSegmentUtils = (
        );
      });

-      return matchingEvent?.audioValue ?? 0;
+      return matchingEvent?.audio ?? 0;
    },
    [motion_events, getSegmentStart, getSegmentEnd],
  );
--- a/web/src/pages/Events.tsx
+++ b/web/src/pages/Events.tsx
@@ -240,22 +240,37 @@ export default function Events() {

  // selected items

-  const selectedData = useMemo(() => {
+  const selectedReviewData = useMemo(() => {
    if (!config) {
      return undefined;
    }

-    if (!selectedReviewId) {
-      return undefined;
-    }
-
    if (!reviewPages) {
      return undefined;
    }

+    if (!selectedReviewId) {
+      return undefined;
+    }
+
    const allCameras = reviewFilter?.cameras ?? Object.keys(config.cameras);

    const allReviews = reviewPages.flat();
+
+    if (selectedReviewId.startsWith("motion")) {
+      const motionData = selectedReviewId.split(",");
+      // format is motion,camera,start_time
+      return {
+        camera: motionData[1],
+        severity: "significant_motion" as ReviewSeverity,
+        start_time: parseFloat(motionData[2]),
+        allCameras: allCameras,
+        cameraSegments: allReviews.filter((seg) =>
+          allCameras.includes(seg.camera),
+        ),
+      };
+    }
+
    const selectedReview = allReviews.find(
      (item) => item.id == selectedReviewId,
    );
@@ -265,7 +280,9 @@ export default function Events() {
    }

    return {
-      selected: selectedReview,
+      camera: selectedReview.camera,
+      severity: selectedReview.severity,
+      start_time: selectedReview.start_time,
      allCameras: allCameras,
      cameraSegments: allReviews.filter((seg) =>
        allCameras.includes(seg.camera),
@@ -280,12 +297,14 @@ export default function Events() {
    return <ActivityIndicator />;
  }

-  if (selectedData) {
+  if (selectedReviewData) {
    if (isMobile) {
      return (
        <MobileRecordingView
-          reviewItems={selectedData.cameraSegments}
-          selectedReview={selectedData.selected}
+          reviewItems={selectedReviewData.cameraSegments}
+          startCamera={selectedReviewData.camera}
+          startTime={selectedReviewData.start_time}
+          severity={selectedReviewData.severity}
          relevantPreviews={allPreviews}
        />
      );
@@ -293,11 +312,11 @@ export default function Events() {

    return (
      <DesktopRecordingView
-        startCamera={selectedData.selected.camera}
-        startTime={selectedData.selected.start_time}
-        allCameras={selectedData.allCameras}
-        severity={selectedData.selected.severity}
-        reviewItems={selectedData.cameraSegments}
+        startCamera={selectedReviewData.camera}
+        startTime={selectedReviewData.start_time}
+        allCameras={selectedReviewData.allCameras}
+        severity={selectedReviewData.severity}
+        reviewItems={selectedReviewData.cameraSegments}
        allPreviews={allPreviews}
      />
    );
--- a/web/src/pages/UIPlayground.tsx
+++ b/web/src/pages/UIPlayground.tsx
@@ -4,7 +4,12 @@ import useSWR from "swr";
 import { FrigateConfig } from "@/types/frigateConfig";
 import ActivityIndicator from "@/components/indicators/activity-indicator";
 import EventReviewTimeline from "@/components/timeline/EventReviewTimeline";
-import { ReviewData, ReviewSegment, ReviewSeverity } from "@/types/review";
+import {
+  MotionData,
+  ReviewData,
+  ReviewSegment,
+  ReviewSeverity,
+} from "@/types/review";
 import { Button } from "@/components/ui/button";
 import CameraActivityIndicator from "@/components/indicators/CameraActivityIndicator";
 import MotionReviewTimeline from "@/components/timeline/MotionReviewTimeline";
@@ -53,14 +58,7 @@ function ColorSwatch({ name, value }: { name: string; value: string }) {
  );
 }

-export type MockMotionData = {
-  start_time: number;
-  end_time: number;
-  motionValue: number;
-  audioValue: number;
-};
-
-function generateRandomMotionAudioData(): MockMotionData[] {
+function generateRandomMotionAudioData(): MotionData[] {
  const now = new Date();
  const endTime = now.getTime() / 1000;
  const startTime = endTime - 24 * 60 * 60; // 24 hours ago
@@ -72,14 +70,12 @@ function generateRandomMotionAudioData(): MockMotionData[] {
    startTimestamp < endTime;
    startTimestamp += interval
  ) {
-    const endTimestamp = startTimestamp + interval;
-    const motionValue = Math.floor(Math.random() * 101); // Random number between 0 and 100
-    const audioValue = Math.random() * -100; // Random negative value between -100 and 0
+    const motion = Math.floor(Math.random() * 101); // Random number between 0 and 100
+    const audio = Math.random() * -100; // Random negative value between -100 and 0
    data.push({
      start_time: startTimestamp,
-      end_time: endTimestamp,
-      motionValue,
-      audioValue,
+      motion,
+      audio,
    });
  }

@@ -126,7 +122,7 @@ function UIPlayground() {
  const { data: config } = useSWR<FrigateConfig>("config");
  const contentRef = useRef<HTMLDivElement>(null);
  const [mockEvents, setMockEvents] = useState<ReviewSegment[]>([]);
-  const [mockMotionData, setMockMotionData] = useState<MockMotionData[]>([]);
+  const [mockMotionData, setMockMotionData] = useState<MotionData[]>([]);
  const [handlebarTime, setHandlebarTime] = useState(
    Math.floor(Date.now() / 1000) - 15 * 60,
  );
--- a/web/src/types/review.ts
+++ b/web/src/types/review.ts
@@ -37,3 +37,9 @@ export type ReviewSummary = {
  total_detection: number;
  total_motion: number;
 };
+
+export type MotionData = {
+  start_time: number;
+  motion: number;
+  audio: number;
+};
--- a/web/src/views/events/EventView.tsx
+++ b/web/src/views/events/EventView.tsx
@@ -14,6 +14,7 @@ import { useScrollLockout } from "@/hooks/use-mouse-listener";
 import { FrigateConfig } from "@/types/frigateConfig";
 import { Preview } from "@/types/preview";
 import {
+  MotionData,
  ReviewFilter,
  ReviewSegment,
  ReviewSeverity,
@@ -33,6 +34,7 @@ import { isDesktop, isMobile } from "react-device-detect";
 import { LuFolderCheck } from "react-icons/lu";
 import { MdCircle } from "react-icons/md";
 import useSWR from "swr";
+import MotionReviewTimeline from "@/components/timeline/MotionReviewTimeline";

 type EventViewProps = {
  reviewPages?: ReviewSegment[][];
@@ -284,6 +286,7 @@ export default function EventView({
            relevantPreviews={relevantPreviews}
            timeRange={timeRange}
            filter={filter}
+            onSelectReview={onSelectReview}
          />
        )}
      </div>
@@ -526,6 +529,7 @@ type MotionReviewProps = {
  relevantPreviews?: Preview[];
  timeRange: { before: number; after: number };
  filter?: ReviewFilter;
+  onSelectReview: (data: string, ctrl: boolean) => void;
 };
 function MotionReview({
  contentRef,
@@ -533,6 +537,7 @@ function MotionReview({
  relevantPreviews,
  timeRange,
  filter,
+  onSelectReview,
 }: MotionReviewProps) {
  const segmentDuration = 30;
  const { data: config } = useSWR<FrigateConfig>("config");
@@ -561,6 +566,17 @@ function MotionReview({
    {},
  );

+  // motion data
+
+  const { data: motionData } = useSWR<MotionData[]>([
+    "review/activity",
+    {
+      before: timeRange.before,
+      after: timeRange.after,
+      scale: segmentDuration / 2,
+    },
+  ]);
+
  // timeline time

  const lastFullHour = useMemo(() => {
@@ -580,6 +596,7 @@ function MotionReview({
  );

  // move to next clip
+
  useEffect(() => {
    if (
      !videoPlayersRef.current &&
@@ -638,12 +655,15 @@ function MotionReview({
                videoPlayersRef.current[camera.name] = controller;
                setPlayerReady(true);
              }}
+              onClick={() =>
+                onSelectReview(`motion,${camera.name},${currentTime}`, false)
+              }
            />
          );
        })}
      </div>
      <div className="w-[55px] md:w-[100px] mt-2 overflow-y-auto no-scrollbar">
-        <EventReviewTimeline
+        <MotionReviewTimeline
          segmentDuration={segmentDuration}
          timestampSpread={15}
          timelineStart={timeRangeSegments.end}
@@ -652,6 +672,7 @@ function MotionReview({
          handlebarTime={currentTime}
          setHandlebarTime={setCurrentTime}
          events={reviewItems.all}
+          motion_events={motionData ?? []}
          severityType="significant_motion"
          contentRef={contentRef}
        />
--- a/web/src/views/events/RecordingView.tsx
+++ b/web/src/views/events/RecordingView.tsx
@@ -2,13 +2,17 @@ import DynamicVideoPlayer, {
  DynamicVideoController,
 } from "@/components/player/DynamicVideoPlayer";
 import EventReviewTimeline from "@/components/timeline/EventReviewTimeline";
+import MotionReviewTimeline from "@/components/timeline/MotionReviewTimeline";
 import { Button } from "@/components/ui/button";
 import { Preview } from "@/types/preview";
-import { ReviewSegment, ReviewSeverity } from "@/types/review";
+import { MotionData, ReviewSegment, ReviewSeverity } from "@/types/review";
 import { getChunkedTimeDay } from "@/utils/timelineUtil";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { IoMdArrowRoundBack } from "react-icons/io";
 import { useNavigate } from "react-router-dom";
+import useSWR from "swr";
+
+const SEGMENT_DURATION = 30;

 type DesktopRecordingViewProps = {
  startCamera: string;
@@ -116,6 +120,21 @@ export function DesktopRecordingView({
    [allCameras, currentTime, mainCamera],
  );

+  // motion timeline data
+
+  const { data: motionData } = useSWR<MotionData[]>(
+    severity == "significant_motion"
+      ? [
+          "review/activity",
+          {
+            before: timeRange.end,
+            after: timeRange.start,
+            scale: SEGMENT_DURATION / 2,
+          },
+        ]
+      : null,
+  );
+
  return (
    <div ref={contentRef} className="relative size-full">
      <Button
@@ -181,31 +200,52 @@ export function DesktopRecordingView({
      </div>

      <div className="absolute overflow-hidden w-56 inset-y-0 right-0">
-        <EventReviewTimeline
-          segmentDuration={30}
-          timestampSpread={15}
-          timelineStart={timeRange.end}
-          timelineEnd={timeRange.start}
-          showHandlebar
-          handlebarTime={currentTime}
-          setHandlebarTime={setCurrentTime}
-          events={reviewItems}
-          severityType={severity}
-          contentRef={contentRef}
-          onHandlebarDraggingChange={(scrubbing) => setScrubbing(scrubbing)}
-        />
+        {severity != "significant_motion" ? (
+          <EventReviewTimeline
+            segmentDuration={30}
+            timestampSpread={15}
+            timelineStart={timeRange.end}
+            timelineEnd={timeRange.start}
+            showHandlebar
+            handlebarTime={currentTime}
+            setHandlebarTime={setCurrentTime}
+            events={reviewItems}
+            severityType={severity}
+            contentRef={contentRef}
+            onHandlebarDraggingChange={(scrubbing) => setScrubbing(scrubbing)}
+          />
+        ) : (
+          <MotionReviewTimeline
+            segmentDuration={30}
+            timestampSpread={15}
+            timelineStart={timeRange.end}
+            timelineEnd={timeRange.start}
+            showHandlebar
+            handlebarTime={currentTime}
+            setHandlebarTime={setCurrentTime}
+            events={reviewItems}
+            motion_events={motionData ?? []}
+            severityType={severity}
+            contentRef={contentRef}
+            onHandlebarDraggingChange={(scrubbing) => setScrubbing(scrubbing)}
+          />
+        )}
      </div>
    </div>
  );
 }

 type MobileRecordingViewProps = {
-  selectedReview: ReviewSegment;
+  startCamera: string;
+  startTime: number;
+  severity: ReviewSeverity;
  reviewItems: ReviewSegment[];
  relevantPreviews?: Preview[];
 };
 export function MobileRecordingView({
-  selectedReview,
+  startCamera,
+  startTime,
+  severity,
  reviewItems,
  relevantPreviews,
 }: MobileRecordingViewProps) {
@@ -219,16 +259,10 @@ export function MobileRecordingView({

  // timeline time

-  const timeRange = useMemo(
-    () => getChunkedTimeDay(selectedReview.start_time),
-    [selectedReview],
-  );
+  const timeRange = useMemo(() => getChunkedTimeDay(startTime), [startTime]);
  const [selectedRangeIdx, setSelectedRangeIdx] = useState(
    timeRange.ranges.findIndex((chunk) => {
-      return (
-        chunk.start <= selectedReview.start_time &&
-        chunk.end >= selectedReview.start_time
-      );
+      return chunk.start <= startTime && chunk.end >= startTime;
    }),
  );

@@ -251,7 +285,7 @@ export function MobileRecordingView({

  const [scrubbing, setScrubbing] = useState(false);
  const [currentTime, setCurrentTime] = useState<number>(
-    selectedReview?.start_time || Date.now() / 1000,
+    startTime || Date.now() / 1000,
  );

  useEffect(() => {
@@ -269,6 +303,21 @@ export function MobileRecordingView({
    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, [scrubbing]);

+  // motion timeline data
+
+  const { data: motionData } = useSWR<MotionData[]>(
+    severity == "significant_motion"
+      ? [
+          "review/activity",
+          {
+            before: timeRange.end,
+            after: timeRange.start,
+            scale: SEGMENT_DURATION / 2,
+          },
+        ]
+      : null,
+  );
+
  return (
    <div ref={contentRef} className="flex flex-col relative w-full h-full">
      <Button className="rounded-lg" onClick={() => navigate(-1)}>
@@ -278,7 +327,7 @@ export function MobileRecordingView({

      <div>
        <DynamicVideoPlayer
-          camera={selectedReview.camera}
+          camera={startCamera}
          timeRange={timeRange.ranges[selectedRangeIdx]}
          cameraPreviews={relevantPreviews || []}
          onControllerReady={(controller) => {
@@ -288,28 +337,42 @@ export function MobileRecordingView({
              setCurrentTime(timestamp);
            });

-            controllerRef.current?.seekToTimestamp(
-              selectedReview.start_time,
-              true,
-            );
+            controllerRef.current?.seekToTimestamp(startTime, true);
          }}
        />
      </div>

      <div className="flex-grow overflow-hidden">
-        <EventReviewTimeline
-          segmentDuration={30}
-          timestampSpread={15}
-          timelineStart={timeRange.end}
-          timelineEnd={timeRange.start}
-          showHandlebar
-          handlebarTime={currentTime}
-          setHandlebarTime={setCurrentTime}
-          events={reviewItems}
-          severityType={selectedReview.severity}
-          contentRef={contentRef}
-          onHandlebarDraggingChange={(scrubbing) => setScrubbing(scrubbing)}
-        />
+        {severity != "significant_motion" ? (
+          <EventReviewTimeline
+            segmentDuration={30}
+            timestampSpread={15}
+            timelineStart={timeRange.end}
+            timelineEnd={timeRange.start}
+            showHandlebar
+            handlebarTime={currentTime}
+            setHandlebarTime={setCurrentTime}
+            events={reviewItems}
+            severityType={severity}
+            contentRef={contentRef}
+            onHandlebarDraggingChange={(scrubbing) => setScrubbing(scrubbing)}
+          />
+        ) : (
+          <MotionReviewTimeline
+            segmentDuration={30}
+            timestampSpread={15}
+            timelineStart={timeRange.end}
+            timelineEnd={timeRange.start}
+            showHandlebar
+            handlebarTime={currentTime}
+            setHandlebarTime={setCurrentTime}
+            events={reviewItems}
+            motion_events={motionData ?? []}
+            severityType={severity}
+            contentRef={contentRef}
+            onHandlebarDraggingChange={(scrubbing) => setScrubbing(scrubbing)}
+          />
+        )}
      </div>
    </div>
  );