Improve graph using pandas (#9234)

* Ensure viewport is always full screen * Protect against hour with no cards and ensure data is consistent * Reduce grouped up image refreshes * Include current hour and fix scrubbing bugginess * Scroll initially selected timeline in to view * Expand timelne class type * Use poster image for preview on video player instead of using separate image view * Fix available streaming modes * Incrase timing for grouping timline items * Fix audio activity listener * Fix player not switching views correctly * Use player time to convert to timeline time * Update sub labels for previous timeline items * Show mini timeline bar for non selected items * Rewrite desktop timeline to use separate dynamic video player component * Extend improvements to mobile as well * Improve time formatting * Fix scroll * Fix no preview case * Mobile fixes * Audio toggle fixes * More fixes for mobile * Improve scaling of graph motion activity * Add keyboard shortcut hook and support shortcuts for playback page * Fix sizing of dialog * Improve height scaling of dialog * simplify and fix layout system for timeline * Fix timeilne items not working * Implement basic Frigate+ submitting from timeline
2024-01-31 05:29:18 -07:00
parent 9c4b69191b
commit af3f6dadcb
28 changed files with 1379 additions and 852 deletions
--- a/web/src/components/player/DynamicVideoPlayer.tsx
+++ b/web/src/components/player/DynamicVideoPlayer.tsx
@@ -0,0 +1,411 @@
+import {
+  MutableRefObject,
+  useCallback,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from "react";
+import VideoPlayer from "./VideoPlayer";
+import Player from "video.js/dist/types/player";
+import TimelineEventOverlay from "../overlay/TimelineDataOverlay";
+import { useApiHost } from "@/api";
+import useSWR from "swr";
+import { FrigateConfig } from "@/types/frigateConfig";
+import ActivityIndicator from "../ui/activity-indicator";
+import useKeyboardListener from "@/hooks/use-keyboard-listener";
+
+/**
+ * Dynamically switches between video playback and scrubbing preview player.
+ */
+type DynamicVideoPlayerProps = {
+  className?: string;
+  camera: string;
+  timeRange: { start: number; end: number };
+  cameraPreviews: Preview[];
+  onControllerReady?: (controller: DynamicVideoController) => void;
+};
+export default function DynamicVideoPlayer({
+  className,
+  camera,
+  timeRange,
+  cameraPreviews,
+  onControllerReady,
+}: DynamicVideoPlayerProps) {
+  const apiHost = useApiHost();
+  const { data: config } = useSWR<FrigateConfig>("config");
+  const timezone = useMemo(
+    () =>
+      config?.ui?.timezone || Intl.DateTimeFormat().resolvedOptions().timeZone,
+    [config]
+  );
+
+  // controlling playback
+
+  const playerRef = useRef<Player | undefined>(undefined);
+  const previewRef = useRef<Player | undefined>(undefined);
+  const [isScrubbing, setIsScrubbing] = useState(false);
+  const [hasPreview, setHasPreview] = useState(false);
+  const [focusedItem, setFocusedItem] = useState<Timeline | undefined>(
+    undefined
+  );
+  const controller = useMemo(() => {
+    if (!config) {
+      return undefined;
+    }
+
+    return new DynamicVideoController(
+      playerRef,
+      previewRef,
+      (config.cameras[camera]?.detect?.annotation_offset || 0) / 1000,
+      setIsScrubbing,
+      setFocusedItem
+    );
+  }, [config]);
+
+  // keyboard control
+  const onKeyboardShortcut = useCallback(
+    (key: string, down: boolean, repeat: boolean) => {
+      switch (key) {
+        case "ArrowLeft":
+          if (down) {
+            const currentTime = playerRef.current?.currentTime();
+
+            if (currentTime) {
+              playerRef.current?.currentTime(Math.max(0, currentTime - 5));
+            }
+          }
+          break;
+        case "ArrowRight":
+          if (down) {
+            const currentTime = playerRef.current?.currentTime();
+
+            if (currentTime) {
+              playerRef.current?.currentTime(currentTime + 5);
+            }
+          }
+          break;
+        case "m":
+          if (down && !repeat && playerRef.current) {
+            playerRef.current.muted(!playerRef.current.muted());
+          }
+          break;
+        case " ":
+          if (down && playerRef.current) {
+            if (playerRef.current.paused()) {
+              playerRef.current.play();
+            } else {
+              playerRef.current.pause();
+            }
+          }
+          break;
+      }
+    },
+    [playerRef]
+  );
+  useKeyboardListener(
+    ["ArrowLeft", "ArrowRight", "m", " "],
+    onKeyboardShortcut
+  );
+
+  // initial state
+
+  const initialPlaybackSource = useMemo(() => {
+    const date = new Date(timeRange.start * 1000);
+    return {
+      src: `${apiHost}vod/${date.getFullYear()}-${
+        date.getMonth() + 1
+      }/${date.getDate()}/${date.getHours()}/${camera}/${timezone.replaceAll(
+        "/",
+        ","
+      )}/master.m3u8`,
+      type: "application/vnd.apple.mpegurl",
+    };
+  }, []);
+  const initialPreviewSource = useMemo(() => {
+    const preview = cameraPreviews.find(
+      (preview) =>
+        Math.round(preview.start) >= timeRange.start &&
+        Math.floor(preview.end) <= timeRange.end
+    );
+
+    if (preview) {
+      setHasPreview(true);
+      return {
+        src: preview.src,
+        type: preview.type,
+      };
+    } else {
+      setHasPreview(false);
+      return undefined;
+    }
+  }, []);
+
+  // state of playback player
+
+  const recordingParams = useMemo(() => {
+    return {
+      before: timeRange.end,
+      after: timeRange.start,
+    };
+  }, [timeRange]);
+  const { data: recordings } = useSWR<Recording[]>(
+    [`${camera}/recordings`, recordingParams],
+    { revalidateOnFocus: false }
+  );
+
+  useEffect(() => {
+    if (!controller || !recordings || recordings.length == 0) {
+      return;
+    }
+
+    const date = new Date(timeRange.start * 1000);
+    const playbackUri = `${apiHost}vod/${date.getFullYear()}-${
+      date.getMonth() + 1
+    }/${date.getDate()}/${date.getHours()}/${camera}/${timezone.replaceAll(
+      "/",
+      ","
+    )}/master.m3u8`;
+
+    const preview = cameraPreviews.find(
+      (preview) =>
+        Math.round(preview.start) >= timeRange.start &&
+        Math.floor(preview.end) <= timeRange.end
+    );
+    setHasPreview(preview != undefined);
+
+    controller.newPlayback({
+      recordings,
+      playbackUri,
+      preview,
+    });
+  }, [controller, recordings]);
+
+  if (!controller) {
+    return <ActivityIndicator />;
+  }
+
+  return (
+    <div className={className}>
+      <div
+        className={`w-full relative ${
+          hasPreview && isScrubbing ? "hidden" : "visible"
+        }`}
+      >
+        <VideoPlayer
+          options={{
+            preload: "auto",
+            autoplay: true,
+            sources: [initialPlaybackSource],
+            controlBar: {
+              remainingTimeDisplay: false,
+              progressControl: {
+                seekBar: false,
+              },
+            },
+          }}
+          seekOptions={{ forward: 10, backward: 5 }}
+          onReady={(player) => {
+            playerRef.current = player;
+            player.on("playing", () => setFocusedItem(undefined));
+            player.on("timeupdate", () => {
+              controller.updateProgress(player.currentTime() || 0);
+            });
+
+            if (onControllerReady) {
+              onControllerReady(controller);
+            }
+          }}
+          onDispose={() => {
+            playerRef.current = undefined;
+          }}
+        >
+          {config && focusedItem && (
+            <TimelineEventOverlay
+              timeline={focusedItem}
+              cameraConfig={config.cameras[camera]}
+            />
+          )}
+        </VideoPlayer>
+      </div>
+      <div
+        className={`w-full ${hasPreview && isScrubbing ? "visible" : "hidden"}`}
+      >
+        <VideoPlayer
+          options={{
+            preload: "auto",
+            autoplay: true,
+            controls: false,
+            muted: true,
+            loadingSpinner: false,
+            sources: hasPreview ? initialPreviewSource : null,
+          }}
+          seekOptions={{}}
+          onReady={(player) => {
+            previewRef.current = player;
+            player.pause();
+            player.on("seeked", () => controller.finishedSeeking());
+          }}
+          onDispose={() => {
+            previewRef.current = undefined;
+          }}
+        />
+      </div>
+    </div>
+  );
+}
+
+export class DynamicVideoController {
+  // main state
+  private playerRef: MutableRefObject<Player | undefined>;
+  private previewRef: MutableRefObject<Player | undefined>;
+  private setScrubbing: (isScrubbing: boolean) => void;
+  private setFocusedItem: (timeline: Timeline) => void;
+  private playerMode: "playback" | "scrubbing" = "playback";
+
+  // playback
+  private recordings: Recording[] = [];
+  private onPlaybackTimestamp: ((time: number) => void) | undefined = undefined;
+  private annotationOffset: number;
+  private timeToStart: number | undefined = undefined;
+
+  // preview
+  private preview: Preview | undefined = undefined;
+  private timeToSeek: number | undefined = undefined;
+  private seeking = false;
+
+  constructor(
+    playerRef: MutableRefObject<Player | undefined>,
+    previewRef: MutableRefObject<Player | undefined>,
+    annotationOffset: number,
+    setScrubbing: (isScrubbing: boolean) => void,
+    setFocusedItem: (timeline: Timeline) => void
+  ) {
+    this.playerRef = playerRef;
+    this.previewRef = previewRef;
+    this.annotationOffset = annotationOffset;
+    this.setScrubbing = setScrubbing;
+    this.setFocusedItem = setFocusedItem;
+  }
+
+  newPlayback(newPlayback: DynamicPlayback) {
+    this.recordings = newPlayback.recordings;
+
+    this.playerRef.current?.src({
+      src: newPlayback.playbackUri,
+      type: "application/vnd.apple.mpegurl",
+    });
+
+    if (this.timeToStart) {
+      this.seekToTimestamp(this.timeToStart);
+      this.timeToStart = undefined;
+    }
+
+    this.preview = newPlayback.preview;
+    if (this.preview && this.previewRef.current) {
+      this.previewRef.current.src({
+        src: this.preview.src,
+        type: this.preview.type,
+      });
+    }
+  }
+
+  seekToTimestamp(time: number, play: boolean = false) {
+    if (this.playerMode != "playback") {
+      this.playerMode = "playback";
+      this.setScrubbing(false);
+      this.timeToSeek = undefined;
+      this.seeking = false;
+    }
+
+    if (this.recordings.length == 0) {
+      this.timeToStart = time;
+    }
+
+    let seekSeconds = 0;
+    (this.recordings || []).every((segment) => {
+      // if the next segment is past the desired time, stop calculating
+      if (segment.start_time > time) {
+        return false;
+      }
+
+      if (segment.end_time < time) {
+        seekSeconds += segment.end_time - segment.start_time;
+        return true;
+      }
+
+      seekSeconds +=
+        segment.end_time - segment.start_time - (segment.end_time - time);
+      return true;
+    });
+    this.playerRef.current?.currentTime(seekSeconds);
+
+    if (play) {
+      this.playerRef.current?.play();
+    }
+  }
+
+  seekToTimelineItem(timeline: Timeline) {
+    this.playerRef.current?.pause();
+    this.seekToTimestamp(timeline.timestamp + this.annotationOffset);
+    this.setFocusedItem(timeline);
+  }
+
+  updateProgress(playerTime: number) {
+    if (this.onPlaybackTimestamp) {
+      // take a player time in seconds and convert to timestamp in timeline
+      let timestamp = 0;
+      let totalTime = 0;
+      (this.recordings || []).every((segment) => {
+        if (totalTime + segment.duration > playerTime) {
+          // segment is here
+          timestamp = segment.start_time + (playerTime - totalTime);
+          return false;
+        } else {
+          totalTime += segment.duration;
+          return true;
+        }
+      });
+
+      this.onPlaybackTimestamp(timestamp);
+    }
+  }
+
+  onPlayerTimeUpdate(listener: (timestamp: number) => void) {
+    this.onPlaybackTimestamp = listener;
+  }
+
+  scrubToTimestamp(time: number) {
+    if (this.playerMode != "scrubbing") {
+      this.playerMode = "scrubbing";
+      this.playerRef.current?.pause();
+      this.setScrubbing(true);
+    }
+
+    if (this.preview) {
+      if (this.seeking) {
+        this.timeToSeek = time;
+      } else {
+        this.previewRef.current?.currentTime(time - this.preview.start);
+        this.seeking = true;
+      }
+    }
+  }
+
+  finishedSeeking() {
+    if (!this.preview || this.playerMode == "playback") {
+      return;
+    }
+
+    if (
+      this.timeToSeek &&
+      this.timeToSeek != this.previewRef.current?.currentTime()
+    ) {
+      this.previewRef.current?.currentTime(
+        this.timeToSeek - this.preview.start
+      );
+    } else {
+      this.seeking = false;
+    }
+  }
+}
--- a/web/src/components/player/PreviewThumbnailPlayer.tsx
+++ b/web/src/components/player/PreviewThumbnailPlayer.tsx
@@ -1,6 +1,4 @@
-import { FrigateConfig } from "@/types/frigateConfig";
 import VideoPlayer from "./VideoPlayer";
-import useSWR from "swr";
 import React, {
  useCallback,
  useEffect,
@@ -12,6 +10,7 @@ import { useApiHost } from "@/api";
 import Player from "video.js/dist/types/player";
 import { AspectRatio } from "../ui/aspect-ratio";
 import { LuPlayCircle } from "react-icons/lu";
+import { isCurrentHour } from "@/utils/dateUtil";

 type PreviewPlayerProps = {
  camera: string;
@@ -38,7 +37,6 @@ export default function PreviewThumbnailPlayer({
  isMobile,
  onClick,
 }: PreviewPlayerProps) {
-  const { data: config } = useSWR("config");
  const playerRef = useRef<Player | null>(null);
  const isSafari = useMemo(() => {
    return /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
@@ -54,7 +52,10 @@ export default function PreviewThumbnailPlayer({
      }

      if (!playerRef.current) {
-        setIsInitiallyVisible(true);
+        if (isHovered) {
+          setIsInitiallyVisible(true);
+        }
+
        return;
      }

@@ -105,6 +106,7 @@ export default function PreviewThumbnailPlayer({
            {
              threshold: 1.0,
              root: document.getElementById("pageRoot"),
+              rootMargin: "-10% 0px -25% 0px",
            }
          );
          if (node) autoPlayObserver.current.observe(node);
@@ -131,7 +133,6 @@ export default function PreviewThumbnailPlayer({
        isInitiallyVisible={isInitiallyVisible}
        startTs={startTs}
        camera={camera}
-        config={config}
        eventId={eventId}
        isMobile={isMobile}
        isSafari={isSafari}
@@ -143,7 +144,6 @@ export default function PreviewThumbnailPlayer({

 type PreviewContentProps = {
  playerRef: React.MutableRefObject<Player | null>;
-  config: FrigateConfig;
  camera: string;
  relevantPreview: Preview | undefined;
  eventId: string;
@@ -156,7 +156,6 @@ type PreviewContentProps = {
 };
 function PreviewContent({
  playerRef,
-  config,
  camera,
  relevantPreview,
  eventId,
@@ -195,22 +194,13 @@ function PreviewContent({

  if (relevantPreview && !isVisible) {
    return <div />;
-  } else if (!relevantPreview) {
-    if (isCurrentHour(startTs)) {
-      return (
-        <img
-          className={`${getPreviewWidth(camera, config)}`}
-          src={`${apiHost}api/preview/${camera}/${startTs}/thumbnail.jpg`}
-        />
-      );
-    } else {
-      return (
-        <img
-          className="w-[160px]"
-          src={`${apiHost}api/events/${eventId}/thumbnail.jpg`}
-        />
-      );
-    }
+  } else if (!relevantPreview && !isCurrentHour(startTs)) {
+    return (
+      <img
+        className="w-[160px]"
+        src={`${apiHost}api/events/${eventId}/thumbnail.jpg`}
+      />
+    );
  } else {
    return (
      <>
@@ -223,17 +213,26 @@ function PreviewContent({
              controls: false,
              muted: true,
              loadingSpinner: false,
-              sources: [
-                {
-                  src: `${relevantPreview.src}`,
-                  type: "video/mp4",
-                },
-              ],
+              poster: relevantPreview
+                ? ""
+                : `${apiHost}api/preview/${camera}/${startTs}/thumbnail.jpg`,
+              sources: relevantPreview
+                ? [
+                    {
+                      src: `${relevantPreview.src}`,
+                      type: "video/mp4",
+                    },
+                  ]
+                : [],
            }}
            seekOptions={{}}
            onReady={(player) => {
              playerRef.current = player;

+              if (!relevantPreview) {
+                return;
+              }
+
              if (!isInitiallyVisible) {
                player.pause(); // autoplay + pause is required for iOS
              }
@@ -249,28 +248,10 @@ function PreviewContent({
            }}
          />
        </div>
-        <LuPlayCircle className="absolute z-10 left-1 bottom-1 w-4 h-4 text-white text-opacity-60" />
+        {relevantPreview && (
+          <LuPlayCircle className="absolute z-10 left-1 bottom-1 w-4 h-4 text-white text-opacity-60" />
+        )}
      </>
    );
  }
 }
-
-function isCurrentHour(timestamp: number) {
-  const now = new Date();
-  now.setMinutes(0, 0, 0);
-  return timestamp > now.getTime() / 1000;
-}
-
-function getPreviewWidth(camera: string, config: FrigateConfig) {
-  const detect = config.cameras[camera].detect;
-
-  if (detect.width / detect.height < 1) {
-    return "w-1/2";
-  }
-
-  if (detect.width / detect.height < 16 / 9) {
-    return "w-2/3";
-  }
-
-  return "w-full";
-}