Live player fixes and improvements (#12202)

* Live player fixes and improvements * remove comment * Simplify wording
2024-06-29 10:02:30 -05:00
parent 48a87b16b8
commit 53a2a865f1
9 changed files with 267 additions and 85 deletions
--- a/web/src/components/player/BirdseyeLivePlayer.tsx
+++ b/web/src/components/player/BirdseyeLivePlayer.tsx
@@ -12,7 +12,7 @@ type LivePlayerProps = {
  birdseyeConfig: BirdseyeConfig;
  liveMode: LivePlayerMode;
  onClick?: () => void;
-  containerRef?: React.MutableRefObject<HTMLDivElement | null>;
+  containerRef: React.MutableRefObject<HTMLDivElement | null>;
 };

 export default function BirdseyeLivePlayer({
@@ -54,6 +54,7 @@ export default function BirdseyeLivePlayer({
        width={birdseyeConfig.width}
        height={birdseyeConfig.height}
        containerRef={containerRef}
+        playbackEnabled={true}
      />
    );
  } else {
@@ -62,6 +63,7 @@ export default function BirdseyeLivePlayer({

  return (
    <div
+      ref={containerRef}
      className={cn(
        "relative flex w-full cursor-pointer justify-center",
        className,
--- a/web/src/components/player/JSMpegPlayer.tsx
+++ b/web/src/components/player/JSMpegPlayer.tsx
@@ -3,14 +3,15 @@ import { useResizeObserver } from "@/hooks/resize-observer";
 import { cn } from "@/lib/utils";
 // @ts-expect-error we know this doesn't have types
 import JSMpeg from "@cycjimmy/jsmpeg-player";
-import React, { useEffect, useMemo, useRef, useId, useState } from "react";
+import React, { useEffect, useMemo, useRef, useState } from "react";

 type JSMpegPlayerProps = {
  className?: string;
  camera: string;
  width: number;
  height: number;
-  containerRef?: React.MutableRefObject<HTMLDivElement | null>;
+  containerRef: React.MutableRefObject<HTMLDivElement | null>;
+  playbackEnabled: boolean;
  onPlaying?: () => void;
 };

@@ -20,18 +21,21 @@ export default function JSMpegPlayer({
  height,
  className,
  containerRef,
+  playbackEnabled,
  onPlaying,
 }: JSMpegPlayerProps) {
  const url = `${baseUrl.replace(/^http/, "ws")}live/jsmpeg/${camera}`;
-  const playerRef = useRef<HTMLDivElement | null>(null);
-  const videoRef = useRef(null);
+  const videoRef = useRef<HTMLDivElement>(null);
+  const canvasRef = useRef<HTMLCanvasElement>(null);
  const internalContainerRef = useRef<HTMLDivElement | null>(null);
  const onPlayingRef = useRef(onPlaying);
  const [showCanvas, setShowCanvas] = useState(false);

  const selectedContainerRef = useMemo(
-    () => containerRef ?? internalContainerRef,
-    [containerRef, internalContainerRef],
+    () => (containerRef.current ? containerRef : internalContainerRef),
+    // we know that these deps are correct
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [containerRef, containerRef.current, internalContainerRef],
  );

  const [{ width: containerWidth, height: containerHeight }] =
@@ -83,39 +87,64 @@ export default function JSMpegPlayer({
    }
  }, [scaledHeight, aspectRatio]);

-  const uniqueId = useId();
-
  useEffect(() => {
    onPlayingRef.current = onPlaying;
  }, [onPlaying]);

  useEffect(() => {
-    if (!playerRef.current || videoRef.current) {
+    if (!selectedContainerRef?.current || !url) {
      return;
    }

-    videoRef.current = new JSMpeg.VideoElement(
-      playerRef.current,
-      url,
-      { canvas: `#${CSS.escape(uniqueId)}` },
-      {
-        protocols: [],
-        audio: false,
-        videoBufferSize: 1024 * 1024 * 4,
-        onPlay: () => {
-          setShowCanvas(true);
-          onPlayingRef.current?.();
-        },
-      },
-    );
-  }, [url, uniqueId]);
+    const videoWrapper = videoRef.current;
+    const canvas = canvasRef.current;
+    let hasData = false;
+    let videoElement: JSMpeg.VideoElement | null = null;
+
+    if (videoWrapper && playbackEnabled) {
+      // Delayed init to avoid issues with react strict mode
+      const initPlayer = setTimeout(() => {
+        videoElement = new JSMpeg.VideoElement(
+          videoWrapper,
+          url,
+          { canvas: canvas },
+          {
+            protocols: [],
+            audio: false,
+            videoBufferSize: 1024 * 1024 * 4,
+            onVideoDecode: () => {
+              if (!hasData) {
+                hasData = true;
+                setShowCanvas(true);
+                onPlayingRef.current?.();
+              }
+            },
+          },
+        );
+      }, 0);
+
+      return () => {
+        clearTimeout(initPlayer);
+        if (videoElement) {
+          try {
+            // this causes issues in react strict mode
+            // https://stackoverflow.com/questions/76822128/issue-with-cycjimmy-jsmpeg-player-in-react-18-cannot-read-properties-of-null-o
+            videoElement.destroy();
+            // eslint-disable-next-line no-empty
+          } catch (e) {}
+        }
+      };
+    }
+    // we know that these deps are correct
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [playbackEnabled, url]);

  return (
-    <div className={className}>
-      <div className="size-full" ref={internalContainerRef}>
-        <div ref={playerRef} className={cn("jsmpeg", !showCanvas && "hidden")}>
+    <div className={cn(className, !containerRef.current && "size-full")}>
+      <div className="internal-jsmpeg-container" ref={internalContainerRef}>
+        <div ref={videoRef} className={cn("jsmpeg", !showCanvas && "hidden")}>
          <canvas
-            id={uniqueId}
+            ref={canvasRef}
            style={{
              width: scaledWidth ?? width,
              height: scaledHeight ?? height,
--- a/web/src/components/player/LivePlayer.tsx
+++ b/web/src/components/player/LivePlayer.tsx
@@ -2,7 +2,7 @@ import WebRtcPlayer from "./WebRTCPlayer";
 import { CameraConfig } from "@/types/frigateConfig";
 import AutoUpdatingCameraImage from "../camera/AutoUpdatingCameraImage";
 import ActivityIndicator from "../indicators/activity-indicator";
-import { useCallback, useEffect, useMemo, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import MSEPlayer from "./MsePlayer";
 import JSMpegPlayer from "./JSMpegPlayer";
 import { MdCircle } from "react-icons/md";
@@ -55,6 +55,7 @@ export default function LivePlayer({
  setFullResolution,
  onError,
 }: LivePlayerProps) {
+  const internalContainerRef = useRef<HTMLDivElement | null>(null);
  // camera activity

  const { activeMotion, activeTracking, objects, offline } =
@@ -73,20 +74,12 @@ export default function LivePlayer({

  const [liveReady, setLiveReady] = useState(false);
  useEffect(() => {
-    if (!autoLive) {
-      return;
-    }
-
-    if (!liveReady) {
-      if (cameraActive && liveMode == "jsmpeg") {
-        setLiveReady(true);
-      }
-
+    if (!autoLive || !liveReady) {
      return;
    }

    if (!cameraActive) {
-      setTimeout(() => setLiveReady(false), 500);
+      setLiveReady(false);
    }
    // live mode won't change
    // eslint-disable-next-line react-hooks/exhaustive-deps
@@ -181,7 +174,8 @@ export default function LivePlayer({
          camera={cameraConfig.live.stream_name}
          width={cameraConfig.detect.width}
          height={cameraConfig.detect.height}
-          containerRef={containerRef}
+          playbackEnabled={cameraActive || !showStillWithoutActivity}
+          containerRef={containerRef ?? internalContainerRef}
          onPlaying={playerIsPlaying}
        />
      );
@@ -194,7 +188,7 @@ export default function LivePlayer({

  return (
    <div
-      ref={cameraRef}
+      ref={cameraRef ?? internalContainerRef}
      data-camera={cameraConfig.name}
      className={cn(
        "relative flex w-full cursor-pointer justify-center outline",
--- a/web/src/components/player/MsePlayer.tsx
+++ b/web/src/components/player/MsePlayer.tsx
@@ -31,7 +31,7 @@ function MSEPlayer({
  setFullResolution,
  onError,
 }: MSEPlayerProps) {
-  const RECONNECT_TIMEOUT: number = 30000;
+  const RECONNECT_TIMEOUT: number = 10000;

  const CODECS: string[] = [
    "avc1.640029", // H.264 high 4.1 (Chromecast 1st and 2nd Gen)
@@ -45,10 +45,12 @@ function MSEPlayer({
  ];

  const visibilityCheck: boolean = !pip;
+  const [safariPlaying, setSafariPlaying] = useState(false);

  const [wsState, setWsState] = useState<number>(WebSocket.CLOSED);
  const [connectTS, setConnectTS] = useState<number>(0);
  const [bufferTimeout, setBufferTimeout] = useState<NodeJS.Timeout>();
+  const [errorCount, setErrorCount] = useState<number>(0);

  const videoRef = useRef<HTMLVideoElement>(null);
  const wsRef = useRef<WebSocket | null>(null);
@@ -117,12 +119,21 @@ function MSEPlayer({
  }, [wsURL]);

  const onDisconnect = useCallback(() => {
-    if (wsRef.current && wsState == WebSocket.OPEN) {
+    if (bufferTimeout) {
+      clearTimeout(bufferTimeout);
+      setBufferTimeout(undefined);
+    }
+
+    if ((isSafari || isIOS) && safariPlaying) {
+      setSafariPlaying(false);
+    }
+
+    if (wsRef.current && wsState != WebSocket.CLOSED) {
      setWsState(WebSocket.CLOSED);
      wsRef.current.close();
      wsRef.current = null;
    }
-  }, [wsState]);
+  }, [wsState, bufferTimeout, safariPlaying]);

  const onOpen = () => {
    setWsState(WebSocket.OPEN);
@@ -162,6 +173,26 @@ function MSEPlayer({
    reconnect();
  };

+  const sendWithTimeout = (value: object, timeout: number) => {
+    return new Promise<void>((resolve, reject) => {
+      const timeoutId = setTimeout(() => {
+        reject(new Error("Timeout waiting for response"));
+      }, timeout);
+
+      send(value);
+
+      // Override the onmessageRef handler for mse type to resolve the promise on response
+      const originalHandler = onmessageRef.current["mse"];
+      onmessageRef.current["mse"] = (msg) => {
+        if (msg.type === "mse") {
+          clearTimeout(timeoutId);
+          if (originalHandler) originalHandler(msg);
+          resolve();
+        }
+      };
+    });
+  };
+
  const onMse = () => {
    if ("ManagedMediaSource" in window) {
      const MediaSource = window.ManagedMediaSource;
@@ -169,10 +200,22 @@ function MSEPlayer({
      msRef.current?.addEventListener(
        "sourceopen",
        () => {
-          send({
-            type: "mse",
-            // @ts-expect-error for typing
-            value: codecs(MediaSource.isTypeSupported),
+          sendWithTimeout(
+            {
+              type: "mse",
+              // @ts-expect-error for typing
+              value: codecs(MediaSource.isTypeSupported),
+            },
+            3000,
+          ).catch(() => {
+            if (wsRef.current) {
+              onDisconnect();
+            }
+            if (isIOS || isSafari) {
+              onError?.("mse-decode");
+            } else {
+              onError?.("startup");
+            }
          });
        },
        { once: true },
@@ -187,9 +230,21 @@ function MSEPlayer({
        "sourceopen",
        () => {
          URL.revokeObjectURL(videoRef.current?.src || "");
-          send({
-            type: "mse",
-            value: codecs(MediaSource.isTypeSupported),
+          sendWithTimeout(
+            {
+              type: "mse",
+              value: codecs(MediaSource.isTypeSupported),
+            },
+            3000,
+          ).catch(() => {
+            if (wsRef.current) {
+              onDisconnect();
+            }
+            if (isIOS || isSafari) {
+              onError?.("mse-decode");
+            } else {
+              onError?.("startup");
+            }
          });
        },
        { once: true },
@@ -260,10 +315,6 @@ function MSEPlayer({

    return () => {
      onDisconnect();
-      if (bufferTimeout) {
-        clearTimeout(bufferTimeout);
-        setBufferTimeout(undefined);
-      }
    };
    // we know that these deps are correct
    // eslint-disable-next-line react-hooks/exhaustive-deps
@@ -305,6 +356,23 @@ function MSEPlayer({
    videoRef.current.requestPictureInPicture();
  }, [pip, videoRef]);

+  // ensure we disconnect for slower connections
+
+  useEffect(() => {
+    if (wsState === WebSocket.OPEN && !playbackEnabled) {
+      if (bufferTimeout) {
+        clearTimeout(bufferTimeout);
+        setBufferTimeout(undefined);
+      }
+
+      setTimeout(() => {
+        if (!playbackEnabled) onDisconnect();
+      }, 10000);
+    }
+    // we know that these deps are correct
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [playbackEnabled]);
+
  return (
    <video
      ref={videoRef}
@@ -317,7 +385,8 @@ function MSEPlayer({
      }}
      muted={!audioEnabled}
      onProgress={() => {
-        if (isSafari || isIOS) {
+        if ((isSafari || isIOS) && !safariPlaying) {
+          setSafariPlaying(true);
          onPlaying?.();
        }
        if (onError != undefined) {
@@ -334,8 +403,10 @@ function MSEPlayer({
            setTimeout(() => {
              if (
                document.visibilityState === "visible" &&
-                wsRef.current != null
+                wsRef.current != null &&
+                videoRef.current
              ) {
+                onDisconnect();
                onError("stalled");
              }
            }, 3000),
@@ -347,6 +418,9 @@ function MSEPlayer({
          // @ts-expect-error code does exist
          e.target.error.code == MediaError.MEDIA_ERR_NETWORK
        ) {
+          if (wsRef.current) {
+            onDisconnect();
+          }
          onError?.("startup");
        }

@@ -355,15 +429,22 @@ function MSEPlayer({
          e.target.error.code == MediaError.MEDIA_ERR_DECODE &&
          (isSafari || isIOS)
        ) {
+          if (wsRef.current) {
+            onDisconnect();
+          }
          onError?.("mse-decode");
-          clearTimeout(bufferTimeout);
-          setBufferTimeout(undefined);
        }

+        setErrorCount((prevCount) => prevCount + 1);
+
        if (wsRef.current) {
-          wsRef.current.close();
-          wsRef.current = null;
-          reconnect(5000);
+          onDisconnect();
+          if (errorCount >= 3) {
+            // too many mse errors, try jsmpeg
+            onError?.("startup");
+          } else {
+            reconnect(5000);
+          }
        }
      }}
    />