chris/frigate
1
0
Fork 0
forked from Github/frigate
Code Pull Requests Packages Activity

Live player fixes (#13143)

Browse Source 
* Jump to live when exceeding buffer time threshold in MSE player

* clean up

* Try adjusting playback rate instead of jumping to live

* clean up

* fallback to webrtc if enabled before jsmpeg

* baseline

* clean up

* remove comments

* adaptive playback rate and intelligent switching improvements

* increase logging and reset live mode after camera is no longer active on dashboard only

* jump to live on safari/iOS

* clean up

* clean up

* refactor camera live mode hook

* remove key listener

* resolve conflicts
This commit is contained in:
Josh Hawkins
2024-08-17 13:16:48 -05:00
committed by GitHub
parent 4133e454c4
commit 8c23ede683
7 changed files with 228 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
web/src/components/player/LivePlayer.tsx
View File

@@ -13,7 +13,6 @@ import {
LivePlayerMode,
VideoResolutionType,
} from "@/types/live";
import useCameraLiveMode from "@/hooks/use-camera-live-mode";
import { getIconForLabel } from "@/utils/iconUtil";
import Chip from "../indicators/Chip";
import { capitalizeFirstLetter } from "@/utils/stringUtil";
@@ -25,7 +24,7 @@ type LivePlayerProps = {
containerRef?: React.MutableRefObject<HTMLDivElement | null>;
className?: string;
cameraConfig: CameraConfig;
preferredLiveMode?: LivePlayerMode;
preferredLiveMode: LivePlayerMode;
showStillWithoutActivity?: boolean;
windowVisible?: boolean;
playAudio?: boolean;
@@ -36,6 +35,7 @@ type LivePlayerProps = {
onClick?: () => void;
setFullResolution?: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
onError?: (error: LivePlayerError) => void;
onResetLiveMode?: () => void;
};
export default function LivePlayer({
@@ -54,6 +54,7 @@ export default function LivePlayer({
onClick,
setFullResolution,
onError,
onResetLiveMode,
}: LivePlayerProps) {
const internalContainerRef = useRef<HTMLDivElement | null>(null);
// camera activity
@@ -70,8 +71,6 @@ export default function LivePlayer({
// camera live state
const liveMode = useCameraLiveMode(cameraConfig, preferredLiveMode);
const [liveReady, setLiveReady] = useState(false);
const liveReadyRef = useRef(liveReady);
@@ -91,6 +90,7 @@ export default function LivePlayer({
const timer = setTimeout(() => {
if (liveReadyRef.current && !cameraActiveRef.current) {
setLiveReady(false);
onResetLiveMode?.();
}
}, 500);
@@ -152,7 +152,7 @@ export default function LivePlayer({
let player;
if (!autoLive) {
player = null;
} else if (liveMode == "webrtc") {
} else if (preferredLiveMode == "webrtc") {
player = (
<WebRtcPlayer
className={`size-full rounded-lg md:rounded-2xl ${liveReady ? "" : "hidden"}`}
@@ -166,7 +166,7 @@ export default function LivePlayer({
onError={onError}
/>
);
} else if (liveMode == "mse") {
} else if (preferredLiveMode == "mse") {
if ("MediaSource" in window || "ManagedMediaSource" in window) {
player = (
<MSEPlayer
@@ -187,7 +187,7 @@ export default function LivePlayer({
</div>
);
}
} else if (liveMode == "jsmpeg") {
} else if (preferredLiveMode == "jsmpeg") {
if (cameraActive || !showStillWithoutActivity || liveReady) {
player = (
<JSMpegPlayer

140
web/src/components/player/MsePlayer.tsx
View File

@@ -32,6 +32,7 @@ function MSEPlayer({
onError,
}: MSEPlayerProps) {
const RECONNECT_TIMEOUT: number = 10000;
const BUFFERING_COOLDOWN_TIMEOUT: number = 5000;
const CODECS: string[] = [
"avc1.640029", // H.264 high 4.1 (Chromecast 1st and 2nd Gen)
@@ -46,6 +47,11 @@ function MSEPlayer({
const visibilityCheck: boolean = !pip;
const [isPlaying, setIsPlaying] = useState(false);
const lastJumpTimeRef = useRef(0);
const MAX_BUFFER_ENTRIES = 10; // Size of the rolling window of buffered times
const bufferTimes = useRef<number[]>([]);
const bufferIndex = useRef(0);
const [wsState, setWsState] = useState<number>(WebSocket.CLOSED);
const [connectTS, setConnectTS] = useState<number>(0);
@@ -133,6 +139,13 @@ function MSEPlayer({
}
}, [bufferTimeout]);
const handlePause = useCallback(() => {
// don't let the user pause the live stream
if (isPlaying && playbackEnabled) {
videoRef.current?.play();
}
}, [isPlaying, playbackEnabled]);
const onOpen = () => {
setWsState(WebSocket.OPEN);
@@ -193,6 +206,7 @@ function MSEPlayer({
const onMse = () => {
if ("ManagedMediaSource" in window) {
// safari
const MediaSource = window.ManagedMediaSource;
msRef.current?.addEventListener(
@@ -224,6 +238,7 @@ function MSEPlayer({
videoRef.current.srcObject = msRef.current;
}
} else {
// non safari
msRef.current?.addEventListener(
"sourceopen",
() => {
@@ -247,15 +262,35 @@ function MSEPlayer({
},
{ once: true },
);
videoRef.current!.src = URL.createObjectURL(msRef.current!);
videoRef.current!.srcObject = null;
if (videoRef.current && msRef.current) {
videoRef.current.src = URL.createObjectURL(msRef.current);
videoRef.current.srcObject = null;
}
}
play();
onmessageRef.current["mse"] = (msg) => {
if (msg.type !== "mse") return;
const sb = msRef.current?.addSourceBuffer(msg.value);
let sb: SourceBuffer | undefined;
try {
sb = msRef.current?.addSourceBuffer(msg.value);
if (sb?.mode) {
sb.mode = "segments";
}
} catch (e) {
// Safari sometimes throws this error
if (e instanceof DOMException && e.name === "InvalidStateError") {
if (wsRef.current) {
onDisconnect();
}
onError?.("mse-decode");
return;
} else {
throw e; // Re-throw if it's not the error we're handling
}
}
sb?.addEventListener("updateend", () => {
if (sb.updating) return;
@@ -302,6 +337,43 @@ function MSEPlayer({
return video.buffered.end(video.buffered.length - 1) - video.currentTime;
};
const jumpToLive = () => {
if (!videoRef.current) return;
const buffered = videoRef.current.buffered;
if (buffered.length > 0) {
const liveEdge = buffered.end(buffered.length - 1);
// Jump to the live edge
videoRef.current.currentTime = liveEdge - 0.75;
lastJumpTimeRef.current = Date.now();
}
};
const calculateAdaptiveBufferThreshold = () => {
const filledEntries = bufferTimes.current.length;
const sum = bufferTimes.current.reduce((a, b) => a + b, 0);
const averageBufferTime = filledEntries ? sum / filledEntries : 0;
return averageBufferTime * (isSafari || isIOS ? 3 : 1.5);
};
const calculateAdaptivePlaybackRate = (
bufferTime: number,
bufferThreshold: number,
) => {
const alpha = 0.2; // aggressiveness of playback rate increase
const beta = 0.5; // steepness of exponential growth
// don't adjust playback rate if we're close enough to live
if (
(bufferTime <= bufferThreshold && bufferThreshold < 3) ||
bufferTime < 3
) {
return 1;
}
const rate = 1 + alpha * Math.exp(beta * bufferTime - bufferThreshold);
return Math.min(rate, 2);
};
useEffect(() => {
if (!playbackEnabled) {
return;
@@ -386,21 +458,71 @@ function MSEPlayer({
handleLoadedMetadata?.();
onPlaying?.();
setIsPlaying(true);
lastJumpTimeRef.current = Date.now();
}}
muted={!audioEnabled}
onPause={() => videoRef.current?.play()}
onPause={handlePause}
onProgress={() => {
const bufferTime = getBufferedTime(videoRef.current);
if (
videoRef.current &&
(videoRef.current.playbackRate === 1 || bufferTime < 3)
) {
if (bufferTimes.current.length < MAX_BUFFER_ENTRIES) {
bufferTimes.current.push(bufferTime);
} else {
bufferTimes.current[bufferIndex.current] = bufferTime;
bufferIndex.current =
(bufferIndex.current + 1) % MAX_BUFFER_ENTRIES;
}
}
const bufferThreshold = calculateAdaptiveBufferThreshold();
// if we have > 3 seconds of buffered data and we're still not playing,
// something might be wrong - maybe codec issue, no audio, etc
// so mark the player as playing so that error handlers will fire
if (
!isPlaying &&
playbackEnabled &&
getBufferedTime(videoRef.current) > 3
) {
if (!isPlaying && playbackEnabled && bufferTime > 3) {
setIsPlaying(true);
lastJumpTimeRef.current = Date.now();
onPlaying?.();
}
// if we have more than 10 seconds of buffer, something's wrong so error out
if (
isPlaying &&
playbackEnabled &&
(bufferThreshold > 10 || bufferTime > 10)
) {
onDisconnect();
onError?.("stalled");
}
const playbackRate = calculateAdaptivePlaybackRate(
bufferTime,
bufferThreshold,
);
// if we're above our rolling average threshold or have > 3 seconds of
// buffered data and we're playing, we may have drifted from actual live
// time, so increase playback rate to compensate - non safari/ios only
if (
videoRef.current &&
isPlaying &&
playbackEnabled &&
Date.now() - lastJumpTimeRef.current > BUFFERING_COOLDOWN_TIMEOUT
) {
// Jump to live on Safari/iOS due to a change of playback rate causing re-buffering
if (isSafari || isIOS) {
if (bufferTime > 3) {
jumpToLive();
}
} else {
videoRef.current.playbackRate = playbackRate;
}
}
if (onError != undefined) {
if (videoRef.current?.paused) {
return;