refactor(web): responsive images on content size, throttle AutoUpdatingCameraImage

fixes #544

README.md

@@ -14,9 +14,25 @@ Use of a [Google Coral Accelerator](https://coral.ai/products/) is optional, but
 - Uses a very low overhead motion detection to determine where to run object detection
 - Object detection with TensorFlow runs in separate processes for maximum FPS
 - Communicates over MQTT for easy integration into other systems
+- Records video clips of detected objects
 - 24/7 recording
 - Re-streaming via RTMP to reduce the number of connections to your camera
 
 ## Documentation
 
 View the documentation at https://blakeblackshear.github.io/frigate
+
+## Screenshots
+Integration into HomeAssistant
+<div>
+<a href="docs/static/img/media_browser.png"><img src="docs/static/img/media_browser.png" height=400></a>
+<a href="docs/static/img/notification.png"><img src="docs/static/img/notification.png" height=400></a>
+</div>
+
+Also comes with a builtin UI:
+<div>
+<a href="docs/static/img/home-ui.png"><img src="docs/static/img/home-ui.png" height=400></a>
+<a href="docs/static/img/camera-ui.png"><img src="docs/static/img/camera-ui.png" height=400></a>
+</div>
+
+![Events](docs/static/img/events-ui.png)

docs/docs/installation.md

@@ -21,6 +21,7 @@ HassOS users can install via the addon repository. Frigate requires an MQTT serv
 ## Docker
 
 Make sure you choose the right image for your architecture:
+
 |Arch|Image Name|
 |-|-|
 |amd64|blakeblackshear/frigate:stable-amd64|

docs/docs/troubleshooting.md

@@ -6,7 +6,6 @@ title: Troubleshooting
 ### My mjpeg stream or snapshots look green and crazy
 This almost always means that the width/height defined for your camera are not correct. Double check the resolution with vlc or another player. Also make sure you don't have the width and height values backwards.
 
-Example:
 ![mismatched-resolution](/img/mismatched-resolution.jpg)
 
 ## "[mov,mp4,m4a,3gp,3g2,mj2 @ 0x5639eeb6e140] moov atom not found"

docs/docusaurus.config.js

@@ -9,6 +9,10 @@ module.exports = {
   organizationName: 'blakeblackshear',
   projectName: 'frigate',
   themeConfig: {
+    algolia: {
+      apiKey: '81ec882db78f7fed05c51daf973f0362',
+      indexName: 'frigate'
+    },
     navbar: {
       title: 'Frigate',
       logo: {

web/src/App.jsx

@@ -26,7 +26,7 @@ export default function App() {
     <Config.Provider value={config}>
       <div className="md:flex flex-col md:flex-row md:min-h-screen w-full bg-gray-100 dark:bg-gray-800 text-gray-900 dark:text-white">
         <Sidebar />
-        <div className="p-4 min-w-0">
+        <div className="flex-auto p-4 lg:pl-8 lg:pr-8 min-w-0">
           <Router>
             <CameraMap path="/cameras/:camera/editor" />
             <Camera path="/cameras/:camera" />
@@ -39,5 +39,4 @@ export default function App() {
       </div>
     </Config.Provider>
   );
-  return;
 }

web/src/CameraMap.jsx

@@ -27,14 +27,26 @@ export default function CameraMasks({ camera, url }) {
     zones,
   } = cameraConfig;
 
+  const resizeObserver = useMemo(
+    () =>
+      new ResizeObserver((entries) => {
+        window.requestAnimationFrame(() => {
+          if (Array.isArray(entries) && entries.length) {
+            const scaledWidth = entries[0].contentRect.width;
+            const scale = scaledWidth / width;
+            setImageScale(scale);
+          }
+        });
+      }),
+    [camera, width, setImageScale]
+  );
+
   useEffect(() => {
     if (!imageRef.current) {
       return;
     }
-    const scaledWidth = imageRef.current.width;
-    const scale = scaledWidth / width;
-    setImageScale(scale);
-  }, [imageRef.current, setImageScale]);
+    resizeObserver.observe(imageRef.current);
+  }, [resizeObserver, imageRef.current]);
 
   const [motionMaskPoints, setMotionMaskPoints] = useState(
     Array.isArray(motionMask)
@@ -177,7 +189,7 @@ ${Object.keys(zonePoints)
   const handleAddObjectMask = useCallback(() => {
     const n = Object.keys(objectMaskPoints).filter((name) => name.startsWith('object_')).length;
     const newObjectName = `object_${n}`;
-    const newObjectMaskPoints = { ...objectMaskPoints, [newObjectName]: [] };
+    const newObjectMaskPoints = { ...objectMaskPoints, [newObjectName]: [[]] };
     setObjectMaskPoints(newObjectMaskPoints);
     setEditing({ set: newObjectMaskPoints, key: newObjectName, subkey: 0, fn: setObjectMaskPoints });
   }, [objectMaskPoints, setObjectMaskPoints, setEditing]);
@@ -185,7 +197,7 @@ ${Object.keys(zonePoints)
   const handleRemoveObjectMask = useCallback(
     (key, subkey) => {
       const newObjectMaskPoints = { ...objectMaskPoints };
-      delete newObjectMaskPoints[key];
+      delete newObjectMaskPoints[key][subkey];
       setObjectMaskPoints(newObjectMaskPoints);
     },
     [objectMaskPoints, setObjectMaskPoints]
@@ -205,6 +217,20 @@ ${Object.keys(objectMaskPoints)
   .join('\n')}`);
   }, [objectMaskPoints]);
 
+  const handleAddToObjectMask = useCallback(
+    (key) => {
+      const newObjectMaskPoints = { ...objectMaskPoints, [key]: [...objectMaskPoints[key], []] };
+      setObjectMaskPoints(newObjectMaskPoints);
+      setEditing({
+        set: newObjectMaskPoints,
+        key,
+        subkey: newObjectMaskPoints[key].length - 1,
+        fn: setObjectMaskPoints,
+      });
+    },
+    [objectMaskPoints, setObjectMaskPoints, setEditing]
+  );
+
   const handleChangeSnap = useCallback(
     (id, value) => {
       setSnap(value);
@@ -226,10 +252,10 @@ ${Object.keys(objectMaskPoints)
 
       <Box className="space-y-4">
         <div className="relative">
-          <img ref={imageRef} className="w-full" src={`${apiHost}/api/${camera}/latest.jpg`} />
+          <img ref={imageRef} src={`${apiHost}/api/${camera}/latest.jpg`} />
           <EditableMask
             onChange={handleUpdateEditable}
-            points={editing.subkey ? editing.set[editing.key][editing.subkey] : editing.set[editing.key]}
+            points={'subkey' in editing ? editing.set[editing.key][editing.subkey] : editing.set[editing.key]}
             scale={imageScale}
             snap={snap}
             width={width}
@@ -268,6 +294,7 @@ ${Object.keys(objectMaskPoints)
           isMulti
           editing={editing}
           title="Object masks"
+          onAdd={handleAddToObjectMask}
           onCopy={handleCopyObjectMasks}
           onCreate={handleAddObjectMask}
           onEdit={handleEditObjectMask}
@@ -397,6 +424,7 @@ function MaskValues({
   isMulti = false,
   editing,
   title,
+  onAdd,
   onCopy,
   onCreate,
   onEdit,
@@ -438,6 +466,14 @@ function MaskValues({
     [onRemove]
   );
 
+  const handleAdd = useCallback(
+    (event) => {
+      const { key } = event.target.dataset;
+      onAdd(key);
+    },
+    [onAdd]
+  );
+
   return (
     <Box className="overflow-hidden" onmouseover={handleMousein} onmouseout={handleMouseout}>
       <div class="flex space-x-4">
@@ -454,15 +490,20 @@ function MaskValues({
             return (
               <div>
                 {`    ${mainkey}:\n      mask:\n`}
+                {onAdd && showButtons ? (
+                  <Button className="absolute -mt-12 right-0 font-sans" data-key={mainkey} onClick={handleAdd}>
+                    {`Add to ${mainkey}`}
+                  </Button>
+                ) : null}
                 {points[mainkey].map((item, subkey) => (
                   <Item
                     mainkey={mainkey}
                     subkey={subkey}
                     editing={editing}
                     handleEdit={handleEdit}
+                    handleRemove={handleRemove}
                     points={item}
                     showButtons={showButtons}
-                    handleRemove={handleRemove}
                     yamlKeyPrefix={yamlKeyPrefix}
                   />
                 ))}
@@ -473,10 +514,11 @@ function MaskValues({
               <Item
                 mainkey={mainkey}
                 editing={editing}
+                handleAdd={onAdd ? handleAdd : undefined}
                 handleEdit={handleEdit}
+                handleRemove={handleRemove}
                 points={points[mainkey]}
                 showButtons={showButtons}
-                handleRemove={handleRemove}
                 yamlKeyPrefix={yamlKeyPrefix}
               />
             );
@@ -487,7 +529,7 @@ function MaskValues({
   );
 }
 
-function Item({ mainkey, subkey, editing, handleEdit, points, showButtons, handleRemove, yamlKeyPrefix }) {
+function Item({ mainkey, subkey, editing, handleEdit, points, showButtons, handleAdd, handleRemove, yamlKeyPrefix }) {
   return (
     <span
       data-key={mainkey}

web/src/Cameras.jsx

@@ -1,5 +1,6 @@
 import { h } from 'preact';
 import Box from './components/Box';
+import CameraImage from './components/CameraImage';
 import Events from './Events';
 import Heading from './components/Heading';
 import { route } from 'preact-router';
@@ -23,7 +24,6 @@ export default function Cameras() {
 }
 
 function Camera({ name }) {
-  const apiHost = useContext(ApiHost);
   const href = `/cameras/${name}`;
 
   return (
@@ -32,7 +32,7 @@ function Camera({ name }) {
       href={href}
     >
       <Heading size="base">{name}</Heading>
-      <img className="w-full" src={`${apiHost}/api/${name}/latest.jpg`} />
+      <CameraImage camera={name} />
     </Box>
   );
 }

web/src/Debug.jsx

@@ -1,4 +1,6 @@
 import { h } from 'preact';
+import Box from './components/Box';
+import Button from './components/Button';
 import Heading from './components/Heading';
 import Link from './components/Link';
 import { ApiHost, Config } from './context';
@@ -39,59 +41,73 @@ export default function Debug() {
   const cameraNames = Object.keys(cameras);
   const cameraDataKeys = Object.keys(cameras[cameraNames[0]]);
 
+  const handleCopyConfig = useCallback(async () => {
+    await window.navigator.clipboard.writeText(JSON.stringify(config, null, 2));
+  }, [config]);
+
   return (
-    <div>
+    <div class="space-y-4">
       <Heading>
         Debug <span className="text-sm">{service.version}</span>
       </Heading>
-      <Table className="w-full">
-        <Thead>
-          <Tr>
-            <Th>detector</Th>
-            {detectorDataKeys.map((name) => (
-              <Th>{name.replace('_', ' ')}</Th>
-            ))}
-          </Tr>
-        </Thead>
-        <Tbody>
-          {detectorNames.map((detector, i) => (
-            <Tr index={i}>
-              <Td>{detector}</Td>
+
+      <Box>
+        <Table className="w-full">
+          <Thead>
+            <Tr>
+              <Th>detector</Th>
               {detectorDataKeys.map((name) => (
-                <Td key={`${name}-${detector}`}>{detectors[detector][name]}</Td>
+                <Th>{name.replace('_', ' ')}</Th>
               ))}
             </Tr>
-          ))}
-        </Tbody>
-      </Table>
-
-      <Table className="w-full">
-        <Thead>
-          <Tr>
-            <Th>camera</Th>
-            {cameraDataKeys.map((name) => (
-              <Th>{name.replace('_', ' ')}</Th>
+          </Thead>
+          <Tbody>
+            {detectorNames.map((detector, i) => (
+              <Tr index={i}>
+                <Td>{detector}</Td>
+                {detectorDataKeys.map((name) => (
+                  <Td key={`${name}-${detector}`}>{detectors[detector][name]}</Td>
+                ))}
+              </Tr>
             ))}
-          </Tr>
-        </Thead>
-        <Tbody>
-          {cameraNames.map((camera, i) => (
-            <Tr index={i}>
-              <Td>
-                <Link href={`/cameras/${camera}`}>{camera}</Link>
-              </Td>
+          </Tbody>
+        </Table>
+      </Box>
+
+      <Box>
+        <Table className="w-full">
+          <Thead>
+            <Tr>
+              <Th>camera</Th>
               {cameraDataKeys.map((name) => (
-                <Td key={`${name}-${camera}`}>{cameras[camera][name]}</Td>
+                <Th>{name.replace('_', ' ')}</Th>
               ))}
             </Tr>
-          ))}
-        </Tbody>
-      </Table>
+          </Thead>
+          <Tbody>
+            {cameraNames.map((camera, i) => (
+              <Tr index={i}>
+                <Td>
+                  <Link href={`/cameras/${camera}`}>{camera}</Link>
+                </Td>
+                {cameraDataKeys.map((name) => (
+                  <Td key={`${name}-${camera}`}>{cameras[camera][name]}</Td>
+                ))}
+              </Tr>
+            ))}
+          </Tbody>
+        </Table>
+      </Box>
 
-      <Heading size="sm">Config</Heading>
-      <pre className="font-mono overflow-y-scroll overflow-x-scroll max-h-96 rounded bg-white dark:bg-gray-900">
-        {JSON.stringify(config, null, 2)}
-      </pre>
+      <Box className="relative">
+        <Heading size="sm">Config</Heading>
+        <Button className="absolute top-4 right-8" onClick={handleCopyConfig}>
+          Copy to Clipboard
+        </Button>
+        <pre className="overflow-auto font-mono text-gray-900 dark:text-gray-100 rounded bg-gray-100 dark:bg-gray-800 p-2 max-h-96">
+          {JSON.stringify(config, null, 2)}
+        </pre>
+      </Box>
     </div>
   );
 }

web/src/Events.jsx

@@ -11,7 +11,7 @@ export default function Events({ url } = {}) {
   const apiHost = useContext(ApiHost);
   const [events, setEvents] = useState([]);
 
-  const searchParams = new URL(`${window.location.protocol}//${window.location.host}${url || '/events'}`).searchParams;
+  const { pathname, searchParams } = new URL(`${window.location.protocol}//${window.location.host}${url || '/events'}`);
   const searchParamsString = searchParams.toString();
 
   useEffect(async () => {
@@ -23,7 +23,7 @@ export default function Events({ url } = {}) {
   const searchKeys = Array.from(searchParams.keys());
 
   return (
-    <div className="space-y-4">
+    <div className="space-y-4 w-full">
       <Heading>Events</Heading>
 
       {searchKeys.length ? (
@@ -32,9 +32,10 @@ export default function Events({ url } = {}) {
           <div className="flex flex-wrap space-x-2">
             {searchKeys.map((filterKey) => (
               <UnFilterable
-                paramName={filterKey}
-                searchParams={searchParamsString}
                 name={`${filterKey}: ${searchParams.get(filterKey)}`}
+                paramName={filterKey}
+                pathname={pathname}
+                searchParams={searchParamsString}
               />
             ))}
           </div>
@@ -42,7 +43,7 @@ export default function Events({ url } = {}) {
       ) : null}
 
       <Box className="min-w-0 overflow-auto">
-        <Table>
+        <Table className="w-full">
           <Thead>
             <Tr>
               <Th></Th>
@@ -71,17 +72,32 @@ export default function Events({ url } = {}) {
                       </a>
                     </Td>
                     <Td>
-                      <Filterable searchParams={searchParamsString} paramName="camera" name={camera} />
+                      <Filterable
+                        pathname={pathname}
+                        searchParams={searchParamsString}
+                        paramName="camera"
+                        name={camera}
+                      />
                     </Td>
                     <Td>
-                      <Filterable searchParams={searchParamsString} paramName="label" name={label} />
+                      <Filterable
+                        pathname={pathname}
+                        searchParams={searchParamsString}
+                        paramName="label"
+                        name={label}
+                      />
                     </Td>
                     <Td>{(score * 100).toFixed(2)}%</Td>
                     <Td>
                       <ul>
                         {zones.map((zone) => (
                           <li>
-                            <Filterable searchParams={searchParamsString} paramName="zone" name={zone} />
+                            <Filterable
+                              pathname={pathname}
+                              searchParams={searchParamsString}
+                              paramName="zone"
+                              name={zone}
+                            />
                           </li>
                         ))}
                       </ul>
@@ -100,19 +116,19 @@ export default function Events({ url } = {}) {
   );
 }
 
-function Filterable({ searchParams, paramName, name }) {
+function Filterable({ pathname, searchParams, paramName, name }) {
   const params = new URLSearchParams(searchParams);
   params.set(paramName, name);
-  return <Link href={`?${params.toString()}`}>{name}</Link>;
+  return <Link href={`${pathname}?${params.toString()}`}>{name}</Link>;
 }
 
-function UnFilterable({ searchParams, paramName, name }) {
+function UnFilterable({ pathname, searchParams, paramName, name }) {
   const params = new URLSearchParams(searchParams);
   params.delete(paramName);
   return (
     <a
       className="bg-gray-700 text-white px-3 py-1 rounded-md hover:bg-gray-300 hover:text-gray-900 dark:bg-gray-300 dark:text-gray-900 dark:hover:bg-gray-700 dark:hover:text-white"
-      href={`?${params.toString()}`}
+      href={`${pathname}?${params.toString()}`}
     >
       {name}
     </a>

web/src/components/AutoUpdatingCameraImage.jsx

@@ -1,27 +1,29 @@
 import { h } from 'preact';
+import CameraImage from './CameraImage';
 import { ApiHost, Config } from '../context';
-import { useCallback, useEffect, useContext, useState } from 'preact/hooks';
+import { useCallback, useState } from 'preact/hooks';
 
-export default function AutoUpdatingCameraImage({ camera, searchParams }) {
-  const config = useContext(Config);
-  const apiHost = useContext(ApiHost);
-  const cameraConfig = config.cameras[camera];
+const MIN_LOAD_TIMEOUT_MS = 200;
 
+export default function AutoUpdatingCameraImage({ camera, searchParams, showFps = true }) {
   const [key, setKey] = useState(Date.now());
-  useEffect(() => {
-    const timeoutId = setTimeout(() => {
-      setKey(Date.now());
-    }, 500);
-    return () => {
-      clearTimeout(timeoutId);
-    };
-  }, [key, searchParams]);
+  const [fps, setFps] = useState(0);
+
+  const handleLoad = useCallback(() => {
+    const loadTime = Date.now() - key;
+    setFps((1000 / Math.max(loadTime, MIN_LOAD_TIMEOUT_MS)).toFixed(1));
+    setTimeout(
+      () => {
+        setKey(Date.now());
+      },
+      loadTime > MIN_LOAD_TIMEOUT_MS ? 1 : MIN_LOAD_TIMEOUT_MS
+    );
+  }, [key, searchParams, setFps]);
 
   return (
-    <img
-      className="w-full"
-      src={`${apiHost}/api/${camera}/latest.jpg?cache=${key}&${searchParams}`}
-      alt={`Auto-updating ${camera} image`}
-    />
+    <div>
+      <CameraImage camera={camera} onload={handleLoad} searchParams={`cache=${key}&${searchParams}`} />
+      {showFps ? <span className="text-xs">Displaying at {fps}fps</span> : null}
+    </div>
   );
 }

web/src/components/CameraImage.jsx

@@ -0,0 +1,60 @@
+import { h } from 'preact';
+import { ApiHost, Config } from '../context';
+import { useCallback, useEffect, useContext, useMemo, useRef, useState } from 'preact/hooks';
+
+export default function CameraImage({ camera, onload, searchParams = '' }) {
+  const config = useContext(Config);
+  const apiHost = useContext(ApiHost);
+  const [availableWidth, setAvailableWidth] = useState(0);
+  const [loadedSrc, setLoadedSrc] = useState(null);
+  const containerRef = useRef(null);
+
+  const { name, width, height } = config.cameras[camera];
+  const aspectRatio = width / height;
+
+  const resizeObserver = useMemo(() => {
+    return new ResizeObserver((entries) => {
+      window.requestAnimationFrame(() => {
+        if (Array.isArray(entries) && entries.length) {
+          setAvailableWidth(entries[0].contentRect.width);
+        }
+      });
+    });
+  }, [setAvailableWidth, width]);
+
+  useEffect(() => {
+    if (!containerRef.current) {
+      return;
+    }
+    resizeObserver.observe(containerRef.current);
+  }, [resizeObserver, containerRef.current]);
+
+  const scaledHeight = useMemo(() => Math.min(Math.ceil(availableWidth / aspectRatio), height), [
+    availableWidth,
+    aspectRatio,
+    height,
+  ]);
+
+  const img = useMemo(() => new Image(), [camera]);
+  img.onload = useCallback(
+    (event) => {
+      const src = event.path[0].currentSrc;
+      setLoadedSrc(src);
+      onload && onload(event);
+    },
+    [searchParams, onload]
+  );
+
+  useEffect(() => {
+    if (!scaledHeight) {
+      return;
+    }
+    img.src = `${apiHost}/api/${name}/latest.jpg?h=${scaledHeight}${searchParams ? `&${searchParams}` : ''}`;
+  }, [apiHost, name, img, searchParams, scaledHeight]);
+
+  return (
+    <div ref={containerRef}>
+      {loadedSrc ? <img width={scaledHeight * aspectRatio} height={scaledHeight} src={loadedSrc} alt={name} /> : null}
+    </div>
+  );
+}

web/src/components/Switch.jsx

@@ -2,13 +2,9 @@ import { h } from 'preact';
 import { useCallback, useState } from 'preact/hooks';
 
 export default function Switch({ checked, label, id, onChange }) {
-  const handleChange = useCallback(
-    (event) => {
-      console.log(event.target.checked, !checked);
-      onChange(id, !checked);
-    },
-    [id, onChange, checked]
-  );
+  const handleChange = useCallback(() => {
+    onChange(id, !checked);
+  }, [id, onChange, checked]);
 
   return (
     <label for={id} className="flex items-center cursor-pointer">