initial commit

frigate/record.py

@@ -3,6 +3,7 @@ import itertools
 import logging
 import os
 import random
+import shutil
 import string
 import subprocess as sp
 import threading
@@ -10,9 +11,11 @@ from pathlib import Path
 
 import psutil
 
+from peewee import JOIN
+
 from frigate.config import FrigateConfig
-from frigate.const import RECORD_DIR
-from frigate.models import Recordings
+from frigate.const import CACHE_DIR, RECORD_DIR
+from frigate.models import Event, Recordings
 
 logger = logging.getLogger(__name__)
 
@@ -45,8 +48,10 @@ class RecordingMaintainer(threading.Thread):
     def move_files(self):
         recordings = [
             d
-            for d in os.listdir(RECORD_DIR)
-            if os.path.isfile(os.path.join(RECORD_DIR, d)) and d.endswith(".mp4")
+            for d in os.listdir(CACHE_DIR)
+            if os.path.isfile(os.path.join(CACHE_DIR, d))
+            and d.endswith(".mp4")
+            and not d.startswith("tmp_clip")
         ]
 
         files_in_use = []
@@ -57,7 +62,7 @@ class RecordingMaintainer(threading.Thread):
                 flist = process.open_files()
                 if flist:
                     for nt in flist:
-                        if nt.path.startswith(RECORD_DIR):
+                        if nt.path.startswith(CACHE_DIR):
                             files_in_use.append(nt.path.split("/")[-1])
             except:
                 continue
@@ -66,6 +71,7 @@ class RecordingMaintainer(threading.Thread):
             if f in files_in_use:
                 continue
 
+            cache_path = os.path.join(CACHE_DIR, f)
             basename = os.path.splitext(f)[0]
             camera, date = basename.rsplit("-", maxsplit=1)
             start_time = datetime.datetime.strptime(date, "%Y%m%d%H%M%S")
@@ -78,7 +84,7 @@ class RecordingMaintainer(threading.Thread):
                 "format=duration",
                 "-of",
                 "default=noprint_wrappers=1:nokey=1",
-                f"{os.path.join(RECORD_DIR, f)}",
+                f"{cache_path}",
             ]
             p = sp.run(ffprobe_cmd, capture_output=True)
             if p.returncode == 0:
@@ -86,7 +92,7 @@ class RecordingMaintainer(threading.Thread):
                 end_time = start_time + datetime.timedelta(seconds=duration)
             else:
                 logger.info(f"bad file: {f}")
-                os.remove(os.path.join(RECORD_DIR, f))
+                Path(cache_path).unlink(missing_ok=True)
                 continue
 
             directory = os.path.join(
@@ -99,7 +105,7 @@ class RecordingMaintainer(threading.Thread):
             file_name = f"{start_time.strftime('%M.%S.mp4')}"
             file_path = os.path.join(directory, file_name)
 
-            os.rename(os.path.join(RECORD_DIR, f), file_path)
+            shutil.move(cache_path, file_path)
 
             rand_id = "".join(
                 random.choices(string.ascii_lowercase + string.digits, k=6)
@@ -113,7 +119,135 @@ class RecordingMaintainer(threading.Thread):
                 duration=duration,
             )
 
+    def expire_recordings(self):
+        event_recordings = Recordings.select(
+            Recordings.id.alias("recording_id"),
+            Recordings.camera,
+            Recordings.path,
+            Recordings.end_time,
+            Event.id.alias("event_id"),
+            Event.label,
+        ).join(
+            Event,
+            on=(
+                (Recordings.camera == Event.camera)
+                & (
+                    (Recordings.start_time.between(Event.start_time, Event.end_time))
+                    | (Recordings.end_time.between(Event.start_time, Event.end_time))
+                ),
+            ),
+        )
+
+        retain = {}
+        for recording in event_recordings:
+            # Set default to delete
+            if recording.path not in retain:
+                retain[recording.path] = False
+
+            # Handle deleted cameras that still have recordings and events
+            if recording.camera in self.config.cameras:
+                record_config = self.config.cameras[recording.camera].record
+            else:
+                record_config = self.config.record
+
+            # Check event retention and set to True if within window
+            expire_days_event = (
+                0
+                if not record_config.events.enabled
+                else record_config.events.retain.objects.get(
+                    recording.event.label, record_config.events.retain.default
+                )
+            )
+            expire_before_event = (
+                datetime.datetime.now() - datetime.timedelta(days=expire_days_event)
+            ).timestamp()
+            if recording.end_time >= expire_before_event:
+                retain[recording.path] = True
+
+            # Check recording retention and set to True if within window
+            expire_days_record = record_config.retain_days
+            expire_before_record = (
+                datetime.datetime.now() - datetime.timedelta(days=expire_days_record)
+            ).timestamp()
+            if recording.end_time > expire_before_record:
+                retain[recording.path] = True
+
+        # Actually expire recordings
+        for path, keep in retain.items():
+            if not keep:
+                Path(path).unlink(missing_ok=True)
+                Recordings.delete_by_id(recording.recording_id)
+
+        # Update Event
+        event_no_recordings = (
+            Event.select()
+            .join(
+                Recordings,
+                JOIN.LEFT_OUTER,
+                on=(
+                    (Recordings.camera == Event.camera)
+                    & (
+                        (
+                            Recordings.start_time.between(
+                                Event.start_time, Event.end_time
+                            )
+                        )
+                        | (
+                            Recordings.end_time.between(
+                                Event.start_time, Event.end_time
+                            )
+                        )
+                    ),
+                ),
+            )
+            .where(Recordings.id.is_null())
+        )
+        update = Event.update(has_clip=False).where(Event.id << event_no_recordings)
+        update.execute()
+
+        event_paths = list(retain.keys())
+
+        # Handle deleted cameras
+        no_camera_recordings: Recordings = Recordings.select().where(
+            Recordings.camera.not_in(list(self.config.cameras.keys())),
+            Recordings.path.not_in(event_paths),
+        )
+
+        for recording in no_camera_recordings:
+            expire_days = self.config.record.retain_days
+            expire_before = (
+                datetime.datetime.now() - datetime.timedelta(days=expire_days)
+            ).timestamp()
+            if recording.end_time >= expire_before:
+                Path(recording.path).unlink(missing_ok=True)
+                Recordings.delete_by_id(recording.id)
+
+        # When deleting recordings without events, we have to keep at LEAST the configured max clip duration
+        for camera, config in self.config.cameras.items():
+            min_end = (
+                datetime.datetime.now()
+                - datetime.timedelta(seconds=config.record.events.max_seconds)
+            ).timestamp()
+            recordings: Recordings = Recordings.select().where(
+                Recordings.camera == camera,
+                Recordings.path.not_in(event_paths),
+                Recordings.end_time < min_end,
+            )
+
+            for recording in recordings:
+                expire_days = config.record.retain_days
+                expire_before = (
+                    datetime.datetime.now() - datetime.timedelta(days=expire_days)
+                ).timestamp()
+                if recording.end_time >= expire_before:
+                    Path(recording.path).unlink(missing_ok=True)
+                    Recordings.delete_by_id(recording.id)
+
     def expire_files(self):
+        default_expire = (
+            datetime.datetime.now().timestamp()
+            - SECONDS_IN_DAY * self.config.record.retain_days
+        )
         delete_before = {}
         for name, camera in self.config.cameras.items():
             delete_before[name] = (
@@ -122,19 +256,22 @@ class RecordingMaintainer(threading.Thread):
             )
 
         for p in Path("/media/frigate/recordings").rglob("*.mp4"):
-            if not p.parent.name in delete_before:
+            # Ignore files that have a record in the recordings DB
+            if Recordings.select().where(Recordings.path == str(p)).count():
                 continue
-            if p.stat().st_mtime < delete_before[p.parent.name]:
-                Recordings.delete().where(Recordings.path == str(p)).execute()
+            if p.stat().st_mtime < delete_before.get(p.parent.name, default_expire):
                 p.unlink(missing_ok=True)
 
     def run(self):
-        for counter in itertools.cycle(range(60)):
-            if self.stop_event.wait(10):
+        # only expire events every 10 minutes, but check for new files every 5 seconds
+        for counter in itertools.cycle(range(120)):
+            if self.stop_event.wait(5):
                 logger.info(f"Exiting recording maintenance...")
                 break
 
-            # only expire events every 10 minutes, but check for new files every 10 seconds
+            if counter % 12 == 0:
+                self.expire_recordings()
+
             if counter == 0:
                 self.expire_files()
                 remove_empty_directories(RECORD_DIR)