chris/frigate
1
0
Fork 0
forked from Github/frigate
Code Pull Requests Packages Activity

Periodically sync for stale recordings (#8433)

Browse Source 
* Periodically cleanup recordings files / DB

* Make automatic sync limited ot last 36 hours
This commit is contained in:
Nicolas Mowen
2023-11-03 20:21:29 -06:00
committed by GitHub
parent 4f7b710112
commit 63233a5830
4 changed files with 133 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
frigate/record/cleanup.py
View File

@@ -3,17 +3,15 @@
import datetime
import itertools
import logging
import os
import threading
from multiprocessing.synchronize import Event as MpEvent
from pathlib import Path
from peewee import DatabaseError, chunked
from frigate.config import FrigateConfig, RetainModeEnum
from frigate.const import CACHE_DIR, RECORD_DIR
from frigate.models import Event, Recordings, RecordingsToDelete
from frigate.record.util import remove_empty_directories
from frigate.models import Event, Recordings
from frigate.record.util import remove_empty_directories, sync_recordings
from frigate.util.builtin import get_tomorrow_at_time
logger = logging.getLogger(__name__)
@@ -180,76 +178,25 @@ class RecordingCleanup(threading.Thread):
logger.debug("End all cameras.")
logger.debug("End expire recordings.")
def sync_recordings(self) -> None:
"""Check the db for stale recordings entries that don't exist in the filesystem."""
logger.debug("Start sync recordings.")
# get all recordings in the db
recordings = Recordings.select(Recordings.id, Recordings.path)
# get all recordings files on disk and put them in a set
files_on_disk = {
os.path.join(root, file)
for root, _, files in os.walk(RECORD_DIR)
for file in files
}
# Use pagination to process records in chunks
page_size = 1000
num_pages = (recordings.count() + page_size - 1) // page_size
recordings_to_delete = set()
for page in range(num_pages):
for recording in recordings.paginate(page, page_size):
if recording.path not in files_on_disk:
recordings_to_delete.add(recording.id)
# convert back to list of dictionaries for insertion
recordings_to_delete = [
{"id": recording_id} for recording_id in recordings_to_delete
]
if len(recordings_to_delete) / max(1, recordings.count()) > 0.5:
logger.debug(
f"Deleting {(len(recordings_to_delete) / recordings.count()):2f}% of recordings could be due to configuration error. Aborting..."
)
return
logger.debug(
f"Deleting {len(recordings_to_delete)} recordings with missing files"
)
# create a temporary table for deletion
RecordingsToDelete.create_table(temporary=True)
# insert ids to the temporary table
max_inserts = 1000
for batch in chunked(recordings_to_delete, max_inserts):
RecordingsToDelete.insert_many(batch).execute()
try:
# delete records in the main table that exist in the temporary table
query = Recordings.delete().where(
Recordings.id.in_(RecordingsToDelete.select(RecordingsToDelete.id))
)
query.execute()
except DatabaseError as e:
logger.error(f"Database error during delete: {e}")
logger.debug("End sync recordings.")
def run(self) -> None:
# on startup sync recordings with disk if enabled
if self.config.record.sync_on_startup:
self.sync_recordings()
sync_recordings(limited=False)
next_sync = get_tomorrow_at_time(3)
# Expire tmp clips every minute, recordings and clean directories every hour.
for counter in itertools.cycle(range(self.config.record.expire_interval)):
if self.stop_event.wait(60):
logger.info("Exiting recording cleanup...")
break
self.clean_tmp_clips()
if datetime.datetime.now().astimezone(datetime.timezone.utc) > next_sync:
sync_recordings(limited=True)
next_sync = get_tomorrow_at_time(3)
if counter == 0:
self.expire_recordings()
remove_empty_directories(RECORD_DIR)