chris/frigate
1
0
Fork 0
forked from Github/frigate
Code Pull Requests Packages Activity

False positives (#6217)

Browse Source 
* add false positive submission

* switch timeline events to x,y,w,h

* update docs

* fix type checks

* convert to upsert

* fix config test
This commit is contained in:
Blake Blackshear
2023-04-24 07:24:28 -05:00
committed by GitHub
parent 9dca1e1d9f
commit 0d16bd0144
18 changed files with 572 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
frigate/config.py
View File

@@ -18,6 +18,7 @@ from frigate.const import (
REGEX_CAMERA_NAME,
YAML_EXT,
)
from frigate.detectors.detector_config import BaseDetectorConfig
from frigate.util import (
create_mask,
deep_merge,
@@ -770,7 +771,7 @@ def verify_config_roles(camera_config: CameraConfig) -> None:
def verify_valid_live_stream_name(
frigate_config: FrigateConfig, camera_config: CameraConfig
) -> None:
) -> ValueError | None:
"""Verify that a restream exists to use for live view."""
if (
camera_config.live.stream_name
@@ -848,7 +849,7 @@ class FrigateConfig(FrigateBaseModel):
model: ModelConfig = Field(
default_factory=ModelConfig, title="Detection model configuration."
)
detectors: Dict[str, DetectorConfig] = Field(
detectors: Dict[str, BaseDetectorConfig] = Field(
default=DEFAULT_DETECTORS,
title="Detector hardware configuration.",
)
@@ -1031,7 +1032,15 @@ class FrigateConfig(FrigateBaseModel):
detector_config.model.dict(exclude_unset=True),
config.model.dict(exclude_unset=True),
)
if not "path" in merged_model:
if detector_config.type == "cpu":
merged_model["path"] = "/cpu_model.tflite"
elif detector_config.type == "edgetpu":
merged_model["path"] = "/edgetpu_model.tflite"
detector_config.model = ModelConfig.parse_obj(merged_model)
detector_config.model.compute_model_hash()
config.detectors[key] = detector_config
return config

13
frigate/detectors/detector_config.py
View File

@@ -1,3 +1,4 @@
import hashlib
import logging
from enum import Enum
from typing import Dict, List, Optional, Tuple, Union, Literal
@@ -49,6 +50,7 @@ class ModelConfig(BaseModel):
)
_merged_labelmap: Optional[Dict[int, str]] = PrivateAttr()
_colormap: Dict[int, Tuple[int, int, int]] = PrivateAttr()
_model_hash: str = PrivateAttr()
@property
def merged_labelmap(self) -> Dict[int, str]:
@@ -58,6 +60,10 @@ class ModelConfig(BaseModel):
def colormap(self) -> Dict[int, Tuple[int, int, int]]:
return self._colormap
@property
def model_hash(self) -> str:
return self._model_hash
def __init__(self, **config):
super().__init__(**config)
@@ -67,6 +73,13 @@ class ModelConfig(BaseModel):
}
self._colormap = {}
def compute_model_hash(self) -> None:
with open(self.path, "rb") as f:
file_hash = hashlib.md5()
while chunk := f.read(8192):
file_hash.update(chunk)
self._model_hash = file_hash.hexdigest()
def create_colormap(self, enabled_labels: set[str]) -> None:
"""Get a list of colors for enabled labels."""
cmap = plt.cm.get_cmap("tab10", len(enabled_labels))

2
frigate/detectors/plugins/cpu_tfl.py
View File

@@ -27,7 +27,7 @@ class CpuTfl(DetectionApi):
def __init__(self, detector_config: CpuDetectorConfig):
self.interpreter = Interpreter(
model_path=detector_config.model.path or "/cpu_model.tflite",
model_path=detector_config.model.path,
num_threads=detector_config.num_threads or 3,
)

2
frigate/detectors/plugins/edgetpu_tfl.py
View File

@@ -37,7 +37,7 @@ class EdgeTpuTfl(DetectionApi):
edge_tpu_delegate = load_delegate("libedgetpu.so.1.0", device_config)
logger.info("TPU found")
self.interpreter = Interpreter(
model_path=detector_config.model.path or "/edgetpu_model.tflite",
model_path=detector_config.model.path,
experimental_delegates=[edge_tpu_delegate],
)
except ValueError:

165
frigate/events.py
View File

@@ -12,6 +12,7 @@ from frigate.const import CLIPS_DIR
from frigate.models import Event
from frigate.timeline import TimelineSourceEnum
from frigate.types import CameraMetricsTypes
from frigate.util import to_relative_box
from multiprocessing.queues import Queue
from multiprocessing.synchronize import Event as MpEvent
@@ -20,22 +21,18 @@ from typing import Dict
logger = logging.getLogger(__name__)
def should_insert_db(prev_event: Event, current_event: Event) -> bool:
"""If current event has new clip or snapshot."""
return (not prev_event["has_clip"] and not prev_event["has_snapshot"]) and (
current_event["has_clip"] or current_event["has_snapshot"]
)
def should_update_db(prev_event: Event, current_event: Event) -> bool:
"""If current_event has updated fields and (clip or snapshot)."""
if current_event["has_clip"] or current_event["has_snapshot"]:
# if this is the first time has_clip or has_snapshot turned true
if not prev_event["has_clip"] and not prev_event["has_snapshot"]:
return True
# or if any of the following values changed
if (
prev_event["top_score"] != current_event["top_score"]
or prev_event["entered_zones"] != current_event["entered_zones"]
or prev_event["thumbnail"] != current_event["thumbnail"]
or prev_event["has_clip"] != current_event["has_clip"]
or prev_event["has_snapshot"] != current_event["has_snapshot"]
or prev_event["end_time"] != current_event["end_time"]
):
return True
return False
@@ -85,81 +82,91 @@ class EventProcessor(threading.Thread):
)
)
event_config: EventsConfig = self.config.cameras[camera].record.events
# if this is the first message, just store it and continue, its not time to insert it in the db
if event_type == "start":
self.events_in_process[event_data["id"]] = event_data
continue
elif event_type == "update" and should_insert_db(
self.events_in_process[event_data["id"]], event_data
):
if should_update_db(self.events_in_process[event_data["id"]], event_data):
camera_config = self.config.cameras[camera]
event_config: EventsConfig = camera_config.record.events
width = camera_config.detect.width
height = camera_config.detect.height
first_detector = list(self.config.detectors.values())[0]
start_time = event_data["start_time"] - event_config.pre_capture
end_time = (
None
if event_data["end_time"] is None
else event_data["end_time"] + event_config.post_capture
)
# score of the snapshot
score = (
None
if event_data["snapshot"] is None
else event_data["snapshot"]["score"]
)
# detection region in the snapshot
region = (
None
if event_data["snapshot"] is None
else to_relative_box(
width,
height,
event_data["snapshot"]["region"],
)
)
# bounding box for the snapshot
box = (
None
if event_data["snapshot"] is None
else to_relative_box(
width,
height,
event_data["snapshot"]["box"],
)
)
# keep these from being set back to false because the event
# may have started while recordings and snapshots were enabled
# this would be an issue for long running events
if self.events_in_process[event_data["id"]]["has_clip"]:
event_data["has_clip"] = True
if self.events_in_process[event_data["id"]]["has_snapshot"]:
event_data["has_snapshot"] = True
event = {
Event.id: event_data["id"],
Event.label: event_data["label"],
Event.camera: camera,
Event.start_time: start_time,
Event.end_time: end_time,
Event.top_score: event_data["top_score"],
Event.score: score,
Event.zones: list(event_data["entered_zones"]),
Event.thumbnail: event_data["thumbnail"],
Event.region: region,
Event.box: box,
Event.has_clip: event_data["has_clip"],
Event.has_snapshot: event_data["has_snapshot"],
Event.model_hash: first_detector.model.model_hash,
Event.model_type: first_detector.model.model_type,
Event.detector_type: first_detector.type,
}
(
Event.insert(event)
.on_conflict(
conflict_target=[Event.id],
update=event,
)
.execute()
)
# update the stored copy for comparison on future update messages
self.events_in_process[event_data["id"]] = event_data
# TODO: this will generate a lot of db activity possibly
Event.insert(
id=event_data["id"],
label=event_data["label"],
camera=camera,
start_time=event_data["start_time"] - event_config.pre_capture,
end_time=None,
top_score=event_data["top_score"],
false_positive=event_data["false_positive"],
zones=list(event_data["entered_zones"]),
thumbnail=event_data["thumbnail"],
region=event_data["region"],
box=event_data["box"],
area=event_data["area"],
has_clip=event_data["has_clip"],
has_snapshot=event_data["has_snapshot"],
).execute()
elif event_type == "update" and should_update_db(
self.events_in_process[event_data["id"]], event_data
):
self.events_in_process[event_data["id"]] = event_data
# TODO: this will generate a lot of db activity possibly
Event.update(
label=event_data["label"],
camera=camera,
start_time=event_data["start_time"] - event_config.pre_capture,
end_time=None,
top_score=event_data["top_score"],
false_positive=event_data["false_positive"],
zones=list(event_data["entered_zones"]),
thumbnail=event_data["thumbnail"],
region=event_data["region"],
box=event_data["box"],
area=event_data["area"],
ratio=event_data["ratio"],
has_clip=event_data["has_clip"],
has_snapshot=event_data["has_snapshot"],
).where(Event.id == event_data["id"]).execute()
elif event_type == "end":
if event_data["has_clip"] or event_data["has_snapshot"]:
# Full update for valid end of event
Event.update(
label=event_data["label"],
camera=camera,
start_time=event_data["start_time"] - event_config.pre_capture,
end_time=event_data["end_time"] + event_config.post_capture,
top_score=event_data["top_score"],
false_positive=event_data["false_positive"],
zones=list(event_data["entered_zones"]),
thumbnail=event_data["thumbnail"],
region=event_data["region"],
box=event_data["box"],
area=event_data["area"],
ratio=event_data["ratio"],
has_clip=event_data["has_clip"],
has_snapshot=event_data["has_snapshot"],
).where(Event.id == event_data["id"]).execute()
else:
# Event ended after clip & snapshot disabled,
# only end time should be updated.
Event.update(
end_time=event_data["end_time"] + event_config.post_capture
).where(Event.id == event_data["id"]).execute()
if event_type == "end":
del self.events_in_process[event_data["id"]]
self.event_processed_queue.put((event_data["id"], camera))

101
frigate/http.py
View File

@@ -35,6 +35,7 @@ from frigate.config import FrigateConfig
from frigate.const import CLIPS_DIR, MAX_SEGMENT_DURATION, RECORD_DIR
from frigate.models import Event, Recordings, Timeline
from frigate.object_processing import TrackedObject
from frigate.plus import PlusApi
from frigate.stats import stats_snapshot
from frigate.util import (
clean_camera_user_pass,
@@ -42,6 +43,7 @@ from frigate.util import (
restart_frigate,
vainfo_hwaccel,
get_tz_modifiers,
to_relative_box,
)
from frigate.storage import StorageMaintainer
from frigate.version import VERSION
@@ -57,7 +59,7 @@ def create_app(
stats_tracking,
detected_frames_processor,
storage_maintainer: StorageMaintainer,
plus_api,
plus_api: PlusApi,
):
app = Flask(__name__)
@@ -179,6 +181,10 @@ def send_to_plus(id):
400,
)
include_annotation = (
request.json.get("include_annotation") if request.is_json else None
)
try:
event = Event.get(Event.id == id)
except DoesNotExist:
@@ -186,6 +192,10 @@ def send_to_plus(id):
logger.error(message)
return make_response(jsonify({"success": False, "message": message}), 404)
# events from before the conversion to relative dimensions cant include annotations
if any(d > 1 for d in event.box):
include_annotation = None
if event.end_time is None:
logger.error(f"Unable to load clean png for in-progress event: {event.id}")
return make_response(
@@ -238,9 +248,96 @@ def send_to_plus(id):
event.plus_id = plus_id
event.save()
if not include_annotation is None:
region = event.region
box = event.box
try:
current_app.plus_api.add_annotation(
event.plus_id,
box,
event.label,
)
except Exception as ex:
logger.exception(ex)
return make_response(
jsonify({"success": False, "message": str(ex)}),
400,
)
return make_response(jsonify({"success": True, "plus_id": plus_id}), 200)
@bp.route("/events/<id>/false_positive", methods=("PUT",))
def false_positive(id):
if not current_app.plus_api.is_active():
message = "PLUS_API_KEY environment variable is not set"
logger.error(message)
return make_response(
jsonify(
{
"success": False,
"message": message,
}
),
400,
)
try:
event = Event.get(Event.id == id)
except DoesNotExist:
message = f"Event {id} not found"
logger.error(message)
return make_response(jsonify({"success": False, "message": message}), 404)
# events from before the conversion to relative dimensions cant include annotations
if any(d > 1 for d in event.box):
message = f"Events prior to 0.13 cannot be submitted as false positives"
logger.error(message)
return make_response(jsonify({"success": False, "message": message}), 400)
if event.false_positive:
message = f"False positive already submitted to Frigate+"
logger.error(message)
return make_response(jsonify({"success": False, "message": message}), 400)
if not event.plus_id:
plus_response = send_to_plus(id)
if plus_response.status_code != 200:
return plus_response
# need to refetch the event now that it has a plus_id
event = Event.get(Event.id == id)
region = event.region
box = event.box
# provide top score if score is unavailable
score = event.top_score if event.score is None else event.score
try:
current_app.plus_api.add_false_positive(
event.plus_id,
region,
box,
score,
event.label,
event.model_hash,
event.model_type,
event.detector_type,
)
except Exception as ex:
logger.exception(ex)
return make_response(
jsonify({"success": False, "message": str(ex)}),
400,
)
event.false_positive = True
event.save()
return make_response(jsonify({"success": True, "plus_id": event.plus_id}), 200)
@bp.route("/events/<id>/retain", methods=("DELETE",))
def delete_retain(id):
try:
@@ -654,6 +751,8 @@ def events():
Event.retain_indefinitely,
Event.sub_label,
Event.top_score,
Event.false_positive,
Event.box,
]
if camera != "all":

4
frigate/models.py
View File

@@ -19,6 +19,7 @@ class Event(Model): # type: ignore[misc]
start_time = DateTimeField()
end_time = DateTimeField()
top_score = FloatField()
score = FloatField()
false_positive = BooleanField()
zones = JSONField()
thumbnail = TextField()
@@ -30,6 +31,9 @@ class Event(Model): # type: ignore[misc]
retain_indefinitely = BooleanField(default=False)
ratio = FloatField(default=1.0)
plus_id = CharField(max_length=30)
model_hash = CharField(max_length=32)
detector_type = CharField(max_length=32)
model_type = CharField(max_length=32)
class Timeline(Model): # type: ignore[misc]

2
frigate/object_processing.py
View File

@@ -185,7 +185,7 @@ class TrackedObject:
"id": self.obj_data["id"],
"camera": self.camera,
"frame_time": self.obj_data["frame_time"],
"snapshot_time": snapshot_time,
"snapshot": self.thumbnail_data,
"label": self.obj_data["label"],
"sub_label": self.obj_data.get("sub_label"),
"top_score": self.top_score,

63
frigate/plus.py
View File

@@ -3,6 +3,7 @@ import json
import logging
import os
import re
from typing import List
import requests
from frigate.const import PLUS_ENV_VAR, PLUS_API_HOST
from requests.models import Response
@@ -79,6 +80,13 @@ class PlusApi:
json=data,
)
def _put(self, path: str, data: dict) -> Response:
return requests.put(
f"{self.host}/v1/{path}",
headers=self._get_authorization_header(),
json=data,
)
def is_active(self) -> bool:
return self._is_active
@@ -124,3 +132,58 @@ class PlusApi:
# return image id
return str(presigned_urls.get("imageId"))
def add_false_positive(
self,
plus_id: str,
region: List[float],
bbox: List[float],
score: float,
label: str,
model_hash: str,
model_type: str,
detector_type: str,
) -> None:
r = self._put(
f"image/{plus_id}/false_positive",
{
"label": label,
"x": bbox[0],
"y": bbox[1],
"w": bbox[2],
"h": bbox[3],
"regionX": region[0],
"regionY": region[1],
"regionW": region[2],
"regionH": region[3],
"score": score,
"model_hash": model_hash,
"model_type": model_type,
"detector_type": detector_type,
},
)
if not r.ok:
raise Exception(r.text)
def add_annotation(
self,
plus_id: str,
bbox: List[float],
label: str,
difficult: bool = False,
) -> None:
r = self._put(
f"image/{plus_id}/annotation",
{
"label": label,
"x": bbox[0],
"y": bbox[1],
"w": bbox[2],
"h": bbox[3],
"difficult": difficult,
},
)
if not r.ok:
raise Exception(r.text)

7
frigate/test/test_config.py
View File

@@ -54,7 +54,8 @@ class TestConfig(unittest.TestCase):
"type": "openvino",
},
},
"model": {"path": "/default.tflite", "width": 512},
# needs to be a file that will exist, doesnt matter what
"model": {"path": "/etc/hosts", "width": 512},
}
frigate_config = FrigateConfig(**(deep_merge(config, self.minimal)))
@@ -72,10 +73,10 @@ class TestConfig(unittest.TestCase):
assert runtime_config.detectors["edgetpu"].device is None
assert runtime_config.detectors["openvino"].device is None
assert runtime_config.model.path == "/default.tflite"
assert runtime_config.model.path == "/etc/hosts"
assert runtime_config.detectors["cpu"].model.path == "/cpu_model.tflite"
assert runtime_config.detectors["edgetpu"].model.path == "/edgetpu_model.tflite"
assert runtime_config.detectors["openvino"].model.path == "/default.tflite"
assert runtime_config.detectors["openvino"].model.path == "/etc/hosts"
assert runtime_config.model.width == 512
assert runtime_config.detectors["cpu"].model.width == 512

95
frigate/timeline.py
View File

@@ -12,6 +12,8 @@ from frigate.models import Timeline
from multiprocessing.queues import Queue
from multiprocessing.synchronize import Event as MpEvent
from frigate.util import to_relative_box
logger = logging.getLogger(__name__)
@@ -64,77 +66,36 @@ class TimelineProcessor(threading.Thread):
"""Handle object detection."""
camera_config = self.config.cameras[camera]
timeline_entry = {
Timeline.timestamp: event_data["frame_time"],
Timeline.camera: camera,
Timeline.source: "tracked_object",
Timeline.source_id: event_data["id"],
Timeline.data: {
"box": to_relative_box(
camera_config.detect.width,
camera_config.detect.height,
event_data["box"],
),
"label": event_data["label"],
"region": to_relative_box(
camera_config.detect.width,
camera_config.detect.height,
event_data["region"],
),
},
}
if event_type == "start":
Timeline.insert(
timestamp=event_data["frame_time"],
camera=camera,
source="tracked_object",
source_id=event_data["id"],
class_type="visible",
data={
"box": [
event_data["box"][0] / camera_config.detect.width,
event_data["box"][1] / camera_config.detect.height,
event_data["box"][2] / camera_config.detect.width,
event_data["box"][3] / camera_config.detect.height,
],
"label": event_data["label"],
"region": [
event_data["region"][0] / camera_config.detect.width,
event_data["region"][1] / camera_config.detect.height,
event_data["region"][2] / camera_config.detect.width,
event_data["region"][3] / camera_config.detect.height,
],
},
).execute()
timeline_entry[Timeline.class_type] = "visible"
Timeline.insert(timeline_entry).execute()
elif (
event_type == "update"
and prev_event_data["current_zones"] != event_data["current_zones"]
and len(event_data["current_zones"]) > 0
):
Timeline.insert(
timestamp=event_data["frame_time"],
camera=camera,
source="tracked_object",
source_id=event_data["id"],
class_type="entered_zone",
data={
"box": [
event_data["box"][0] / camera_config.detect.width,
event_data["box"][1] / camera_config.detect.height,
event_data["box"][2] / camera_config.detect.width,
event_data["box"][3] / camera_config.detect.height,
],
"label": event_data["label"],
"region": [
event_data["region"][0] / camera_config.detect.width,
event_data["region"][1] / camera_config.detect.height,
event_data["region"][2] / camera_config.detect.width,
event_data["region"][3] / camera_config.detect.height,
],
"zones": event_data["current_zones"],
},
).execute()
timeline_entry[Timeline.class_type] = "entered_zone"
timeline_entry[Timeline.data]["zones"] = event_data["current_zones"]
Timeline.insert(timeline_entry).execute()
elif event_type == "end":
Timeline.insert(
timestamp=event_data["frame_time"],
camera=camera,
source="tracked_object",
source_id=event_data["id"],
class_type="gone",
data={
"box": [
event_data["box"][0] / camera_config.detect.width,
event_data["box"][1] / camera_config.detect.height,
event_data["box"][2] / camera_config.detect.width,
event_data["box"][3] / camera_config.detect.height,
],
"label": event_data["label"],
"region": [
event_data["region"][0] / camera_config.detect.width,
event_data["region"][1] / camera_config.detect.height,
event_data["region"][2] / camera_config.detect.width,
event_data["region"][3] / camera_config.detect.height,
],
},
).execute()
timeline_entry[Timeline.class_type] = "gone"
Timeline.insert(timeline_entry).execute()

11
frigate/util.py
View File

@@ -1065,3 +1065,14 @@ def get_tz_modifiers(tz_name: str) -> Tuple[str, str]:
hour_modifier = f"{hours_offset} hour"
minute_modifier = f"{minutes_offset} minute"
return hour_modifier, minute_modifier
def to_relative_box(
width: int, height: int, box: Tuple[int, int, int, int]
) -> Tuple[int, int, int, int]:
return (
box[0] / width, # x
box[1] / height, # y
(box[2] - box[0]) / width, # w
(box[3] - box[1]) / height, # h
)