chris/frigate
1
0
Fork 0
forked from Github/frigate
Code Pull Requests Packages Activity

Remove all AGPL licensed YOLO references from Frigate (#10717)

Browse Source 
* Remove yolov8 support from Frigate

* Remove yolov8 from dev

* Remove builds

* Formatting and remove yolov5

* Fix lint

* remove models download

---------

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>
This commit is contained in:
Blake Blackshear
2024-03-30 06:46:17 -04:00
committed by GitHub
parent 0223d6df60
commit 14235c42b9
16 changed files with 81 additions and 671 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
frigate/detectors/detector_config.py
View File

@@ -30,8 +30,6 @@ class InputTensorEnum(str, Enum):
class ModelTypeEnum(str, Enum):
ssd = "ssd"
yolox = "yolox"
yolov5 = "yolov5"
yolov8 = "yolov8"
class ModelConfig(BaseModel):

18
frigate/detectors/plugins/edgetpu_tfl.py
View File

@@ -6,7 +6,6 @@ from typing_extensions import Literal
from frigate.detectors.detection_api import DetectionApi
from frigate.detectors.detector_config import BaseDetectorConfig
from frigate.detectors.util import yolov8_postprocess
try:
from tflite_runtime.interpreter import Interpreter, load_delegate
@@ -58,26 +57,9 @@ class EdgeTpuTfl(DetectionApi):
self.model_type = detector_config.model.model_type
def detect_raw(self, tensor_input):
if self.model_type == "yolov8":
scale, zero_point = self.tensor_input_details[0]["quantization"]
tensor_input = (
(tensor_input - scale * zero_point * 255) * (1.0 / (scale * 255))
).astype(self.tensor_input_details[0]["dtype"])
self.interpreter.set_tensor(self.tensor_input_details[0]["index"], tensor_input)
self.interpreter.invoke()
if self.model_type == "yolov8":
scale, zero_point = self.tensor_output_details[0]["quantization"]
tensor_output = self.interpreter.get_tensor(
self.tensor_output_details[0]["index"]
)
tensor_output = (tensor_output.astype(np.float32) - zero_point) * scale
model_input_shape = self.tensor_input_details[0]["shape"]
tensor_output[:, [0, 2]] *= model_input_shape[2]
tensor_output[:, [1, 3]] *= model_input_shape[1]
return yolov8_postprocess(model_input_shape, tensor_output)
boxes = self.interpreter.tensor(self.tensor_output_details[0]["index"])()[0]
class_ids = self.interpreter.tensor(self.tensor_output_details[1]["index"])()[0]
scores = self.interpreter.tensor(self.tensor_output_details[2]["index"])()[0]

28
frigate/detectors/plugins/onnx.py
View File

@@ -1,4 +1,3 @@
import glob
import logging
import numpy as np
@@ -6,7 +5,7 @@ from typing_extensions import Literal
from frigate.detectors.detection_api import DetectionApi
from frigate.detectors.detector_config import BaseDetectorConfig
from frigate.detectors.util import preprocess, yolov8_postprocess
from frigate.detectors.util import preprocess
logger = logging.getLogger(__name__)
@@ -31,24 +30,6 @@ class ONNXDetector(DetectionApi):
)
raise
assert (
detector_config.model.model_type == "yolov8"
), "ONNX: detector_config.model.model_type: only yolov8 supported"
assert (
detector_config.model.input_tensor == "nhwc"
), "ONNX: detector_config.model.input_tensor: only nhwc supported"
if detector_config.model.input_pixel_format != "rgb":
logger.warn(
"ONNX: detector_config.model.input_pixel_format: should be 'rgb' for yolov8, but '{detector_config.model.input_pixel_format}' specified!"
)
assert detector_config.model.path is not None, (
"ONNX: No model.path configured, please configure model.path and model.labelmap_path; some suggestions: "
+ ", ".join(glob.glob("/config/model_cache/yolov8/*.onnx"))
+ " and "
+ ", ".join(glob.glob("/config/model_cache/yolov8/*_labels.txt"))
)
path = detector_config.model.path
logger.info(f"ONNX: loading {detector_config.model.path}")
self.model = onnxruntime.InferenceSession(path)
@@ -57,9 +38,10 @@ class ONNXDetector(DetectionApi):
def detect_raw(self, tensor_input):
model_input_name = self.model.get_inputs()[0].name
model_input_shape = self.model.get_inputs()[0].shape
tensor_input = preprocess(tensor_input, model_input_shape, np.float32)
# ruff: noqa: F841
tensor_output = self.model.run(None, {model_input_name: tensor_input})[0]
return yolov8_postprocess(model_input_shape, tensor_output)
raise Exception(
"No models are currently supported via onnx. See the docs for more info."
)

41
frigate/detectors/plugins/openvino.py
View File

@@ -131,44 +131,3 @@ class OvDetector(DetectionApi):
object_detected[6], object_detected[5], object_detected[:4]
)
return detections
elif self.ov_model_type == ModelTypeEnum.yolov8:
out_tensor = infer_request.get_output_tensor()
results = out_tensor.data[0]
output_data = np.transpose(results)
scores = np.max(output_data[:, 4:], axis=1)
if len(scores) == 0:
return np.zeros((20, 6), np.float32)
scores = np.expand_dims(scores, axis=1)
# add scores to the last column
dets = np.concatenate((output_data, scores), axis=1)
# filter out lines with scores below threshold
dets = dets[dets[:, -1] > 0.5, :]
# limit to top 20 scores, descending order
ordered = dets[dets[:, -1].argsort()[::-1]][:20]
detections = np.zeros((20, 6), np.float32)
for i, object_detected in enumerate(ordered):
detections[i] = self.process_yolo(
np.argmax(object_detected[4:-1]),
object_detected[-1],
object_detected[:4],
)
return detections
elif self.ov_model_type == ModelTypeEnum.yolov5:
out_tensor = infer_request.get_output_tensor()
output_data = out_tensor.data[0]
# filter out lines with scores below threshold
conf_mask = (output_data[:, 4] >= 0.5).squeeze()
output_data = output_data[conf_mask]
# limit to top 20 scores, descending order
ordered = output_data[output_data[:, 4].argsort()[::-1]][:20]
detections = np.zeros((20, 6), np.float32)
for i, object_detected in enumerate(ordered):
detections[i] = self.process_yolo(
np.argmax(object_detected[5:]),
object_detected[4],
object_detected[:4],
)
return detections

94
frigate/detectors/plugins/rknn.py
View File

@@ -1,10 +1,7 @@
import logging
import os.path
import urllib.request
from typing import Literal
import numpy as np
try:
from hide_warnings import hide_warnings
except: # noqa: E722
@@ -24,14 +21,6 @@ DETECTOR_KEY = "rknn"
supported_socs = ["rk3562", "rk3566", "rk3568", "rk3588"]
yolov8_suffix = {
"default-yolov8n": "n",
"default-yolov8s": "s",
"default-yolov8m": "m",
"default-yolov8l": "l",
"default-yolov8x": "x",
}
class RknnDetectorConfig(BaseDetectorConfig):
type: Literal[DETECTOR_KEY]
@@ -68,35 +57,12 @@ class Rknn(DetectionApi):
elif "rk3588" in soc:
os.rename("/usr/lib/librknnrt_rk3588.so", "/usr/lib/librknnrt.so")
self.model_path = config.model.path or "default-yolov8n"
self.core_mask = config.core_mask
self.height = config.model.height
self.width = config.model.width
if self.model_path in yolov8_suffix:
if self.model_path == "default-yolov8n":
self.model_path = "/models/rknn/yolov8n-320x320-{soc}.rknn".format(
soc=soc
)
else:
model_suffix = yolov8_suffix[self.model_path]
self.model_path = (
"/config/model_cache/rknn/yolov8{suffix}-320x320-{soc}.rknn".format(
suffix=model_suffix, soc=soc
)
)
os.makedirs("/config/model_cache/rknn", exist_ok=True)
if not os.path.isfile(self.model_path):
logger.info(
"Downloading yolov8{suffix} model.".format(suffix=model_suffix)
)
urllib.request.urlretrieve(
"https://github.com/MarcA711/rknn-models/releases/download/v1.5.2-{soc}/yolov8{suffix}-320x320-{soc}.rknn".format(
soc=soc, suffix=model_suffix
),
self.model_path,
)
if True:
os.makedirs("/config/model_cache/rknn", exist_ok=True)
if (config.model.width != 320) or (config.model.height != 320):
logger.error(
@@ -132,60 +98,12 @@ class Rknn(DetectionApi):
"Error initializing rknn runtime. Do you run docker in privileged mode?"
)
def __del__(self):
self.rknn.release()
def postprocess(self, results):
"""
Processes yolov8 output.
Args:
results: array with shape: (1, 84, n, 1) where n depends on yolov8 model size (for 320x320 model n=2100)
Returns:
detections: array with shape (20, 6) with 20 rows of (class, confidence, y_min, x_min, y_max, x_max)
"""
results = np.transpose(results[0, :, :, 0]) # array shape (2100, 84)
scores = np.max(
results[:, 4:], axis=1
) # array shape (2100,); max confidence of each row
# remove lines with score scores < 0.4
filtered_arg = np.argwhere(scores > 0.4)
results = results[filtered_arg[:, 0]]
scores = scores[filtered_arg[:, 0]]
num_detections = len(scores)
if num_detections == 0:
return np.zeros((20, 6), np.float32)
if num_detections > 20:
top_arg = np.argpartition(scores, -20)[-20:]
results = results[top_arg]
scores = scores[top_arg]
num_detections = 20
classes = np.argmax(results[:, 4:], axis=1)
boxes = np.transpose(
np.vstack(
(
(results[:, 1] - 0.5 * results[:, 3]) / self.height,
(results[:, 0] - 0.5 * results[:, 2]) / self.width,
(results[:, 1] + 0.5 * results[:, 3]) / self.height,
(results[:, 0] + 0.5 * results[:, 2]) / self.width,
)
)
raise Exception(
"RKNN does not currently support any models. Please see the docs for more info."
)
detections = np.zeros((20, 6), np.float32)
detections[:num_detections, 0] = classes
detections[:num_detections, 1] = scores
detections[:num_detections, 2:] = boxes
return detections
def __del__(self):
self.rknn.release()
@hide_warnings
def inference(self, tensor_input):

29
frigate/detectors/plugins/rocm.py
View File

@@ -1,5 +1,4 @@
import ctypes
import glob
import logging
import os
import subprocess
@@ -11,7 +10,7 @@ from typing_extensions import Literal
from frigate.detectors.detection_api import DetectionApi
from frigate.detectors.detector_config import BaseDetectorConfig
from frigate.detectors.util import preprocess, yolov8_postprocess
from frigate.detectors.util import preprocess
logger = logging.getLogger(__name__)
@@ -75,27 +74,6 @@ class ROCmDetector(DetectionApi):
logger.error("AMD/ROCm: module loading failed, missing ROCm environment?")
raise
if detector_config.conserve_cpu:
logger.info("AMD/ROCm: switching HIP to blocking mode to conserve CPU")
ctypes.CDLL("/opt/rocm/lib/libamdhip64.so").hipSetDeviceFlags(4)
assert (
detector_config.model.model_type == "yolov8"
), "AMD/ROCm: detector_config.model.model_type: only yolov8 supported"
assert (
detector_config.model.input_tensor == "nhwc"
), "AMD/ROCm: detector_config.model.input_tensor: only nhwc supported"
if detector_config.model.input_pixel_format != "rgb":
logger.warn(
"AMD/ROCm: detector_config.model.input_pixel_format: should be 'rgb' for yolov8, but '{detector_config.model.input_pixel_format}' specified!"
)
assert detector_config.model.path is not None, (
"No model.path configured, please configure model.path and model.labelmap_path; some suggestions: "
+ ", ".join(glob.glob("/config/model_cache/yolov8/*.onnx"))
+ " and "
+ ", ".join(glob.glob("/config/model_cache/yolov8/*_labels.txt"))
)
path = detector_config.model.path
mxr_path = os.path.splitext(path)[0] + ".mxr"
if path.endswith(".mxr"):
@@ -136,8 +114,11 @@ class ROCmDetector(DetectionApi):
detector_result = self.model.run({model_input_name: tensor_input})[0]
addr = ctypes.cast(detector_result.data_ptr(), ctypes.POINTER(ctypes.c_float))
# ruff: noqa: F841
tensor_output = np.ctypeslib.as_array(
addr, shape=detector_result.get_shape().lens()
)
return yolov8_postprocess(model_input_shape, tensor_output)
raise Exception(
"No models are currently supported for rocm. See the docs for more info."
)

47
frigate/detectors/util.py
View File

@@ -34,50 +34,3 @@ def preprocess(tensor_input, model_input_shape, model_input_element_type):
None,
swapRB=False,
)
def yolov8_postprocess(
model_input_shape,
tensor_output,
box_count=20,
score_threshold=0.5,
nms_threshold=0.5,
):
model_box_count = tensor_output.shape[2]
probs = tensor_output[0, 4:, :]
all_ids = np.argmax(probs, axis=0)
all_confidences = probs.T[np.arange(model_box_count), all_ids]
all_boxes = tensor_output[0, 0:4, :].T
mask = all_confidences > score_threshold
class_ids = all_ids[mask]
confidences = all_confidences[mask]
cx, cy, w, h = all_boxes[mask].T
if model_input_shape[3] == 3:
scale_y, scale_x = 1 / model_input_shape[1], 1 / model_input_shape[2]
else:
scale_y, scale_x = 1 / model_input_shape[2], 1 / model_input_shape[3]
detections = np.stack(
(
class_ids,
confidences,
scale_y * (cy - h / 2),
scale_x * (cx - w / 2),
scale_y * (cy + h / 2),
scale_x * (cx + w / 2),
),
axis=1,
)
if detections.shape[0] > box_count:
# if too many detections, do nms filtering to suppress overlapping boxes
boxes = np.stack((cx - w / 2, cy - h / 2, w, h), axis=1)
indexes = cv2.dnn.NMSBoxes(boxes, confidences, score_threshold, nms_threshold)
detections = detections[indexes]
# if still too many, trim the rest by confidence
if detections.shape[0] > box_count:
detections = detections[
np.argpartition(detections[:, 1], -box_count)[-box_count:]
]
detections = detections.copy()
detections.resize((box_count, 6))
return detections