import logging import os import sys import traceback from configparser import ConfigParser from pathlib import Path import cv2 import numpy as np from PyQt5.QtCore import (QFileSystemWatcher, QLineF, QMutex, QPointF, QRectF, Qt, QThread, pyqtSignal) from PyQt5.QtGui import (QBrush, QColor, QFont, QImage, QPainter, QPainterPath, QPen, QPixmap) from PyQt5.QtWidgets import QApplication from .component import Component from .consumer import Consumer if "--no-gpu" in sys.argv: os.environ["CUDA_VISIBLE_DEVICES"] = "-1" import tensorflow as tf from lib.helpers.object_detection.utils import label_map_util if "--no-edgetpu" not in sys.argv: try: from pycoral.utils.edgetpu import make_interpreter except (ImportError, ModuleNotFoundError): logging.exception(traceback.format_exc()) def make_interpreter(*args, **kwargs): raise ValueError("the 'pycoral' module is not available") else: def make_interpreter(*args, **kwargs): raise ValueError("\"--no-edgetpu\" in sys.argv") if "--no-tflite" not in sys.argv: try: Interpreter = tf.lite.Interpreter # from tflite_runtime.interpreter import Interpreter except (ImportError, ModuleNotFoundError): logging.exception(traceback.format_exc()) def Interpreter(*args, **kwargs): raise ValueError("the 'tflite-runtime' module is not available") else: def Interpreter(*args, **kwargs): raise ValueError("\"--no-tflite\" in sys.argv") if "--fail-vision" not in sys.argv: vision_override = None else: vision_override = False # # Patch the location of gfile # tf.gfile = tf.io.gfile class Vision(Component): """everything is expected the have shape with height (y) first then width (x)""" status_signal = pyqtSignal(dict) loading_model_signal = pyqtSignal(dict) def __init__(self, config=None, name=None, period=None, lazy=True, paused=False, threaded=True): super().__init__(config=config, name=name, period=period, lazy=lazy, paused=paused, threaded=threaded) self.lock = QMutex() self.simulate = "--sim-vision" in sys.argv def start(self): self.model = None # VISION THREAD self.vision_consumer = Consumer(work=self.vision_consumer_work, work_fifo=True, drop_fifo=True, work_maxlen=1, name="vision_consumer", paused=False) self.vision_consumer_thread = QThread() self.vision_consumer_thread.setTerminationEnabled(True) self.vision_consumer.moveToThread(self.vision_consumer_thread) self.vision_consumer_thread.started.connect(self.vision_consumer.start) self.vision_consumer_thread.start() if "--debugger-workaround" in sys.argv: QApplication.processEvents() QThread.msleep(1000) QApplication.processEvents() self.vision_consumer.wait_ready() # RENDER THREAD self.render_consumer = Consumer(work=self.render_consumer_work, work_fifo=True, drop_fifo=True, work_maxlen=1, name="render_consumer", paused=False) self.render_consumer_thread = QThread() self.render_consumer_thread.setTerminationEnabled(True) self.render_consumer.moveToThread(self.render_consumer_thread) self.render_consumer_thread.started.connect(self.render_consumer.start) self.render_consumer_thread.start() if "--debugger-workaround" in sys.argv: QApplication.processEvents() QThread.msleep(1000) QApplication.processEvents() self.render_consumer.wait_ready() # CONNECT CONSUMERS self.vision_consumer.out.connect(self.process_vision_consumed) self.render_consumer.out.connect(self.process_render_consumed) super().start() def config_changed(self): # OBJECT DETECTION self.detection_threshold = float(self.config[self.name].get("detection_threshold", 0.5)) # recipe self.zones = None self.labels = None # LOAD RECIPE self.recipes_dir = Path(self.config[self.name].get("recipes_dir", "./config/vision/recipes")) self.set_recipe(None) self.recipe_watcher = QFileSystemWatcher([]) self.recipe_watcher.fileChanged.connect(self._set_recipe) # LOAD MODEL self.models_dir = Path(self.config[self.name].get("models_dir", "./data/neural_networks")) self.allowed_modes = dict.fromkeys([ "edgetpu", "tflite", "normal", ]) if "--no-edgetpu" in sys.argv: self.allowed_modes.pop("edgetpu", None) if "--no-tflite" in sys.argv: self.allowed_modes.pop("edgetpu", None) self.allowed_modes.pop("tflite", None) self.load_model(self.config[self.name].get("neural_network", None)) # LOAD LABELS label_map = label_map_util.load_labelmap("./config/vision/labels/labels.pbtxt") self.num_classes = len(label_map.item) categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=self.num_classes, use_display_name=True) self.category_index = label_map_util.create_category_index(categories) for k in self.category_index: self.category_index[k]["color"] = self.category_index[k]["color"].replace("0x", "#") self.classes_map = {c["name"]: k for k, c in self.category_index.items()} self.zone_detection_filter_mode = self.config[self.name].get("zone_detection_filter_mode", "box_inside") self.zone_detection_preference_mode = self.config[self.name].get("zone_detection_preference_mode", "score") @staticmethod def get_center(rect): return [(rect[0] + rect[2]) / 2, (rect[1] + rect[3]) / 2] @staticmethod def get_size(rect): return [rect[2] - rect[0], rect[3] - rect[1]] @staticmethod def get_box(center, size): return [center[0] - size[0] / 2, center[1] - size[1] / 2, center[0] + size[0] / 2, center[1] + size[1] / 2] @staticmethod def get_distance(p1, p2): return pow((p1[0] - p2[0]) ** 2 + (p1[1] - p2[1]) ** 2, 1 / 2) def clear_recipe(self): self.recipe = None self.markers = {} self.zones = {} self.labels = {} def _set_recipe(self, recipe_path): recipe_path = str(recipe_path) self.log.info(f"changing recipe to {recipe_path!r}") watched = self.recipe_watcher.files() if recipe_path == "" and len(watched) == 0: # skip bad watcher signals return if len(watched) > 0: self.recipe_watcher.removePaths(watched) self.recipe_path = recipe_path try: if not os.path.isfile(self.recipe_path): raise AssertionError(f"Recipe file {self.recipe_path!r} could not be found.") config = ConfigParser(inline_comment_prefixes="#") read = config.read(self.recipe_path) if len(read) != 1 or self.recipe_path not in read: raise AssertionError("Recipe could not be read.") os.path.splitext(os.path.basename(read[0]))[0] self.markers = self.parse_markers(config._sections.get("markers", None)) self.zones = self.parse_zones(config._sections.get("zones", None)) self.labels = self.parse_labels(config._sections.get("labels", None)) self.recipe_watcher.addPath(str(self.recipe_path)) except Exception: self.log.exception(traceback.format_exc()) self.log.exception(f"Error reading {self.recipe_path!r}:") self.clear_recipe() self.status_signal.emit({ "recipe": self.recipe, "markers": self.markers, "zones": self.zones, "labels": self.labels, }) def set_recipe(self, recipe=None): if recipe is None: self.clear_recipe() else: self.recipe = recipe self._set_recipe(self.recipes_dir / str(recipe)) def parse_markers(self, config=None): if config is None: raise AssertionError(f"Recipe file {self.recipe_path!r} does not contain the 'markers' section.") config = {} markers = {} for marker_name, marker_spec in config.items(): try: center, size, fill_color, border_color, border_thickness, shape = marker_spec.split(" ") center = list(reversed(list(map(float, center.split(","))))) size = list(reversed(list(map(float, size.split(","))))) if len(size) == 1: size = [size[0], size[0]] fill_color = QColor(fill_color.replace("0x", "#")) border_color = QColor(border_color.replace("0x", "#")) border_thickness = float(border_thickness) shape = shape.lower() markers[marker_name] = { "border_color": border_color, "border_thickness": border_thickness, "center": center, "fill_color": fill_color, "shape": shape, "size": size, } except Exception: self.log.exception(traceback.format_exc()) self.log.exception(f"marker {marker_name!r} in recipe file {self.recipe_path!r} could not be parsed. spec: {marker_spec!r}") return markers def parse_zones(self, config=None): if config is None: raise AssertionError(f"Recipe file {self.recipe_path!r} does not contain the 'zones' section.") config = {} zones = {} for zone_name, zone_spec in config.items(): zone_name = zone_name.upper() try: center, size, d_class = zone_spec.split(" ") center = list(reversed(list(map(float, center.split(","))))) if "," in size: size = list(reversed(list(map(float, size.split(","))))) shape = "rect" else: size = [float(size)] * 2 shape = "ellipse" d_class = self.category_index[self.classes_map[d_class]] zones[zone_name] = { "border_color": QColor(d_class["color"]), "border_thickness": 25, "box": self.get_box(center, size), "convert_negative_placement": False, "center": center, "class": d_class, "fill_color": QColor("#00000000"), "pen_line": "DashLine", "shape": shape, "size": size, } except Exception: self.log.exception(traceback.format_exc()) self.log.exception(f"region {zone_name!r} in recipe file {self.recipe_path!r} could not be parsed. spec: {zone_spec!r}") return zones def parse_labels(self, config=None): if config is None: config = {} labels = {} for label_name, label_spec in config.items(): try: location, font_size, fill_color, border_color, border_thickness, text = label_spec.split(" ", 5) location = list(reversed(list(map(float, location.split(","))))) font_size = float(font_size) fill_color = QColor(fill_color.replace("0x", "#")) border_color = QColor(border_color.replace("0x", "#")) border_thickness = float(border_thickness) text = text.replace("\\n", "\n").replace("\\t", "\t") labels[label_name] = { "border_color": border_color, "border_thickness": border_thickness, "fill_color": fill_color, "font_size": font_size, "location": location, "opacity": 1, "shape": "text", "text": text, } except Exception: self.log.exception(traceback.format_exc()) self.log.exception(f"label {label_name!r} in recipe file {self.recipe_path!r} could not be parsed. spec: {label_spec!r}") return labels def load_model(self, model=None): # print("VISION CONSUMER", str(int(QThread.currentThreadId())), flush=True) self.log.info(f"requested neural network: {model!r}") if model is None or model.lower() in [ "", "any", "last", "latest", "newest", "none", ]: model_name = sorted([d for d in os.listdir(self.models_dir) if os.path.isdir(self.models_dir / d)], reverse=True)[0] else: model_name = model self.log.info(f"loading neural network: {model_name!r}") self.loading_model_signal.emit({"status": "loading"}) self.lock.lock() tf_mode = None # reset tflite variables interpreter = None if self.simulate: tf_mode = "simulation" interpreter = None if tf_mode is None and "edgetpu" in self.allowed_modes: try: # create tflite edgetpu interpreter interpreter = make_interpreter(str(self.models_dir / model_name / f"{model_name}_edgetpu.tflite")) tf_mode = "edgetpu" except Exception: self.log.exception(traceback.format_exc()) if tf_mode is None and "tflite" in self.allowed_modes: try: # create tflite cpu interpreter interpreter = Interpreter(str(self.models_dir / model_name / f"{model_name}.tflite")) tf_mode = "tflite" except Exception: self.log.exception(traceback.format_exc()) # reset tensorflow variables model = None if tf_mode is None and "normal" in self.allowed_modes: try: # create tensorflow model model = tf.saved_model.load(str(self.models_dir / model_name)).signatures["serving_default"] tf_mode = "normal" except Exception: self.log.exception(traceback.format_exc()) self.lock.unlock() if tf_mode is None: raise RuntimeError("failed initialize any neural network model") self.tf_mode = tf_mode self.model_name = model_name if interpreter is not None: # if there is a new tflite interpreter initialize it interpreter.allocate_tensors() interpreter.invoke() # warmup self.interpreter = interpreter # if there is a new model to be used, remove previous model if present if model is not None and self.model is not None: tf.keras.backend.clear_session() self.model = model self.log.info(f"initialized model {self.model!r} with mode {self.tf_mode!r}") self.loading_model_signal.emit({"status": "done"}) def check_features(self, frame, lock=True): if self.interpreter is not None and frame.shape != self.interpreter.get_input_details()[0]["shape"][1:3]: tensor = np.expand_dims(cv2.resize(frame, self.interpreter.get_input_details()[0]["shape"][1:3], interpolation=cv2.INTER_LINEAR), axis=0) else: frame_resized = cv2.resize(frame, (256, 256), interpolation=cv2.INTER_LINEAR) tensor = tf.convert_to_tensor(np.asarray(frame_resized)) tensor = tensor[tf.newaxis, ...] # tensor = np.expand_dims(frame, axis=0) # Run inference if lock: self.lock.lock() if self.simulate or self.tf_mode == "simulation": detections = { "detection_scores": [[1.0]], "detection_boxes": [[[0.2, 0.2, 0.8, 0.8]]], "detection_classes": [[1]], } if lock: self.lock.unlock() elif self.tf_mode in {"edgetpu", "tflite"}: i_d = self.interpreter.get_input_details() # print(i_d) o_d = self.interpreter.get_output_details() # print(o_d) self.interpreter.set_tensor(i_d[0]["index"], tensor) self.interpreter.invoke() # PARSE TFLITE DETECTIONS # signature_list = self.interpreter._get_full_signature_list() # if signature_list: # if len(signature_list) > 1: # raise ValueError("Only support model with one signature.") # signature = signature_list[next(iter(signature_list))] # # count = int(self.interpreter.get_tensor(signature["outputs"]["output_0"])[0]) # scores = self.interpreter.get_tensor(signature["outputs"]["output_1"])[0] # class_ids = self.interpreter.get_tensor(signature["outputs"]["output_2"])[0] # boxes = self.interpreter.get_tensor(signature["outputs"]["output_3"])[0] if self.interpreter.get_tensor(o_d[3]["index"]).size == 1: boxes = self.interpreter.get_tensor(o_d[0]["index"])[0] class_ids = self.interpreter.get_tensor(o_d[1]["index"])[0] scores = self.interpreter.get_tensor(o_d[2]["index"])[0] # count = int(self.interpreter.get_tensor(o_d[3]["index"])[0]) else: scores = self.interpreter.get_tensor(o_d[0]["index"])[0] boxes = self.interpreter.get_tensor(o_d[1]["index"])[0] # count = int(self.interpreter.get_tensor(o_d[2]["index"])[0]) class_ids = self.interpreter.get_tensor(o_d[3]["index"])[0] if lock: self.lock.unlock() detections = { "detection_scores": [scores], "detection_boxes": [boxes], "detection_classes": [map(lambda class_id: class_id + 1, class_ids)], } else: detections = self.model(tensor) if lock: self.lock.unlock() detections = { "detection_scores": detections["detection_scores"].numpy().tolist(), "detection_boxes": detections["detection_boxes"].numpy().tolist(), "detection_classes": detections["detection_classes"], } # WARNING: results other than the ones related to tensor[-1] will be discarded parsed_detections = [] for d_score, d_box, d_class in zip( # , d_mask in zip( detections["detection_scores"][-1], detections["detection_boxes"][-1], detections["detection_classes"][-1], # detections["detection_masks"][-1], ): if d_score < self.detection_threshold: continue box = list(d_box) box = [i * s for i, s in zip(box, frame.shape[:2] * 2)] # rescale detection to frame size detection = { "score": d_score, "box": box, "class": self.category_index[int(d_class)], # "mask": d_mask, "center": self.get_center(box), "size": self.get_size(box), } parsed_detections.append(detection) return parsed_detections def detections_to_items(self, detections): # DRAW DETECTIONS if detections is not None and len(detections): style = { "border_thickness": 25, "fill_color": QColor("#00000000"), "shape": "rect", "convert_negative_placement": False, } items = {} for item_name, item in enumerate(detections): items[str(item_name)] = { **item, **style, "border_color": QColor(item["class"]["color"]), } return items else: return {} def process_detections(self, detections): if self.zones is None or not len(self.zones): return None # MATCH DETECTIONS WITH RECIPE results = dict.fromkeys(self.zones) for detection in detections: # find closest zone center to the detection # filtering out those that do not contain the detection min_distance = sys.maxsize closest_zone = None for zone_name, zone in self.zones.items(): distance = self.get_distance(detection["center"], zone["center"]) if zone["shape"] == "rect": if self.zone_detection_filter_mode == "center_inside": outside_zone = any([ detection["center"][0] < zone["box"][0], detection["center"][0] > zone["box"][2], detection["center"][1] < zone["box"][1], detection["center"][1] > zone["box"][3], ]) elif self.zone_detection_filter_mode == "box_inside": outside_zone = any([ detection["box"][0] < zone["box"][0], detection["box"][2] > zone["box"][2], detection["box"][1] < zone["box"][1], detection["box"][3] > zone["box"][3], ]) elif self.zone_detection_filter_mode == "box_touches": outside_zone = any([ detection["box"][2] < zone["box"][0], detection["box"][0] > zone["box"][2], detection["box"][3] < zone["box"][1], detection["box"][1] > zone["box"][3], ]) else: raise NotImplementedError(f"invalid zone_detection_filter_mode: {self.zone_detection_filter_mode!r}") elif zone["shape"] == "ellipse": # it's a circle if self.zone_detection_filter_mode == "center_inside": outside_zone = distance > zone["size"][0] / 2 elif self.zone_detection_filter_mode == "box_inside": outside_zone = distance + max(detection["size"]) / 2 > zone["size"][0] / 2 elif self.zone_detection_filter_mode == "box_touches": outside_zone = distance - max(detection["size"]) / 2 > zone["size"][0] / 2 else: raise NotImplementedError(f"invalid zone_detection_filter_mode: {self.zone_detection_filter_mode!r}") else: raise NotImplementedError(f"invalid zone shape: {zone['shape']!r}") if not outside_zone and distance < min_distance: min_distance = distance closest_zone = zone_name if closest_zone is not None: # if closest zone already has a matching detection # replace it only if the current one is preferred if results[closest_zone] is not None: if self.zone_detection_preference_mode == "distance": if min_distance >= results[closest_zone]["zone_distance"]: continue elif self.zone_detection_preference_mode == "score": if detection["score"] <= results[closest_zone]["score"]: continue else: raise NotImplementedError(f"invalid zone_detection_preference_mode: {self.zone_detection_preference_mode!r}") results[closest_zone] = detection.copy() results[closest_zone]["zone_distance"] = min_distance # check detections against recipe checked = {} for zone_name, detection in results.items(): if detection is None: # dummy empty detection if nothing detected detection = { "box": [0, 0, 0, 0], "center": [0, 0], "class": { "id": None, "name": "no_detection", "color": "rgb(0,0,0)", }, # "mask": [], "score": 1, "size": [0, 0], } if zone_name not in self.zones or self.zones[zone_name]["class"]["id"] in {"no_detection", "none", }: expected_class = { "id": None, "name": "no_detection", "color": "rgb(0,0,0)", } else: expected_class = self.zones[zone_name]["class"] checked[zone_name] = { "ok": detection is not None and detection["class"]["id"] == expected_class["id"], "expected": expected_class, "detection": detection, } global vision_override if vision_override is None: ok = all(map(lambda detection: detection["ok"] is True, checked.values())) else: ok = vision_override return { "ok": ok, "results": checked, } def results_to_items(self, results, ): # DRAW ZONES RESULTS if self.zones is not None and len(self.zones) and results is not None and len(results): style = { "pen_line": "SolidLine", "border_thickness": 50, "fill_color": QColor("#00000000"), } items = { "_global_result": { "box": [ style["border_thickness"], style["border_thickness"], -style["border_thickness"] * 2, -style["border_thickness"] * 2, ], "shape": "rect", **style, "border_color": Qt.green if results["ok"] else Qt.red, } } for item_name, item in results["results"].items(): zone = self.zones[item_name] items[str(item_name)] = { **zone, **style, "border_color": Qt.green if item["ok"] else Qt.red, } return items else: return {} @staticmethod def convert_negative_placement(p, painter): for k in p: if p[k] < 0: if k.startswith("x") or k.startswith("w"): p[k] += painter.device().width() elif k.startswith("y") or k.startswith("h"): p[k] += painter.device().height() else: raise AssertionError("could not detect variable direction") @staticmethod def apply_placement_offset(p, offset): for k in p: if k.startswith("x"): p[k] += offset["x"] elif k.startswith("y"): p[k] += offset["y"] else: raise AssertionError("could not detect variable direction") def render_items(self, items, offset=None, qimage=None, painter=None): if offset is None: offset = {"x": 0, "y": 0} if painter is None: if qimage is None: raise AssertionError("one of 'qimage' or 'painter' parameter must not be None") painter = QPainter() painter.begin(qimage) for item_name, item in items.items(): try: convert_negative_placement = item.get("convert_negative_placement", True) if "box" in item: p = {"x1": item["box"][1], "y1": item["box"][0], "x2": item["box"][3], "y2": item["box"][2], } if convert_negative_placement: Vision.convert_negative_placement(p, painter) Vision.apply_placement_offset(p, offset) p["w"], p["h"] = p["x2"] - p["x1"], p["y2"] - p["y1"] p["xc"], p["yc"] = Vision.get_center([p["x1"], p["y1"], p["x2"], p["y2"]]) elif "location" in item: p = {"x1": item["location"][1], "y1": item["location"][0], } if convert_negative_placement: Vision.convert_negative_placement(p, painter) Vision.apply_placement_offset(p, offset) if "size" in item: p["w"], p["h"] = item["size"][1], item["size"][0] p["x2"], p["y2"] = p["x1"] + p["w"], p["y1"] + p["h"] p["xc"], p["yc"] = Vision.get_center([p["x1"], p["y1"], p["x2"], p["y2"]]) else: p["w"], p["h"] = 0, 0 p["x2"], p["y2"] = p["x1"], p["y1"] p["xc"], p["yc"] = p["x1"], p["y1"] elif "center" in item and "size" in item: p = {"xc": item["center"][1], "yc": item["center"][0], } if convert_negative_placement: Vision.convert_negative_placement(p, painter) Vision.apply_placement_offset(p, offset) p["w"], p["h"] = item["size"][1], item["size"][0] p["x1"], p["y1"], p["x2"], p["y2"] = Vision.get_box([p["xc"], p["yc"]], [p["w"], p["h"]]) else: raise AssertionError("item has no valid positioning information") painter.setOpacity(item.get("opacity", 0.5)) painter.setBrush(QBrush(item.get("fill_color", QColor("#ffffff")), getattr(Qt, item.get("brush_pattern", "SolidPattern")))) painter.setPen(QPen( item.get("border_color", QColor("#000000")), item.get("border_thickness", 1), getattr(Qt, item.get("pen_line", "SolidLine")), getattr(Qt, item.get("pen_cap", "SquareCap")), getattr(Qt, item.get("pen_join", "MiterJoin")), )) if item["shape"] == "ellipse": painter.drawEllipse(QPointF(p["xc"], p["yc"]), p["w"] / 2, p["h"] / 2) elif item["shape"] == "cross": painter.drawLine(QLineF(p["xc"], p["y1"], p["xc"], p["y2"])) painter.drawLine(QLineF(p["x1"], p["yc"], p["x2"], p["yc"])) elif item["shape"] == "line": painter.drawLine(QLineF(p["xc"], p["yc"], p["x2"], p["y2"])) elif item["shape"] == "rect": painter.drawRect(QRectF(QPointF(p["x1"], p["y1"]), QPointF(p["x2"], p["y2"]))) elif item["shape"] == "text": old_render_hints = painter.renderHints() painter.setRenderHints(QPainter.Antialiasing | QPainter.TextAntialiasing, True) font = QFont() font.setStyleHint(QFont.SansSerif, QFont.PreferDefault | QFont.PreferAntialias) font.setBold(item.get("font_bold", False)) font.setItalic(item.get("font_italic", False)) font.setKerning(item.get("font_kerning", True)) font.setLetterSpacing(QFont.AbsoluteSpacing, item.get("font_letter_spacing", 0)) font.setWordSpacing(item.get("font_word_spacing", 0)) font.setPixelSize(round(item.get("font_size", 25))) path = QPainterPath() path.addText(p["x1"], p["y1"], font, item["text"]) painter.drawPath(path) painter.setRenderHints(old_render_hints, True) else: raise NotImplementedError(f"item {item_name!r} has an invalid shape: {item['shape']!r}") except Exception: self.log.exception("".join(traceback.format_stack())) self.log.exception(traceback.format_exc()) self.log.error(f"item {item_name!r} could not be drawn.") def render(self, frame, detections=None, vision_results=None, mask=True, offset=None): if mask is True: qframe = QImage( frame.shape[1], # width frame.shape[0], # height QImage.Format_RGBA8888 ) else: aframe = cv2.cvtColor(frame, cv2.COLOR_RGB2RGBA) qframe = QImage( aframe.data, aframe.shape[1], # width aframe.shape[0], # height aframe.shape[2] * frame.shape[1], # width * channels QImage.Format_RGBA8888 ) painter = QPainter() painter.begin(qframe) for items in [ self.markers, self.zones, self.labels, self.detections_to_items(detections), self.results_to_items(vision_results) ]: if items is not None: try: self.render_items( items, offset=offset, painter=painter, ) except Exception: self.log.exception(traceback.format_exc()) painter.end() return qframe def _get(self, data=None): # print("VISION", str(int(QThread.currentThreadId())), flush=True) if data is None: return frame = data[-1][list(self.sources)[0]] if frame is not None: # ADD FRAMETO VISION_CONSUMER QUEUE self.vision_consumer.add_consumable({"frame": frame}) super()._get(emit=False) def vision_consumer_work(self, consumable=None): # VISION_CONSUMER TASK if consumable is None: return detections = self.check_features(consumable["frame"]) results = self.process_detections(detections) return {"detections": detections, "results": results} def process_vision_consumed(self, data=None): # print("VISION", str(int(QThread.currentThreadId())), flush=True) if data is None: return # ADD VISION RETURNED FROM VISION_CONSUMER TO RENDER_CONSUMER QUEUE data = data[-1][self.vision_consumer.name] if data is not None: # super()._get([{ # "frame": data["consumed"]["frame"], # "detections": data["result"]["detections"], # "results": data["result"]["results"], # }]) self.render_consumer.add_consumable({ "frame": data["consumed"]["frame"], "detections": data["result"]["detections"], "results": data["result"]["results"], }) def render_consumer_work(self, consumable=None): # RENDER_CONSUMER TASK if consumable is None: return render = QPixmap.fromImage(self.render( consumable["frame"], detections=consumable["detections"], vision_results=consumable["results"], mask=False, offset=None, )) return render def process_render_consumed(self, data=None): # print("VISION", str(int(QThread.currentThreadId())), flush=True) if data is None: return # EMIT VISION AND RENDER RESULTS RETURNED FROM RENDER_CONSUMER data = data[-1][self.render_consumer.name] if data is not None: super()._get([{ "frame": data["consumed"]["frame"], "detections": data["consumed"]["detections"], "results": data["consumed"]["results"], "render": data["result"], }])