From b339f53bb45081e6cca68bdfaf20453e92f81027 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Tue, 23 May 2023 01:06:14 +0200 Subject: [PATCH 1/7] Add isort and ruff linter Both linters are pretty common among modern python code bases. The isort tool provides stable sorting and grouping, as well as pruning of unused imports. Ruff is a modern linter, that is very fast due to being written in rust. It can detect many common issues in a python codebase. Removes the pylint dev requirement, since ruff replaces it. --- .github/workflows/pull_request.yml | 14 ++++++++++---- pyproject.toml | 5 +++++ requirements-dev.txt | 3 ++- 3 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 1c1d2cd76f..7e38c3b99f 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -70,11 +70,17 @@ jobs: python-version: ${{ env.DEFAULT_PYTHON }} - name: Install requirements run: | - pip install pip - pip install -r requirements-dev.txt - - name: Lint + python3 -m pip install -U pip + python3 -m pip install -r requirements-dev.txt + - name: Check black + run: | + black --check --diff frigate migrations docker *.py + - name: Check isort + run: | + isort --check --diff frigate migrations docker *.py + - name: Check ruff run: | - python3 -m black frigate --check + ruff check frigate migrations docker *.py python_tests: runs-on: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..e3ef3faf5a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,5 @@ +[tool.isort] +profile = "black" + +[tool.ruff] +ignore = ["E501"] \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 29f004455c..0acd15ae4c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ -pylint == 2.17.* black == 23.3.* +isort +ruff From 2807a229a2708554156470bbbaf0f05c84c6aa42 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Tue, 23 May 2023 02:38:21 +0200 Subject: [PATCH 2/7] treewide: fix issues detected by ruff --- benchmark.py | 2 +- frigate/__main__.py | 8 +++-- frigate/app.py | 8 ++--- frigate/comms/dispatcher.py | 8 ++--- frigate/comms/mqtt.py | 10 +++--- frigate/comms/ws.py | 4 +-- frigate/config.py | 13 +++---- frigate/detectors/__init__.py | 6 ---- frigate/detectors/detector_config.py | 4 +-- frigate/detectors/plugins/cpu_tfl.py | 3 +- frigate/detectors/plugins/deepstack.py | 6 ++-- frigate/detectors/plugins/edgetpu_tfl.py | 2 +- frigate/detectors/plugins/openvino.py | 2 +- frigate/detectors/plugins/tensorrt.py | 4 +-- frigate/events/cleanup.py | 35 ++++++++++--------- frigate/events/external.py | 1 - frigate/events/maintainer.py | 7 ++-- frigate/http.py | 43 ++++++++++++------------ frigate/log.py | 5 ++- frigate/models.py | 1 - frigate/object_detection.py | 2 +- frigate/object_processing.py | 10 +++--- frigate/objects.py | 9 +---- frigate/output.py | 13 ++++--- frigate/plus.py | 2 +- frigate/ptz.py | 2 +- frigate/record/cleanup.py | 4 +-- frigate/record/maintainer.py | 6 ++-- frigate/stats.py | 2 +- frigate/storage.py | 4 +-- frigate/test/test_config.py | 6 ++-- frigate/test/test_gpu_stats.py | 2 +- frigate/test/test_http.py | 12 +++---- frigate/test/test_reduce_boxes.py | 1 - frigate/test/test_storage.py | 5 +-- frigate/test/test_yuv_region_2_rgb.py | 2 +- frigate/util.py | 3 +- frigate/video.py | 14 ++++---- frigate/watchdog.py | 4 +-- 39 files changed, 123 insertions(+), 152 deletions(-) diff --git a/benchmark.py b/benchmark.py index 3d0cacd87a..ea388ef13a 100755 --- a/benchmark.py +++ b/benchmark.py @@ -53,7 +53,7 @@ def start(id, num_detections, detection_queue, event): frame_times = [] for x in range(0, num_detections): start_frame = datetime.datetime.now().timestamp() - detections = object_detector.detect(my_frame) + object_detector.detect(my_frame) frame_times.append(datetime.datetime.now().timestamp() - start_frame) duration = datetime.datetime.now().timestamp() - start diff --git a/frigate/__main__.py b/frigate/__main__.py index b36881151b..8fa98fe530 100644 --- a/frigate/__main__.py +++ b/frigate/__main__.py @@ -1,13 +1,15 @@ import faulthandler +import threading + from flask import cli +from frigate.app import FrigateApp + + faulthandler.enable() -import threading threading.current_thread().name = "frigate" -from frigate.app import FrigateApp - cli.show_server_banner = lambda *x: None if __name__ == "__main__": diff --git a/frigate/app.py b/frigate/app.py index 5e934cc4b0..c66fab187e 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -133,10 +133,10 @@ def set_log_levels(self) -> None: for log, level in self.config.logger.logs.items(): logging.getLogger(log).setLevel(level.value.upper()) - if not "werkzeug" in self.config.logger.logs: + if "werkzeug" not in self.config.logger.logs: logging.getLogger("werkzeug").setLevel("ERROR") - if not "ws4py" in self.config.logger.logs: + if "ws4py" not in self.config.logger.logs: logging.getLogger("ws4py").setLevel("ERROR") def init_queues(self) -> None: @@ -294,7 +294,7 @@ def start_detected_frames_processor(self) -> None: def start_video_output_processor(self) -> None: output_processor = mp.Process( target=output_frames, - name=f"output_processor", + name="output_processor", args=( self.config, self.video_output_queue, @@ -467,7 +467,7 @@ def receiveSignal(signalNumber: int, frame: Optional[FrameType]) -> None: self.stop() def stop(self) -> None: - logger.info(f"Stopping...") + logger.info("Stopping...") self.stop_event.set() for detector in self.detectors.values(): diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py index d7fd2b4d42..61e1c75221 100644 --- a/frigate/comms/dispatcher.py +++ b/frigate/comms/dispatcher.py @@ -72,7 +72,7 @@ def _receive(self, topic: str, payload: str) -> None: camera_name = topic.split("/")[-3] command = topic.split("/")[-2] self._camera_settings_handlers[command](camera_name, payload) - except IndexError as e: + except IndexError: logger.error(f"Received invalid set command: {topic}") return elif topic.endswith("ptz"): @@ -80,7 +80,7 @@ def _receive(self, topic: str, payload: str) -> None: # example /cam_name/ptz payload=MOVE_UP|MOVE_DOWN|STOP... camera_name = topic.split("/")[-2] self._on_ptz_command(camera_name, payload) - except IndexError as e: + except IndexError: logger.error(f"Received invalid ptz command: {topic}") return elif topic == "restart": @@ -128,7 +128,7 @@ def _on_motion_command(self, camera_name: str, payload: str) -> None: elif payload == "OFF": if self.camera_metrics[camera_name]["detection_enabled"].value: logger.error( - f"Turning off motion is not allowed when detection is enabled." + "Turning off motion is not allowed when detection is enabled." ) return @@ -196,7 +196,7 @@ def _on_recordings_command(self, camera_name: str, payload: str) -> None: if payload == "ON": if not self.config.cameras[camera_name].record.enabled_in_config: logger.error( - f"Recordings must be enabled in the config to be turned on via MQTT." + "Recordings must be enabled in the config to be turned on via MQTT." ) return diff --git a/frigate/comms/mqtt.py b/frigate/comms/mqtt.py index 37fdd44aec..0e8fb4d711 100644 --- a/frigate/comms/mqtt.py +++ b/frigate/comms/mqtt.py @@ -177,10 +177,10 @@ def _start(self) -> None: f"{self.mqtt_config.topic_prefix}/restart", self.on_mqtt_command ) - if not self.mqtt_config.tls_ca_certs is None: + if self.mqtt_config.tls_ca_certs is not None: if ( - not self.mqtt_config.tls_client_cert is None - and not self.mqtt_config.tls_client_key is None + self.mqtt_config.tls_client_cert is not None + and self.mqtt_config.tls_client_key is not None ): self.client.tls_set( self.mqtt_config.tls_ca_certs, @@ -189,9 +189,9 @@ def _start(self) -> None: ) else: self.client.tls_set(self.mqtt_config.tls_ca_certs) - if not self.mqtt_config.tls_insecure is None: + if self.mqtt_config.tls_insecure is not None: self.client.tls_insecure_set(self.mqtt_config.tls_insecure) - if not self.mqtt_config.user is None: + if self.mqtt_config.user is not None: self.client.username_pw_set( self.mqtt_config.user, password=self.mqtt_config.password ) diff --git a/frigate/comms/ws.py b/frigate/comms/ws.py index c9cc3988a5..d036da8e70 100644 --- a/frigate/comms/ws.py +++ b/frigate/comms/ws.py @@ -45,7 +45,7 @@ def received_message(self, message: WebSocket.received_message) -> None: "topic": json_message.get("topic"), "payload": json_message.get("payload"), } - except Exception as e: + except Exception: logger.warning( f"Unable to parse websocket message as valid json: {message.data.decode('utf-8')}" ) @@ -82,7 +82,7 @@ def publish(self, topic: str, payload: str, _: bool) -> None: "payload": payload, } ) - except Exception as e: + except Exception: # if the payload can't be decoded don't relay to clients logger.debug(f"payload for {topic} wasn't text. Skipping...") return diff --git a/frigate/config.py b/frigate/config.py index 56e4dc4965..f9203582b5 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -8,7 +8,6 @@ import matplotlib.pyplot as plt import numpy as np -import yaml from pydantic import BaseModel, Extra, Field, validator, parse_obj_as from pydantic.fields import PrivateAttr @@ -26,7 +25,6 @@ get_ffmpeg_arg_list, escape_special_characters, load_config_with_no_duplicates, - load_labels, ) from frigate.ffmpeg_presets import ( parse_preset_hardware_acceleration_decode, @@ -36,12 +34,9 @@ parse_preset_output_rtmp, ) from frigate.detectors import ( - PixelFormatEnum, - InputTensorEnum, ModelConfig, DetectorConfig, ) -from frigate.version import VERSION logger = logging.getLogger(__name__) @@ -487,7 +482,7 @@ def validate_roles(cls, v): if len(roles) > len(roles_set): raise ValueError("Each input role may only be used once.") - if not "detect" in roles: + if "detect" not in roles: raise ValueError("The detect role is required.") return v @@ -776,12 +771,12 @@ def verify_config_roles(camera_config: CameraConfig) -> None: set([r for i in camera_config.ffmpeg.inputs for r in i.roles]) ) - if camera_config.record.enabled and not "record" in assigned_roles: + if camera_config.record.enabled and "record" not in assigned_roles: raise ValueError( f"Camera {camera_config.name} has record enabled, but record is not assigned to an input." ) - if camera_config.rtmp.enabled and not "rtmp" in assigned_roles: + if camera_config.rtmp.enabled and "rtmp" not in assigned_roles: raise ValueError( f"Camera {camera_config.name} has rtmp enabled, but rtmp is not assigned to an input." ) @@ -1062,7 +1057,7 @@ def runtime_config(self, plus_api: PlusApi = None) -> FrigateConfig: config.model.dict(exclude_unset=True), ) - if not "path" in merged_model: + if "path" not in merged_model: if detector_config.type == "cpu": merged_model["path"] = "/cpu_model.tflite" elif detector_config.type == "edgetpu": diff --git a/frigate/detectors/__init__.py b/frigate/detectors/__init__.py index 7cbd82f084..c04891f50a 100644 --- a/frigate/detectors/__init__.py +++ b/frigate/detectors/__init__.py @@ -1,11 +1,5 @@ import logging -from .detection_api import DetectionApi -from .detector_config import ( - PixelFormatEnum, - InputTensorEnum, - ModelConfig, -) from .detector_types import DetectorTypeEnum, api_types, DetectorConfig diff --git a/frigate/detectors/detector_config.py b/frigate/detectors/detector_config.py index e949065b4c..ce11adb2f7 100644 --- a/frigate/detectors/detector_config.py +++ b/frigate/detectors/detector_config.py @@ -3,12 +3,12 @@ import logging from enum import Enum import os -from typing import Dict, List, Optional, Tuple, Union, Literal +from typing import Dict, Optional, Tuple import requests import matplotlib.pyplot as plt -from pydantic import BaseModel, Extra, Field, validator +from pydantic import BaseModel, Extra, Field from pydantic.fields import PrivateAttr from frigate.plus import PlusApi diff --git a/frigate/detectors/plugins/cpu_tfl.py b/frigate/detectors/plugins/cpu_tfl.py index a831a91907..6439a9c3db 100644 --- a/frigate/detectors/plugins/cpu_tfl.py +++ b/frigate/detectors/plugins/cpu_tfl.py @@ -3,8 +3,9 @@ from frigate.detectors.detection_api import DetectionApi from frigate.detectors.detector_config import BaseDetectorConfig + from typing_extensions import Literal -from pydantic import Extra, Field +from pydantic import Field try: from tflite_runtime.interpreter import Interpreter diff --git a/frigate/detectors/plugins/deepstack.py b/frigate/detectors/plugins/deepstack.py index 639cd396c1..6ef702cb1b 100644 --- a/frigate/detectors/plugins/deepstack.py +++ b/frigate/detectors/plugins/deepstack.py @@ -6,7 +6,7 @@ from frigate.detectors.detection_api import DetectionApi from frigate.detectors.detector_config import BaseDetectorConfig from typing_extensions import Literal -from pydantic import Extra, Field +from pydantic import Field from PIL import Image @@ -64,11 +64,11 @@ def detect_raw(self, tensor_input): for i, detection in enumerate(response_json.get("predictions")): logger.debug(f"Response: {detection}") if detection["confidence"] < 0.4: - logger.debug(f"Break due to confidence < 0.4") + logger.debug("Break due to confidence < 0.4") break label = self.get_label_index(detection["label"]) if label < 0: - logger.debug(f"Break due to unknown label") + logger.debug("Break due to unknown label") break detections[i] = [ label, diff --git a/frigate/detectors/plugins/edgetpu_tfl.py b/frigate/detectors/plugins/edgetpu_tfl.py index d671dea4d8..7e35877c4e 100644 --- a/frigate/detectors/plugins/edgetpu_tfl.py +++ b/frigate/detectors/plugins/edgetpu_tfl.py @@ -4,7 +4,7 @@ from frigate.detectors.detection_api import DetectionApi from frigate.detectors.detector_config import BaseDetectorConfig from typing_extensions import Literal -from pydantic import Extra, Field +from pydantic import Field try: from tflite_runtime.interpreter import Interpreter, load_delegate diff --git a/frigate/detectors/plugins/openvino.py b/frigate/detectors/plugins/openvino.py index fe87ac567c..3b65e5c293 100644 --- a/frigate/detectors/plugins/openvino.py +++ b/frigate/detectors/plugins/openvino.py @@ -5,7 +5,7 @@ from frigate.detectors.detection_api import DetectionApi from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum from typing_extensions import Literal -from pydantic import Extra, Field +from pydantic import Field logger = logging.getLogger(__name__) diff --git a/frigate/detectors/plugins/tensorrt.py b/frigate/detectors/plugins/tensorrt.py index ac0c4befcd..26fa92ccb7 100644 --- a/frigate/detectors/plugins/tensorrt.py +++ b/frigate/detectors/plugins/tensorrt.py @@ -8,7 +8,7 @@ from cuda import cuda TRT_SUPPORT = True -except ModuleNotFoundError as e: +except ModuleNotFoundError: TRT_SUPPORT = False from frigate.detectors.detection_api import DetectionApi @@ -172,7 +172,7 @@ def _do_inference(self): if not self.context.execute_async_v2( bindings=self.bindings, stream_handle=self.stream ): - logger.warn(f"Execute returned false") + logger.warn("Execute returned false") # Transfer predictions back from the GPU. [ diff --git a/frigate/events/cleanup.py b/frigate/events/cleanup.py index 09b1964fd7..a78794a830 100644 --- a/frigate/events/cleanup.py +++ b/frigate/events/cleanup.py @@ -7,7 +7,6 @@ from pathlib import Path -from peewee import fn from frigate.config import FrigateConfig from frigate.const import CLIPS_DIR @@ -45,9 +44,9 @@ def expire(self, media_type: str) -> None: ) # loop over object types in db - for l in distinct_labels: + for event in distinct_labels: # get expiration time for this label - expire_days = retain_config.objects.get(l.label, retain_config.default) + expire_days = retain_config.objects.get(event.label, retain_config.default) expire_after = ( datetime.datetime.now() - datetime.timedelta(days=expire_days) ).timestamp() @@ -55,8 +54,8 @@ def expire(self, media_type: str) -> None: expired_events = Event.select().where( Event.camera.not_in(self.camera_keys), Event.start_time < expire_after, - Event.label == l.label, - Event.retain_indefinitely == False, + Event.label == event.label, + Event.retain_indefinitely is False, ) # delete the media from disk for event in expired_events: @@ -75,8 +74,8 @@ def expire(self, media_type: str) -> None: update_query = Event.update(update_params).where( Event.camera.not_in(self.camera_keys), Event.start_time < expire_after, - Event.label == l.label, - Event.retain_indefinitely == False, + Event.label == event.label, + Event.retain_indefinitely is False, ) update_query.execute() @@ -92,9 +91,9 @@ def expire(self, media_type: str) -> None: ) # loop over object types in db - for l in distinct_labels: + for event in distinct_labels: # get expiration time for this label - expire_days = retain_config.objects.get(l.label, retain_config.default) + expire_days = retain_config.objects.get(event.label, retain_config.default) expire_after = ( datetime.datetime.now() - datetime.timedelta(days=expire_days) ).timestamp() @@ -102,8 +101,8 @@ def expire(self, media_type: str) -> None: expired_events = Event.select().where( Event.camera == name, Event.start_time < expire_after, - Event.label == l.label, - Event.retain_indefinitely == False, + Event.label == event.label, + Event.retain_indefinitely is False, ) # delete the grabbed clips from disk for event in expired_events: @@ -121,8 +120,8 @@ def expire(self, media_type: str) -> None: update_query = Event.update(update_params).where( Event.camera == name, Event.start_time < expire_after, - Event.label == l.label, - Event.retain_indefinitely == False, + Event.label == event.label, + Event.retain_indefinitely is False, ) update_query.execute() @@ -131,9 +130,9 @@ def purge_duplicates(self) -> None: select id, label, camera, - has_snapshot, - has_clip, - row_number() over ( + has_snapshot, + has_clip, + row_number() over ( partition by label, camera, round(start_time/5,0)*5 order by end_time-start_time desc ) as copy_number @@ -169,8 +168,8 @@ def run(self) -> None: # drop events from db where has_clip and has_snapshot are false delete_query = Event.delete().where( - Event.has_clip == False, Event.has_snapshot == False + Event.has_clip is False, Event.has_snapshot is False ) delete_query.execute() - logger.info(f"Exiting event cleanup...") + logger.info("Exiting event cleanup...") diff --git a/frigate/events/external.py b/frigate/events/external.py index ee2906e02c..2c17c9a032 100644 --- a/frigate/events/external.py +++ b/frigate/events/external.py @@ -3,7 +3,6 @@ import base64 import cv2 import datetime -import glob import logging import os import random diff --git a/frigate/events/maintainer.py b/frigate/events/maintainer.py index 898218ef04..f674c5ccfa 100644 --- a/frigate/events/maintainer.py +++ b/frigate/events/maintainer.py @@ -5,7 +5,6 @@ from enum import Enum -from peewee import fn from frigate.config import EventsConfig, FrigateConfig from frigate.models import Event @@ -65,7 +64,7 @@ def __init__( def run(self) -> None: # set an end_time on events without an end_time on startup Event.update(end_time=Event.start_time + 30).where( - Event.end_time == None + Event.end_time is None ).execute() while not self.stop_event.is_set(): @@ -99,9 +98,9 @@ def run(self) -> None: # set an end_time on events without an end_time before exiting Event.update(end_time=datetime.datetime.now().timestamp()).where( - Event.end_time == None + Event.end_time is None ).execute() - logger.info(f"Exiting event processor...") + logger.info("Exiting event processor...") def handle_object_detection( self, diff --git a/frigate/http.py b/frigate/http.py index de9fd033c3..7abaa805b8 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -105,10 +105,10 @@ def events_summary(): clauses = [] - if not has_clip is None: + if has_clip is not None: clauses.append((Event.has_clip == has_clip)) - if not has_snapshot is None: + if has_snapshot is not None: clauses.append((Event.has_snapshot == has_snapshot)) if len(clauses) == 0: @@ -253,7 +253,7 @@ def send_to_plus(id): event.plus_id = plus_id event.save() - if not include_annotation is None: + if include_annotation is not None: box = event.data["box"] try: @@ -296,12 +296,12 @@ def false_positive(id): # events from before the conversion to relative dimensions cant include annotations if event.data.get("box") is None: - message = f"Events prior to 0.13 cannot be submitted as false positives" + message = "Events prior to 0.13 cannot be submitted as false positives" logger.error(message) return make_response(jsonify({"success": False, "message": message}), 400) if event.false_positive: - message = f"False positive already submitted to Frigate+" + message = "False positive already submitted to Frigate+" logger.error(message) return make_response(jsonify({"success": False, "message": message}), 400) @@ -437,7 +437,7 @@ def get_sub_labels(): parts = label.split(",") for part in parts: - if not (part.strip()) in sub_labels: + if part.strip() not in sub_labels: sub_labels.append(part.strip()) sub_labels.sort() @@ -476,7 +476,7 @@ def event_thumbnail(id, max_cache_age=2592000): event_complete = False try: event = Event.get(Event.id == id) - if not event.end_time is None: + if event.end_time is not None: event_complete = True thumbnail_bytes = base64.b64decode(event.thumbnail) except DoesNotExist: @@ -486,7 +486,7 @@ def event_thumbnail(id, max_cache_age=2592000): for camera_state in camera_states: if id in camera_state.tracked_objects: tracked_obj = camera_state.tracked_objects.get(id) - if not tracked_obj is None: + if tracked_obj is not None: thumbnail_bytes = tracked_obj.get_thumbnail() except: return "Event not found", 404 @@ -593,7 +593,7 @@ def event_snapshot(id): event_complete = False jpg_bytes = None try: - event = Event.get(Event.id == id, Event.end_time != None) + event = Event.get(Event.id == id, Event.end_time is not None) event_complete = True if not event.has_snapshot: return "Snapshot not available", 404 @@ -609,7 +609,7 @@ def event_snapshot(id): for camera_state in camera_states: if id in camera_state.tracked_objects: tracked_obj = camera_state.tracked_objects.get(id) - if not tracked_obj is None: + if tracked_obj is not None: jpg_bytes = tracked_obj.get_jpg_bytes( timestamp=request.args.get("timestamp", type=int), bounding_box=request.args.get("bbox", type=int), @@ -645,7 +645,7 @@ def label_snapshot(camera_name, label): event_query = ( Event.select() .where(Event.camera == camera_name) - .where(Event.has_snapshot == True) + .where(Event.has_snapshot is True) .order_by(Event.start_time.desc()) ) else: @@ -653,7 +653,7 @@ def label_snapshot(camera_name, label): Event.select() .where(Event.camera == camera_name) .where(Event.label == label) - .where(Event.has_snapshot == True) + .where(Event.has_snapshot is True) .order_by(Event.start_time.desc()) ) @@ -820,13 +820,13 @@ def events(): if before: clauses.append((Event.start_time < before)) - if not has_clip is None: + if has_clip is not None: clauses.append((Event.has_clip == has_clip)) - if not has_snapshot is None: + if has_snapshot is not None: clauses.append((Event.has_snapshot == has_snapshot)) - if not in_progress is None: + if in_progress is not None: clauses.append((Event.end_time.is_null(in_progress))) if not include_thumbnails: @@ -899,7 +899,7 @@ def end_event(event_id): {"success": False, "message": f"{event_id} must be set and valid."}, 404 ) - return jsonify({"success": True, "message": f"Event successfully ended."}, 200) + return jsonify({"success": True, "message": "Event successfully ended."}, 200) @bp.route("/config") @@ -959,9 +959,8 @@ def config_save(): # Validate the config schema try: - new_yaml = FrigateConfig.parse_raw(new_config) - check_runtime = new_yaml.runtime_config - except Exception as e: + FrigateConfig.parse_raw(new_config) + except Exception: return make_response( jsonify( { @@ -985,12 +984,12 @@ def config_save(): with open(config_file, "w") as f: f.write(new_config) f.close() - except Exception as e: + except Exception: return make_response( jsonify( { "success": False, - "message": f"Could not write config file, be sure that Frigate has write permission on the config file.", + "message": "Could not write config file, be sure that Frigate has write permission on the config file.", } ), 400, @@ -1531,7 +1530,7 @@ def ffprobe(): if not path_param: return jsonify( - {"success": False, "message": f"Path needs to be provided."}, "404" + {"success": False, "message": "Path needs to be provided."}, "404" ) if path_param.startswith("camera"): diff --git a/frigate/log.py b/frigate/log.py index 67866942cc..f835b59002 100644 --- a/frigate/log.py +++ b/frigate/log.py @@ -10,8 +10,7 @@ from typing import Optional from types import FrameType from setproctitle import setproctitle -from typing import Deque, Optional -from types import FrameType +from typing import Deque from collections import deque from frigate.util import clean_camera_user_pass @@ -44,7 +43,7 @@ def root_configurer(queue: Queue) -> None: def log_process(log_queue: Queue) -> None: - threading.current_thread().name = f"logger" + threading.current_thread().name = "logger" setproctitle("frigate.logger") listener_configurer() diff --git a/frigate/models.py b/frigate/models.py index 0d5f1ab4a7..6a81eeda4d 100644 --- a/frigate/models.py +++ b/frigate/models.py @@ -1,4 +1,3 @@ -from numpy import unique from peewee import ( Model, CharField, diff --git a/frigate/object_detection.py b/frigate/object_detection.py index 129fd6b26a..de9494fb15 100644 --- a/frigate/object_detection.py +++ b/frigate/object_detection.py @@ -161,7 +161,7 @@ def stop(self): def start_or_restart(self): self.detection_start.value = 0.0 - if (not self.detect_process is None) and self.detect_process.is_alive(): + if (self.detect_process is not None) and self.detect_process.is_alive(): self.stop() self.detect_process = mp.Process( target=run_detector, diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 3088857fa5..09181c26fd 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -141,7 +141,7 @@ def update(self, current_frame_time, obj_data): # check each zone for name, zone in self.camera_config.zones.items(): # if the zone is not for this object type, skip - if len(zone.objects) > 0 and not obj_data["label"] in zone.objects: + if len(zone.objects) > 0 and obj_data["label"] not in zone.objects: continue contour = zone.contour # check if the object is in the zone @@ -177,9 +177,9 @@ def update(self, current_frame_time, obj_data): return (thumb_update, significant_change) def to_dict(self, include_thumbnail: bool = False): - snapshot_time = ( + ( self.thumbnail_data["frame_time"] - if not self.thumbnail_data is None + if self.thumbnail_data is not None else 0.0 ) event = { @@ -526,7 +526,7 @@ def update(self, frame_time, current_detections, motion_boxes, regions): for id in removed_ids: # publish events to mqtt removed_obj = tracked_objects[id] - if not "end_time" in removed_obj.obj_data: + if "end_time" not in removed_obj.obj_data: removed_obj.obj_data["end_time"] = frame_time for c in self.callbacks["end"]: c(self.name, removed_obj, frame_time) @@ -1028,4 +1028,4 @@ def run(self): event_id, camera = self.event_processed_queue.get() self.camera_states[camera].finished(event_id) - logger.info(f"Exiting object processor...") + logger.info("Exiting object processor...") diff --git a/frigate/objects.py b/frigate/objects.py index ed04ae1918..d11aff2938 100644 --- a/frigate/objects.py +++ b/frigate/objects.py @@ -1,14 +1,7 @@ -import copy -import datetime -import itertools -import multiprocessing as mp import random import string -import threading -import time from collections import defaultdict -import cv2 import numpy as np from scipy.spatial import distance as dist @@ -160,7 +153,7 @@ def match_and_update(self, frame_time, new_objects): # update any tracked objects with labels that are not # seen in the current objects and deregister if needed for obj in list(self.tracked_objects.values()): - if not obj["label"] in new_object_groups: + if obj["label"] not in new_object_groups: if self.disappeared[obj["id"]] >= self.max_disappeared: self.deregister(obj["id"]) else: diff --git a/frigate/output.py b/frigate/output.py index 79a081837c..323613e435 100644 --- a/frigate/output.py +++ b/frigate/output.py @@ -4,7 +4,6 @@ import math import multiprocessing as mp import os -import operator import queue import signal import subprocess as sp @@ -185,7 +184,7 @@ def __init__(self, config: FrigateConfig, frame_manager: SharedMemoryFrameManage if len(logo_files) > 0: birdseye_logo = cv2.imread(logo_files[0], cv2.IMREAD_UNCHANGED) - if not birdseye_logo is None: + if birdseye_logo is not None: transparent_layer = birdseye_logo[:, :, 3] y_offset = height // 2 - transparent_layer.shape[0] // 2 x_offset = width // 2 - transparent_layer.shape[1] // 2 @@ -229,7 +228,7 @@ def __init__(self, config: FrigateConfig, frame_manager: SharedMemoryFrameManage self.last_output_time = 0.0 def clear_frame(self): - logger.debug(f"Clearing the birdseye frame") + logger.debug("Clearing the birdseye frame") self.frame[:] = self.blank_frame def copy_to_position(self, position, camera=None, frame_time=None): @@ -301,7 +300,7 @@ def update_frame(self): # reset the layout if it needs to be different if layout_dim != self.layout_dim or reset_layout: if reset_layout: - logger.debug(f"Added new cameras, resetting layout...") + logger.debug("Added new cameras, resetting layout...") logger.debug(f"Changing layout size from {self.layout_dim} to {layout_dim}") self.layout_dim = layout_dim @@ -385,7 +384,7 @@ def update_frame(self): ] # if not an empty spot and the camera has a newer frame, copy it elif ( - not camera is None + camera is not None and self.cameras[camera]["current_frame"] != self.cameras[camera]["layout_frame"] ): @@ -423,8 +422,8 @@ def update(self, camera, object_count, motion_count, frame_time, frame) -> bool: def output_frames(config: FrigateConfig, video_output_queue): - threading.current_thread().name = f"output" - setproctitle(f"frigate.output") + threading.current_thread().name = "output" + setproctitle("frigate.output") stop_event = mp.Event() diff --git a/frigate/plus.py b/frigate/plus.py index 7ba67a62d9..8638989176 100644 --- a/frigate/plus.py +++ b/frigate/plus.py @@ -3,7 +3,7 @@ import logging import os import re -from typing import Any, Dict, List +from typing import Any, List import requests from frigate.const import PLUS_ENV_VAR, PLUS_API_HOST from requests.models import Response diff --git a/frigate/ptz.py b/frigate/ptz.py index a52006b5c6..3d02b8a9ea 100644 --- a/frigate/ptz.py +++ b/frigate/ptz.py @@ -145,7 +145,7 @@ def _move(self, camera_name: str, command: OnvifCommandEnum) -> None: onvif.get_service("ptz").ContinuousMove(move_request) def _move_to_preset(self, camera_name: str, preset: str) -> None: - if not preset in self.cams[camera_name]["presets"]: + if preset not in self.cams[camera_name]["presets"]: logger.error(f"{preset} is not a valid preset for {camera_name}") return diff --git a/frigate/record/cleanup.py b/frigate/record/cleanup.py index 605979ee42..f1f01486cf 100644 --- a/frigate/record/cleanup.py +++ b/frigate/record/cleanup.py @@ -225,7 +225,7 @@ def sync_recordings(self) -> None: recordings_to_delete = [] for recording in recordings.objects().iterator(): - if not recording.path in files_on_disk: + if recording.path not in files_on_disk: recordings_to_delete.append(recording.id) logger.debug( @@ -247,7 +247,7 @@ def run(self) -> None: # Expire tmp clips every minute, recordings and clean directories every hour. for counter in itertools.cycle(range(self.config.record.expire_interval)): if self.stop_event.wait(60): - logger.info(f"Exiting recording cleanup...") + logger.info("Exiting recording cleanup...") break self.clean_tmp_clips() diff --git a/frigate/record/maintainer.py b/frigate/record/maintainer.py index 651bd12149..fc8a00d8b9 100644 --- a/frigate/record/maintainer.py +++ b/frigate/record/maintainer.py @@ -115,7 +115,7 @@ def move_files(self) -> None: Event.select() .where( Event.camera == camera, - (Event.end_time == None) + (Event.end_time is None) | (Event.end_time >= recordings[0]["start_time"].timestamp()), Event.has_clip, ) @@ -127,7 +127,7 @@ def move_files(self) -> None: # Just delete files if recordings are turned off if ( - not camera in self.config.cameras + camera not in self.config.cameras or not self.process_info[camera]["record_enabled"].value ): Path(cache_path).unlink(missing_ok=True) @@ -394,4 +394,4 @@ def run(self) -> None: duration = datetime.datetime.now().timestamp() - run_start wait_time = max(0, 5 - duration) - logger.info(f"Exiting recording maintenance...") + logger.info("Exiting recording maintenance...") diff --git a/frigate/stats.py b/frigate/stats.py index 55db809d38..3f042cf4a9 100644 --- a/frigate/stats.py +++ b/frigate/stats.py @@ -308,4 +308,4 @@ def run(self) -> None: ) self.dispatcher.publish("stats", json.dumps(stats), retain=False) logger.debug("Finished stats collection") - logger.info(f"Exiting stats emitter...") + logger.info("Exiting stats emitter...") diff --git a/frigate/storage.py b/frigate/storage.py index 5b66abbdc4..c22a6363f9 100644 --- a/frigate/storage.py +++ b/frigate/storage.py @@ -107,7 +107,7 @@ def reduce_storage_consumption(self) -> None: retained_events: Event = ( Event.select() .where( - Event.retain_indefinitely == True, + Event.retain_indefinitely is True, Event.has_clip, ) .order_by(Event.start_time.asc()) @@ -188,4 +188,4 @@ def run(self): if self.check_storage_needs_cleanup(): self.reduce_storage_consumption() - logger.info(f"Exiting storage maintainer...") + logger.info("Exiting storage maintainer...") diff --git a/frigate/test/test_config.py b/frigate/test/test_config.py index 7cd5e22178..51109b35ce 100644 --- a/frigate/test/test_config.py +++ b/frigate/test/test_config.py @@ -675,7 +675,7 @@ def test_role_assigned_but_not_enabled(self): runtime_config = frigate_config.runtime_config() ffmpeg_cmds = runtime_config.cameras["back"].ffmpeg_cmds assert len(ffmpeg_cmds) == 1 - assert not "clips" in ffmpeg_cmds[0]["roles"] + assert "clips" not in ffmpeg_cmds[0]["roles"] def test_max_disappeared_default(self): config = { @@ -986,7 +986,7 @@ def test_works_on_missing_role_multiple_cams(self): } frigate_config = FrigateConfig(**config) - runtime_config = frigate_config.runtime_config() + frigate_config.runtime_config() def test_global_detect(self): config = { @@ -1145,7 +1145,7 @@ def test_global_snapshots_merge(self): assert config == frigate_config.dict(exclude_unset=True) runtime_config = frigate_config.runtime_config() - assert runtime_config.cameras["back"].snapshots.bounding_box == False + assert runtime_config.cameras["back"].snapshots.bounding_box is False assert runtime_config.cameras["back"].snapshots.height == 150 assert runtime_config.cameras["back"].snapshots.enabled diff --git a/frigate/test/test_gpu_stats.py b/frigate/test/test_gpu_stats.py index 5742a583d1..2cf92def1d 100644 --- a/frigate/test/test_gpu_stats.py +++ b/frigate/test/test_gpu_stats.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import MagicMock, patch -from frigate.util import get_amd_gpu_stats, get_intel_gpu_stats, get_nvidia_gpu_stats +from frigate.util import get_amd_gpu_stats, get_intel_gpu_stats class TestGpuStats(unittest.TestCase): diff --git a/frigate/test/test_http.py b/frigate/test/test_http.py index ecb5003836..1eb0de1978 100644 --- a/frigate/test/test_http.py +++ b/frigate/test/test_http.py @@ -128,22 +128,22 @@ def test_get_event_list(self): with app.test_client() as client: _insert_mock_event(id) - events = client.get(f"/events").json + events = client.get("/events").json assert events assert len(events) == 1 assert events[0]["id"] == id _insert_mock_event(id2) - events = client.get(f"/events").json + events = client.get("/events").json assert events assert len(events) == 2 events = client.get( - f"/events", + "/events", query_string={"limit": 1}, ).json assert events assert len(events) == 1 events = client.get( - f"/events", + "/events", query_string={"has_clip": 0}, ).json assert not events @@ -230,12 +230,12 @@ def test_event_retention(self): event = client.get(f"/events/{id}").json assert event assert event["id"] == id - assert event["retain_indefinitely"] == True + assert event["retain_indefinitely"] is True client.delete(f"/events/{id}/retain") event = client.get(f"/events/{id}").json assert event assert event["id"] == id - assert event["retain_indefinitely"] == False + assert event["retain_indefinitely"] is False def test_set_delete_sub_label(self): app = create_app( diff --git a/frigate/test/test_reduce_boxes.py b/frigate/test/test_reduce_boxes.py index 69d6f71773..1b961a3d4d 100644 --- a/frigate/test/test_reduce_boxes.py +++ b/frigate/test/test_reduce_boxes.py @@ -1,4 +1,3 @@ -import numpy as np from unittest import TestCase, main from frigate.video import box_overlaps, reduce_boxes diff --git a/frigate/test/test_storage.py b/frigate/test/test_storage.py index 22ea6fc1eb..a17e067d41 100644 --- a/frigate/test/test_storage.py +++ b/frigate/test/test_storage.py @@ -1,18 +1,15 @@ import datetime -import json import logging import os import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock from peewee import DoesNotExist from peewee_migrate import Router from playhouse.sqlite_ext import SqliteExtDatabase from playhouse.sqliteq import SqliteQueueDatabase -from playhouse.shortcuts import model_to_dict from frigate.config import FrigateConfig -from frigate.http import create_app from frigate.models import Event, Recordings from frigate.storage import StorageMaintainer diff --git a/frigate/test/test_yuv_region_2_rgb.py b/frigate/test/test_yuv_region_2_rgb.py index 96f421565d..6681b56362 100644 --- a/frigate/test/test_yuv_region_2_rgb.py +++ b/frigate/test/test_yuv_region_2_rgb.py @@ -33,7 +33,7 @@ def test_crop_yuv_portrait(self): # cv2.imwrite(f"bgr_frame.jpg", self.bgr_frame) yuv_frame = cv2.cvtColor(bgr_frame, cv2.COLOR_BGR2YUV_I420) - cropped = yuv_region_2_rgb(yuv_frame, (0, 852, 648, 1500)) + yuv_region_2_rgb(yuv_frame, (0, 852, 648, 1500)) # cv2.imwrite(f"cropped.jpg", cv2.cvtColor(cropped, cv2.COLOR_RGB2BGR)) diff --git a/frigate/util.py b/frigate/util.py index 47b7b43230..53d85bf9ab 100755 --- a/frigate/util.py +++ b/frigate/util.py @@ -20,7 +20,6 @@ import cv2 import numpy as np -import os import psutil import pytz @@ -457,7 +456,7 @@ def copy_yuv_to_position( # clear v2 destination_frame[v2[1] : v2[3], v2[0] : v2[2]] = 128 - if not source_frame is None: + if source_frame is not None: # calculate the resized frame, maintaining the aspect ratio source_aspect_ratio = source_frame.shape[1] / (source_frame.shape[0] // 3 * 2) dest_aspect_ratio = destination_shape[1] / destination_shape[0] diff --git a/frigate/video.py b/frigate/video.py index a3356a5866..263d8f893f 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -45,7 +45,7 @@ def filtered(obj, objects_to_track, object_filters): object_area = obj[3] object_ratio = obj[4] - if not object_name in objects_to_track: + if object_name not in objects_to_track: return True if object_name in object_filters: @@ -73,7 +73,7 @@ def filtered(obj, objects_to_track, object_filters): if obj_settings.max_ratio < object_ratio: return True - if not obj_settings.mask is None: + if obj_settings.mask is not None: # compute the coordinates of the object and make sure # the location isn't outside the bounds of the image (can happen from rounding) object_xmin = object_box[0] @@ -169,20 +169,20 @@ def capture_frames( skipped_eps.start() while True: fps.value = frame_rate.eps() - skipped_fps = skipped_eps.eps() + skipped_eps.eps() current_frame.value = datetime.datetime.now().timestamp() frame_name = f"{camera_name}{current_frame.value}" frame_buffer = frame_manager.create(frame_name, frame_size) try: frame_buffer[:] = ffmpeg_process.stdout.read(frame_size) - except Exception as e: + except Exception: # shutdown has been initiated if stop_event.is_set(): break logger.error(f"{camera_name}: Unable to read frames from ffmpeg process.") - if ffmpeg_process.poll() != None: + if ffmpeg_process.poll() is not None: logger.error( f"{camera_name}: ffmpeg process is not running. exiting capture thread..." ) @@ -604,7 +604,7 @@ def process_frames( while not stop_event.is_set(): if exit_on_empty and frame_queue.empty(): - logger.info(f"Exiting track_objects...") + logger.info("Exiting track_objects...") break try: @@ -655,7 +655,7 @@ def process_frames( tracked_object_boxes = [ obj["box"] for obj in object_tracker.tracked_objects.values() - if not obj["id"] in stationary_object_ids + if obj["id"] not in stationary_object_ids ] # combine motion boxes with known locations of existing objects diff --git a/frigate/watchdog.py b/frigate/watchdog.py index 5df826f0ed..94612e4faf 100644 --- a/frigate/watchdog.py +++ b/frigate/watchdog.py @@ -2,8 +2,6 @@ import logging import threading import time -import os -import signal from frigate.object_detection import ObjectDetectProcess from frigate.util import restart_frigate @@ -39,4 +37,4 @@ def run(self) -> None: logger.info("Detection appears to have stopped. Exiting Frigate...") restart_frigate() - logger.info(f"Exiting watchdog...") + logger.info("Exiting watchdog...") From f525b9cf688302cb4fe58fc7ece439e640264967 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Tue, 23 May 2023 02:57:35 +0200 Subject: [PATCH 3/7] treewide: fix bare except clauses --- frigate/detectors/plugins/openvino.py | 2 +- frigate/http.py | 8 ++++---- frigate/output.py | 2 +- frigate/record/maintainer.py | 2 +- frigate/stats.py | 3 ++- frigate/util.py | 10 +++++----- 6 files changed, 14 insertions(+), 13 deletions(-) diff --git a/frigate/detectors/plugins/openvino.py b/frigate/detectors/plugins/openvino.py index 3b65e5c293..3592cb7db8 100644 --- a/frigate/detectors/plugins/openvino.py +++ b/frigate/detectors/plugins/openvino.py @@ -41,7 +41,7 @@ def __init__(self, detector_config: OvDetectorConfig): tensor_shape = self.interpreter.output(self.output_indexes).shape logger.info(f"Model Output-{self.output_indexes} Shape: {tensor_shape}") self.output_indexes += 1 - except: + except Exception: logger.info(f"Model has {self.output_indexes} Output Tensors") break if self.ov_model_type == ModelTypeEnum.yolox: diff --git a/frigate/http.py b/frigate/http.py index 7abaa805b8..dd3ddfd76f 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -488,7 +488,7 @@ def event_thumbnail(id, max_cache_age=2592000): tracked_obj = camera_state.tracked_objects.get(id) if tracked_obj is not None: thumbnail_bytes = tracked_obj.get_thumbnail() - except: + except Exception: return "Event not found", 404 if thumbnail_bytes is None: @@ -617,9 +617,9 @@ def event_snapshot(id): height=request.args.get("h", type=int), quality=request.args.get("quality", default=70, type=int), ) - except: + except Exception: return "Event not found", 404 - except: + except Exception: return "Event not found", 404 if jpg_bytes is None: @@ -894,7 +894,7 @@ def create_event(camera_name, label): def end_event(event_id): try: current_app.external_processor.finish_manual_event(event_id) - except: + except Exception: return jsonify( {"success": False, "message": f"{event_id} must be set and valid."}, 404 ) diff --git a/frigate/output.py b/frigate/output.py index 323613e435..1a394d79fb 100644 --- a/frigate/output.py +++ b/frigate/output.py @@ -148,7 +148,7 @@ def run(self): ): try: ws.send(buf, binary=True) - except: + except ValueError: pass elif self.converter.process.poll() is not None: break diff --git a/frigate/record/maintainer.py b/frigate/record/maintainer.py index fc8a00d8b9..522da48d81 100644 --- a/frigate/record/maintainer.py +++ b/frigate/record/maintainer.py @@ -63,7 +63,7 @@ def move_files(self) -> None: for nt in flist: if nt.path.startswith(CACHE_DIR): files_in_use.append(nt.path.split("/")[-1]) - except: + except psutil.Error: continue # group recordings by camera diff --git a/frigate/stats.py b/frigate/stats.py index 3f042cf4a9..3359a76027 100644 --- a/frigate/stats.py +++ b/frigate/stats.py @@ -9,6 +9,7 @@ import requests from typing import Optional, Any from multiprocessing.synchronize import Event as MpEvent +from requests.exceptions import RequestException from frigate.comms.dispatcher import Dispatcher from frigate.config import FrigateConfig @@ -31,7 +32,7 @@ def get_latest_version(config: FrigateConfig) -> str: "https://api.github.com/repos/blakeblackshear/frigate/releases/latest", timeout=10, ) - except: + except RequestException: return "unknown" response = request.json() diff --git a/frigate/util.py b/frigate/util.py index 53d85bf9ab..2f93fefa05 100755 --- a/frigate/util.py +++ b/frigate/util.py @@ -839,7 +839,7 @@ def get_cpu_stats() -> dict[str, dict]: "mem": f"{mem_pct}", "cmdline": " ".join(cmdline), } - except: + except Exception: continue return usages @@ -870,7 +870,7 @@ def get_bandwidth_stats() -> dict[str, dict]: usages[process[len(process) - 2]] = { "bandwidth": round(float(stats[1]) + float(stats[2]), 1), } - except: + except (IndexError, ValueError): continue return usages @@ -990,11 +990,11 @@ def get_nvidia_gpu_stats() -> dict[int, dict]: "gpu": gpu_util, "mem": gpu_mem_util, } - except: + except Exception: + pass + finally: return results - return results - def ffprobe_stream(path: str) -> sp.CompletedProcess: """Run ffprobe on stream.""" From bea7c8239a1a274c5b2f10301f7aa9b3e3817128 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Tue, 23 May 2023 03:04:47 +0200 Subject: [PATCH 4/7] .devcontainer: Set up isort --- .devcontainer/devcontainer.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3e654335f5..60065a9650 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -52,7 +52,8 @@ "mikestead.dotenv", "csstools.postcss", "blanu.vscode-styled-jsx", - "bradlc.vscode-tailwindcss" + "bradlc.vscode-tailwindcss", + "ms-python.isort" ], "settings": { "remote.autoForwardPorts": false, @@ -68,6 +69,9 @@ "python.testing.unittestArgs": ["-v", "-s", "./frigate/test"], "files.trimTrailingWhitespace": true, "eslint.workingDirectories": ["./web"], + "isort.args": [ + "--settings-path=./pyproject.toml" + ], "[python]": { "editor.defaultFormatter": "ms-python.black-formatter", "editor.formatOnSave": true From 3fc4e7be0a778088ea55f1264ba4198317ac8cba Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Tue, 23 May 2023 03:43:14 +0200 Subject: [PATCH 5/7] treewide: optimize imports --- benchmark.py | 8 ++--- .../rootfs/usr/local/go2rtc/create_config.py | 7 +++-- frigate/__main__.py | 1 - frigate/app.py | 14 ++++----- frigate/comms/dispatcher.py | 7 ++--- frigate/comms/mqtt.py | 2 -- frigate/comms/ws.py | 4 +-- frigate/config.py | 31 +++++++------------ frigate/detectors/__init__.py | 4 +-- frigate/detectors/detection_api.py | 1 - frigate/detectors/detector_config.py | 8 ++--- frigate/detectors/detector_types.py | 8 ++--- frigate/detectors/plugins/cpu_tfl.py | 6 ++-- frigate/detectors/plugins/deepstack.py | 10 +++--- frigate/detectors/plugins/edgetpu_tfl.py | 5 +-- frigate/detectors/plugins/openvino.py | 6 ++-- frigate/detectors/plugins/tensorrt.py | 7 +++-- frigate/events/cleanup.py | 5 +-- frigate/events/external.py | 5 ++- frigate/events/maintainer.py | 9 ++---- frigate/ffmpeg_presets.py | 4 +-- frigate/http.py | 19 +++++------- frigate/log.py | 14 ++++----- frigate/models.py | 6 ++-- frigate/motion.py | 1 + frigate/object_detection.py | 1 - frigate/object_processing.py | 4 +-- frigate/plus.py | 8 +++-- frigate/ptz.py | 3 +- frigate/record/cleanup.py | 4 +-- frigate/record/maintainer.py | 6 ++-- frigate/record/record.py | 3 +- frigate/stats.py | 24 ++++++++------ frigate/storage.py | 2 +- frigate/test/test_config.py | 6 ++-- frigate/test/test_copy_yuv_to_position.py | 6 ++-- frigate/test/test_ffmpeg_presets.py | 1 + frigate/test/test_http.py | 3 +- frigate/test/test_object_detector.py | 4 +-- frigate/test/test_reduce_boxes.py | 1 + frigate/test/test_storage.py | 1 - frigate/test/test_yuv_region_2_rgb.py | 4 ++- frigate/timeline.py | 8 ++--- frigate/types.py | 4 +-- frigate/util.py | 13 ++++---- frigate/video.py | 6 ++-- frigate/watchdog.py | 2 +- migrations/001_create_events_table.py | 7 ----- migrations/002_add_clip_snapshot.py | 8 +---- migrations/003_create_recordings_table.py | 2 -- migrations/004_add_bbox_region_area.py | 10 ++---- migrations/005_make_end_time_nullable.py | 9 +----- migrations/006_add_motion_active_objects.py | 9 +----- migrations/007_add_retain_indefinitely.py | 9 +----- migrations/008_add_sub_label.py | 9 +----- migrations/009_add_object_filter_ratio.py | 1 + migrations/010_add_plus_image_id.py | 9 +----- migrations/012_add_segment_size.py | 9 +----- migrations/013_create_timeline_table.py | 9 ------ migrations/014_event_updates_for_fp.py | 9 +----- migrations/015_event_refactor.py | 10 ++---- migrations/016_sublabel_increase.py | 3 +- process_clip.py | 27 ++++++++-------- 63 files changed, 171 insertions(+), 275 deletions(-) diff --git a/benchmark.py b/benchmark.py index ea388ef13a..8ba22d0932 100755 --- a/benchmark.py +++ b/benchmark.py @@ -1,11 +1,11 @@ -import os -from statistics import mean +import datetime import multiprocessing as mp +from statistics import mean + import numpy as np -import datetime + from frigate.config import DetectorTypeEnum from frigate.object_detection import ( - LocalObjectDetector, ObjectDetectProcess, RemoteObjectDetector, load_labels, diff --git a/docker/rootfs/usr/local/go2rtc/create_config.py b/docker/rootfs/usr/local/go2rtc/create_config.py index d201eb3819..1397adee8e 100644 --- a/docker/rootfs/usr/local/go2rtc/create_config.py +++ b/docker/rootfs/usr/local/go2rtc/create_config.py @@ -3,11 +3,14 @@ import json import os import sys + import yaml sys.path.insert(0, "/opt/frigate") -from frigate.const import BIRDSEYE_PIPE, BTBN_PATH -from frigate.ffmpeg_presets import parse_preset_hardware_acceleration_encode +from frigate.const import BIRDSEYE_PIPE, BTBN_PATH # noqa: E402 +from frigate.ffmpeg_presets import ( # noqa: E402 + parse_preset_hardware_acceleration_encode, +) sys.path.remove("/opt/frigate") diff --git a/frigate/__main__.py b/frigate/__main__.py index 8fa98fe530..8442069082 100644 --- a/frigate/__main__.py +++ b/frigate/__main__.py @@ -5,7 +5,6 @@ from frigate.app import FrigateApp - faulthandler.enable() threading.current_thread().name = "frigate" diff --git a/frigate/app.py b/frigate/app.py index c66fab187e..bcbd3ed2d0 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -1,16 +1,16 @@ import logging import multiprocessing as mp -from multiprocessing.queues import Queue -from multiprocessing.synchronize import Event as MpEvent import os import shutil import signal import sys -from typing import Optional +import traceback +from multiprocessing.queues import Queue +from multiprocessing.synchronize import Event as MpEvent from types import FrameType -import psutil +from typing import Optional -import traceback +import psutil from peewee_migrate import Router from playhouse.sqlite_ext import SqliteExtDatabase from playhouse.sqliteq import SqliteQueueDatabase @@ -27,13 +27,13 @@ MODEL_CACHE_DIR, RECORD_DIR, ) -from frigate.object_detection import ObjectDetectProcess from frigate.events.cleanup import EventCleanup from frigate.events.external import ExternalEventProcessor from frigate.events.maintainer import EventProcessor from frigate.http import create_app from frigate.log import log_process, root_configurer from frigate.models import Event, Recordings, Timeline +from frigate.object_detection import ObjectDetectProcess from frigate.object_processing import TrackedObjectProcessor from frigate.output import output_frames from frigate.plus import PlusApi @@ -42,10 +42,10 @@ from frigate.stats import StatsEmitter, stats_init from frigate.storage import StorageMaintainer from frigate.timeline import TimelineProcessor +from frigate.types import CameraMetricsTypes, RecordMetricsTypes from frigate.version import VERSION from frigate.video import capture_camera, track_camera from frigate.watchdog import FrigateWatchdog -from frigate.types import CameraMetricsTypes, RecordMetricsTypes logger = logging.getLogger(__name__) diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py index 61e1c75221..b7e9e88586 100644 --- a/frigate/comms/dispatcher.py +++ b/frigate/comms/dispatcher.py @@ -1,17 +1,14 @@ """Handle communication between Frigate and other applications.""" import logging - -from typing import Any, Callable - from abc import ABC, abstractmethod +from typing import Any, Callable from frigate.config import FrigateConfig -from frigate.ptz import OnvifController, OnvifCommandEnum +from frigate.ptz import OnvifCommandEnum, OnvifController from frigate.types import CameraMetricsTypes, RecordMetricsTypes from frigate.util import restart_frigate - logger = logging.getLogger(__name__) diff --git a/frigate/comms/mqtt.py b/frigate/comms/mqtt.py index 0e8fb4d711..07799f9dab 100644 --- a/frigate/comms/mqtt.py +++ b/frigate/comms/mqtt.py @@ -1,6 +1,5 @@ import logging import threading - from typing import Any, Callable import paho.mqtt.client as mqtt @@ -8,7 +7,6 @@ from frigate.comms.dispatcher import Communicator from frigate.config import FrigateConfig - logger = logging.getLogger(__name__) diff --git a/frigate/comms/ws.py b/frigate/comms/ws.py index d036da8e70..3c1d2eb12b 100644 --- a/frigate/comms/ws.py +++ b/frigate/comms/ws.py @@ -3,10 +3,9 @@ import json import logging import threading - from typing import Callable - from wsgiref.simple_server import make_server + from ws4py.server.wsgirefserver import ( WebSocketWSGIHandler, WebSocketWSGIRequestHandler, @@ -18,7 +17,6 @@ from frigate.comms.dispatcher import Communicator from frigate.config import FrigateConfig - logger = logging.getLogger(__name__) diff --git a/frigate/config.py b/frigate/config.py index f9203582b5..5c2f27b5aa 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -8,24 +8,14 @@ import matplotlib.pyplot as plt import numpy as np -from pydantic import BaseModel, Extra, Field, validator, parse_obj_as +from pydantic import BaseModel, Extra, Field, parse_obj_as, validator from pydantic.fields import PrivateAttr -from frigate.const import ( - CACHE_DIR, - DEFAULT_DB_PATH, - REGEX_CAMERA_NAME, - YAML_EXT, -) +from frigate.const import CACHE_DIR, DEFAULT_DB_PATH, REGEX_CAMERA_NAME, YAML_EXT +from frigate.detectors import DetectorConfig, ModelConfig +from frigate.detectors.detector_config import InputTensorEnum # noqa: F401 +from frigate.detectors.detector_config import PixelFormatEnum # noqa: F401 from frigate.detectors.detector_config import BaseDetectorConfig -from frigate.plus import PlusApi -from frigate.util import ( - create_mask, - deep_merge, - get_ffmpeg_arg_list, - escape_special_characters, - load_config_with_no_duplicates, -) from frigate.ffmpeg_presets import ( parse_preset_hardware_acceleration_decode, parse_preset_hardware_acceleration_scale, @@ -33,12 +23,15 @@ parse_preset_output_record, parse_preset_output_rtmp, ) -from frigate.detectors import ( - ModelConfig, - DetectorConfig, +from frigate.plus import PlusApi +from frigate.util import ( + create_mask, + deep_merge, + escape_special_characters, + get_ffmpeg_arg_list, + load_config_with_no_duplicates, ) - logger = logging.getLogger(__name__) # TODO: Identify what the default format to display timestamps is diff --git a/frigate/detectors/__init__.py b/frigate/detectors/__init__.py index c04891f50a..7465ed7c0c 100644 --- a/frigate/detectors/__init__.py +++ b/frigate/detectors/__init__.py @@ -1,7 +1,7 @@ import logging -from .detector_types import DetectorTypeEnum, api_types, DetectorConfig - +from .detector_config import InputTensorEnum, ModelConfig, PixelFormatEnum # noqa: F401 +from .detector_types import DetectorConfig, DetectorTypeEnum, api_types # noqa: F401 logger = logging.getLogger(__name__) diff --git a/frigate/detectors/detection_api.py b/frigate/detectors/detection_api.py index 60a90b1b5a..f829da27a5 100644 --- a/frigate/detectors/detection_api.py +++ b/frigate/detectors/detection_api.py @@ -1,7 +1,6 @@ import logging from abc import ABC, abstractmethod - logger = logging.getLogger(__name__) diff --git a/frigate/detectors/detector_config.py b/frigate/detectors/detector_config.py index ce11adb2f7..f65826a572 100644 --- a/frigate/detectors/detector_config.py +++ b/frigate/detectors/detector_config.py @@ -1,20 +1,18 @@ import hashlib import json import logging -from enum import Enum import os +from enum import Enum from typing import Dict, Optional, Tuple - -import requests import matplotlib.pyplot as plt +import requests from pydantic import BaseModel, Extra, Field from pydantic.fields import PrivateAttr -from frigate.plus import PlusApi +from frigate.plus import PlusApi from frigate.util import load_labels - logger = logging.getLogger(__name__) diff --git a/frigate/detectors/detector_types.py b/frigate/detectors/detector_types.py index 5c96b2e16b..418fcd625c 100644 --- a/frigate/detectors/detector_types.py +++ b/frigate/detectors/detector_types.py @@ -1,16 +1,16 @@ -import logging import importlib +import logging import pkgutil -from typing import Union -from typing_extensions import Annotated from enum import Enum +from typing import Union + from pydantic import Field +from typing_extensions import Annotated from . import plugins from .detection_api import DetectionApi from .detector_config import BaseDetectorConfig - logger = logging.getLogger(__name__) diff --git a/frigate/detectors/plugins/cpu_tfl.py b/frigate/detectors/plugins/cpu_tfl.py index 6439a9c3db..8a54363e1a 100644 --- a/frigate/detectors/plugins/cpu_tfl.py +++ b/frigate/detectors/plugins/cpu_tfl.py @@ -1,12 +1,12 @@ import logging + import numpy as np +from pydantic import Field +from typing_extensions import Literal from frigate.detectors.detection_api import DetectionApi from frigate.detectors.detector_config import BaseDetectorConfig -from typing_extensions import Literal -from pydantic import Field - try: from tflite_runtime.interpreter import Interpreter except ModuleNotFoundError: diff --git a/frigate/detectors/plugins/deepstack.py b/frigate/detectors/plugins/deepstack.py index 6ef702cb1b..fd79e27d0c 100644 --- a/frigate/detectors/plugins/deepstack.py +++ b/frigate/detectors/plugins/deepstack.py @@ -1,14 +1,14 @@ +import io import logging + import numpy as np import requests -import io +from PIL import Image +from pydantic import Field +from typing_extensions import Literal from frigate.detectors.detection_api import DetectionApi from frigate.detectors.detector_config import BaseDetectorConfig -from typing_extensions import Literal -from pydantic import Field -from PIL import Image - logger = logging.getLogger(__name__) diff --git a/frigate/detectors/plugins/edgetpu_tfl.py b/frigate/detectors/plugins/edgetpu_tfl.py index 7e35877c4e..ca03d483b1 100644 --- a/frigate/detectors/plugins/edgetpu_tfl.py +++ b/frigate/detectors/plugins/edgetpu_tfl.py @@ -1,10 +1,11 @@ import logging + import numpy as np +from pydantic import Field +from typing_extensions import Literal from frigate.detectors.detection_api import DetectionApi from frigate.detectors.detector_config import BaseDetectorConfig -from typing_extensions import Literal -from pydantic import Field try: from tflite_runtime.interpreter import Interpreter, load_delegate diff --git a/frigate/detectors/plugins/openvino.py b/frigate/detectors/plugins/openvino.py index 3592cb7db8..5cb1ea39c3 100644 --- a/frigate/detectors/plugins/openvino.py +++ b/frigate/detectors/plugins/openvino.py @@ -1,12 +1,12 @@ import logging + import numpy as np import openvino.runtime as ov +from pydantic import Field +from typing_extensions import Literal from frigate.detectors.detection_api import DetectionApi from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum -from typing_extensions import Literal -from pydantic import Field - logger = logging.getLogger(__name__) diff --git a/frigate/detectors/plugins/tensorrt.py b/frigate/detectors/plugins/tensorrt.py index 26fa92ccb7..7251b8751e 100644 --- a/frigate/detectors/plugins/tensorrt.py +++ b/frigate/detectors/plugins/tensorrt.py @@ -1,6 +1,6 @@ +import ctypes import logging -import ctypes import numpy as np try: @@ -11,10 +11,11 @@ except ModuleNotFoundError: TRT_SUPPORT = False +from pydantic import Field +from typing_extensions import Literal + from frigate.detectors.detection_api import DetectionApi from frigate.detectors.detector_config import BaseDetectorConfig -from typing_extensions import Literal -from pydantic import Field logger = logging.getLogger(__name__) diff --git a/frigate/events/cleanup.py b/frigate/events/cleanup.py index a78794a830..5afd285c49 100644 --- a/frigate/events/cleanup.py +++ b/frigate/events/cleanup.py @@ -4,16 +4,13 @@ import logging import os import threading - +from multiprocessing.synchronize import Event as MpEvent from pathlib import Path - from frigate.config import FrigateConfig from frigate.const import CLIPS_DIR from frigate.models import Event -from multiprocessing.synchronize import Event as MpEvent - logger = logging.getLogger(__name__) diff --git a/frigate/events/external.py b/frigate/events/external.py index 2c17c9a032..5422de2608 100644 --- a/frigate/events/external.py +++ b/frigate/events/external.py @@ -1,16 +1,15 @@ """Handle external events created by the user.""" import base64 -import cv2 import datetime import logging import os import random import string - +from multiprocessing.queues import Queue from typing import Optional -from multiprocessing.queues import Queue +import cv2 from frigate.config import CameraConfig, FrigateConfig from frigate.const import CLIPS_DIR diff --git a/frigate/events/maintainer.py b/frigate/events/maintainer.py index f674c5ccfa..5459fea6c3 100644 --- a/frigate/events/maintainer.py +++ b/frigate/events/maintainer.py @@ -2,19 +2,16 @@ import logging import queue import threading - from enum import Enum - +from multiprocessing.queues import Queue +from multiprocessing.synchronize import Event as MpEvent +from typing import Dict from frigate.config import EventsConfig, FrigateConfig from frigate.models import Event from frigate.types import CameraMetricsTypes from frigate.util import to_relative_box -from multiprocessing.queues import Queue -from multiprocessing.synchronize import Event as MpEvent -from typing import Dict - logger = logging.getLogger(__name__) diff --git a/frigate/ffmpeg_presets.py b/frigate/ffmpeg_presets.py index 353a8b9229..d78cd72ef1 100644 --- a/frigate/ffmpeg_presets.py +++ b/frigate/ffmpeg_presets.py @@ -2,13 +2,11 @@ import logging import os - from typing import Any -from frigate.version import VERSION from frigate.const import BTBN_PATH from frigate.util import vainfo_hwaccel - +from frigate.version import VERSION logger = logging.getLogger(__name__) diff --git a/frigate/http.py b/frigate/http.py index dd3ddfd76f..9d7aa2c47f 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -1,23 +1,20 @@ import base64 -from datetime import datetime, timedelta, timezone import copy -import logging import glob import json +import logging import os import subprocess as sp -import pytz import time import traceback - +from datetime import datetime, timedelta, timezone from functools import reduce from pathlib import Path -from tzlocal import get_localzone_name from urllib.parse import unquote import cv2 - import numpy as np +import pytz from flask import ( Blueprint, Flask, @@ -27,26 +24,26 @@ make_response, request, ) - -from peewee import SqliteDatabase, operator, fn, DoesNotExist +from peewee import DoesNotExist, SqliteDatabase, fn, operator from playhouse.shortcuts import model_to_dict +from tzlocal import get_localzone_name from frigate.config import FrigateConfig from frigate.const import CLIPS_DIR, MAX_SEGMENT_DURATION, RECORD_DIR -from frigate.models import Event, Recordings, Timeline from frigate.events.external import ExternalEventProcessor +from frigate.models import Event, Recordings, Timeline from frigate.object_processing import TrackedObject from frigate.plus import PlusApi from frigate.ptz import OnvifController from frigate.stats import stats_snapshot +from frigate.storage import StorageMaintainer from frigate.util import ( clean_camera_user_pass, ffprobe_stream, + get_tz_modifiers, restart_frigate, vainfo_hwaccel, - get_tz_modifiers, ) -from frigate.storage import StorageMaintainer from frigate.version import VERSION logger = logging.getLogger(__name__) diff --git a/frigate/log.py b/frigate/log.py index f835b59002..4a44545adf 100644 --- a/frigate/log.py +++ b/frigate/log.py @@ -1,17 +1,17 @@ # adapted from https://medium.com/@jonathonbao/python3-logging-with-multiprocessing-f51f460b8778 import logging -import threading +import multiprocessing as mp import os -import signal import queue -import multiprocessing as mp -from multiprocessing.queues import Queue +import signal +import threading +from collections import deque from logging import handlers -from typing import Optional +from multiprocessing.queues import Queue from types import FrameType +from typing import Deque, Optional + from setproctitle import setproctitle -from typing import Deque -from collections import deque from frigate.util import clean_camera_user_pass diff --git a/frigate/models.py b/frigate/models.py index 6a81eeda4d..5f6ac7339c 100644 --- a/frigate/models.py +++ b/frigate/models.py @@ -1,11 +1,11 @@ from peewee import ( - Model, + BooleanField, CharField, DateTimeField, FloatField, - BooleanField, - TextField, IntegerField, + Model, + TextField, ) from playhouse.sqlite_ext import JSONField diff --git a/frigate/motion.py b/frigate/motion.py index 5580803a38..19d1629561 100644 --- a/frigate/motion.py +++ b/frigate/motion.py @@ -1,6 +1,7 @@ import cv2 import imutils import numpy as np + from frigate.config import MotionConfig diff --git a/frigate/object_detection.py b/frigate/object_detection.py index de9494fb15..9201dfcd19 100644 --- a/frigate/object_detection.py +++ b/frigate/object_detection.py @@ -12,7 +12,6 @@ from frigate.config import InputTensorEnum from frigate.detectors import create_detector - from frigate.util import EventsPerSecond, SharedMemoryFrameManager, listen, load_labels logger = logging.getLogger(__name__) diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 09181c26fd..8c16ef64ed 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -15,10 +15,10 @@ from frigate.comms.dispatcher import Dispatcher from frigate.config import ( CameraConfig, + FrigateConfig, MqttConfig, - SnapshotsConfig, RecordConfig, - FrigateConfig, + SnapshotsConfig, ) from frigate.const import CLIPS_DIR from frigate.events.maintainer import EventTypeEnum diff --git a/frigate/plus.py b/frigate/plus.py index 8638989176..b219e02a2d 100644 --- a/frigate/plus.py +++ b/frigate/plus.py @@ -4,11 +4,13 @@ import os import re from typing import Any, List -import requests -from frigate.const import PLUS_ENV_VAR, PLUS_API_HOST -from requests.models import Response + import cv2 +import requests from numpy import ndarray +from requests.models import Response + +from frigate.const import PLUS_API_HOST, PLUS_ENV_VAR logger = logging.getLogger(__name__) diff --git a/frigate/ptz.py b/frigate/ptz.py index 3d02b8a9ea..385a230bc9 100644 --- a/frigate/ptz.py +++ b/frigate/ptz.py @@ -2,13 +2,12 @@ import logging import site - from enum import Enum + from onvif import ONVIFCamera, ONVIFError from frigate.config import FrigateConfig - logger = logging.getLogger(__name__) diff --git a/frigate/record/cleanup.py b/frigate/record/cleanup.py index f1f01486cf..bb54d8b866 100644 --- a/frigate/record/cleanup.py +++ b/frigate/record/cleanup.py @@ -5,12 +5,12 @@ import logging import os import threading +from multiprocessing.synchronize import Event as MpEvent from pathlib import Path from peewee import DoesNotExist -from multiprocessing.synchronize import Event as MpEvent -from frigate.config import RetainModeEnum, FrigateConfig +from frigate.config import FrigateConfig, RetainModeEnum from frigate.const import RECORD_DIR, SECONDS_IN_DAY from frigate.models import Event, Recordings, Timeline from frigate.record.util import remove_empty_directories diff --git a/frigate/record/maintainer.py b/frigate/record/maintainer.py index 522da48d81..afcd6af28d 100644 --- a/frigate/record/maintainer.py +++ b/frigate/record/maintainer.py @@ -9,14 +9,14 @@ import string import subprocess as sp import threading -import psutil - from collections import defaultdict from multiprocessing.synchronize import Event as MpEvent from pathlib import Path from typing import Any, Tuple -from frigate.config import RetainModeEnum, FrigateConfig +import psutil + +from frigate.config import FrigateConfig, RetainModeEnum from frigate.const import CACHE_DIR, MAX_SEGMENT_DURATION, RECORD_DIR from frigate.models import Event, Recordings from frigate.types import RecordMetricsTypes diff --git a/frigate/record/record.py b/frigate/record/record.py index 59fda095b2..3aaf56476a 100644 --- a/frigate/record/record.py +++ b/frigate/record/record.py @@ -4,12 +4,11 @@ import multiprocessing as mp import signal import threading - -from setproctitle import setproctitle from types import FrameType from typing import Optional from playhouse.sqliteq import SqliteQueueDatabase +from setproctitle import setproctitle from frigate.config import FrigateConfig from frigate.models import Event, Recordings, Timeline diff --git a/frigate/stats.py b/frigate/stats.py index 3359a76027..3fd8810290 100644 --- a/frigate/stats.py +++ b/frigate/stats.py @@ -1,24 +1,30 @@ import asyncio import json import logging +import os +import shutil import threading import time +from multiprocessing.synchronize import Event as MpEvent +from typing import Any, Optional + import psutil -import shutil -import os import requests -from typing import Optional, Any -from multiprocessing.synchronize import Event as MpEvent from requests.exceptions import RequestException from frigate.comms.dispatcher import Dispatcher from frigate.config import FrigateConfig -from frigate.const import DRIVER_AMD, DRIVER_ENV_VAR, RECORD_DIR, CLIPS_DIR, CACHE_DIR -from frigate.types import StatsTrackingTypes, CameraMetricsTypes -from frigate.util import get_amd_gpu_stats, get_intel_gpu_stats, get_nvidia_gpu_stats -from frigate.version import VERSION -from frigate.util import get_cpu_stats, get_bandwidth_stats +from frigate.const import CACHE_DIR, CLIPS_DIR, DRIVER_AMD, DRIVER_ENV_VAR, RECORD_DIR from frigate.object_detection import ObjectDetectProcess +from frigate.types import CameraMetricsTypes, StatsTrackingTypes +from frigate.util import ( + get_amd_gpu_stats, + get_bandwidth_stats, + get_cpu_stats, + get_intel_gpu_stats, + get_nvidia_gpu_stats, +) +from frigate.version import VERSION logger = logging.getLogger(__name__) diff --git a/frigate/storage.py b/frigate/storage.py index c22a6363f9..6cdd54bdcb 100644 --- a/frigate/storage.py +++ b/frigate/storage.py @@ -1,9 +1,9 @@ """Handle storage retention and usage.""" import logging -from pathlib import Path import shutil import threading +from pathlib import Path from peewee import fn diff --git a/frigate/test/test_config.py b/frigate/test/test_config.py index 51109b35ce..8c1b5b5865 100644 --- a/frigate/test/test_config.py +++ b/frigate/test/test_config.py @@ -1,13 +1,11 @@ import json import os import unittest + import numpy as np from pydantic import ValidationError -from frigate.config import ( - BirdseyeModeEnum, - FrigateConfig, -) +from frigate.config import BirdseyeModeEnum, FrigateConfig from frigate.const import MODEL_CACHE_DIR from frigate.detectors import DetectorTypeEnum from frigate.plus import PlusApi diff --git a/frigate/test/test_copy_yuv_to_position.py b/frigate/test/test_copy_yuv_to_position.py index 16c97b79d7..33582e2d1e 100644 --- a/frigate/test/test_copy_yuv_to_position.py +++ b/frigate/test/test_copy_yuv_to_position.py @@ -1,7 +1,9 @@ +from unittest import TestCase, main + import cv2 import numpy as np -from unittest import TestCase, main -from frigate.util import get_yuv_crop, copy_yuv_to_position + +from frigate.util import copy_yuv_to_position, get_yuv_crop class TestCopyYuvToPosition(TestCase): diff --git a/frigate/test/test_ffmpeg_presets.py b/frigate/test/test_ffmpeg_presets.py index 92e0fa3bd9..0d3dab8569 100644 --- a/frigate/test/test_ffmpeg_presets.py +++ b/frigate/test/test_ffmpeg_presets.py @@ -1,4 +1,5 @@ import unittest + from frigate.config import FFMPEG_INPUT_ARGS_DEFAULT, FrigateConfig from frigate.ffmpeg_presets import parse_preset_input diff --git a/frigate/test/test_http.py b/frigate/test/test_http.py index 1eb0de1978..3557eccd3a 100644 --- a/frigate/test/test_http.py +++ b/frigate/test/test_http.py @@ -6,15 +6,14 @@ from unittest.mock import patch from peewee_migrate import Router +from playhouse.shortcuts import model_to_dict from playhouse.sqlite_ext import SqliteExtDatabase from playhouse.sqliteq import SqliteQueueDatabase -from playhouse.shortcuts import model_to_dict from frigate.config import FrigateConfig from frigate.http import create_app from frigate.models import Event, Recordings from frigate.plus import PlusApi - from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS diff --git a/frigate/test/test_object_detector.py b/frigate/test/test_object_detector.py index 9cdeeb6c78..1e1efa9576 100644 --- a/frigate/test/test_object_detector.py +++ b/frigate/test/test_object_detector.py @@ -4,10 +4,10 @@ import numpy as np from pydantic import parse_obj_as -from frigate.config import DetectorConfig, InputTensorEnum, ModelConfig -from frigate.detectors import DetectorTypeEnum import frigate.detectors as detectors import frigate.object_detection +from frigate.config import DetectorConfig, InputTensorEnum, ModelConfig +from frigate.detectors import DetectorTypeEnum class TestLocalObjectDetector(unittest.TestCase): diff --git a/frigate/test/test_reduce_boxes.py b/frigate/test/test_reduce_boxes.py index 1b961a3d4d..d26fcd40c9 100644 --- a/frigate/test/test_reduce_boxes.py +++ b/frigate/test/test_reduce_boxes.py @@ -1,4 +1,5 @@ from unittest import TestCase, main + from frigate.video import box_overlaps, reduce_boxes diff --git a/frigate/test/test_storage.py b/frigate/test/test_storage.py index a17e067d41..468d319169 100644 --- a/frigate/test/test_storage.py +++ b/frigate/test/test_storage.py @@ -12,7 +12,6 @@ from frigate.config import FrigateConfig from frigate.models import Event, Recordings from frigate.storage import StorageMaintainer - from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS diff --git a/frigate/test/test_yuv_region_2_rgb.py b/frigate/test/test_yuv_region_2_rgb.py index 6681b56362..a56a78b1cc 100644 --- a/frigate/test/test_yuv_region_2_rgb.py +++ b/frigate/test/test_yuv_region_2_rgb.py @@ -1,6 +1,8 @@ +from unittest import TestCase, main + import cv2 import numpy as np -from unittest import TestCase, main + from frigate.util import yuv_region_2_rgb diff --git a/frigate/timeline.py b/frigate/timeline.py index e40549d08e..9ca617ba91 100644 --- a/frigate/timeline.py +++ b/frigate/timeline.py @@ -1,16 +1,14 @@ """Record events for object, audio, etc. detections.""" import logging -import threading import queue +import threading +from multiprocessing.queues import Queue +from multiprocessing.synchronize import Event as MpEvent from frigate.config import FrigateConfig from frigate.events.maintainer import EventTypeEnum from frigate.models import Timeline - -from multiprocessing.queues import Queue -from multiprocessing.synchronize import Event as MpEvent - from frigate.util import to_relative_box logger = logging.getLogger(__name__) diff --git a/frigate/types.py b/frigate/types.py index 3cc401ebba..8c3e546541 100644 --- a/frigate/types.py +++ b/frigate/types.py @@ -1,7 +1,7 @@ -from typing import Optional, TypedDict +from multiprocessing.context import Process from multiprocessing.queues import Queue from multiprocessing.sharedctypes import Synchronized -from multiprocessing.context import Process +from typing import Optional, TypedDict from frigate.object_detection import ObjectDetectProcess diff --git a/frigate/util.py b/frigate/util.py index 2f93fefa05..056c5eb1b2 100755 --- a/frigate/util.py +++ b/frigate/util.py @@ -1,27 +1,26 @@ import copy import datetime -import logging -import shlex -import subprocess as sp import json +import logging +import os import re +import shlex import signal +import subprocess as sp import traceback import urllib.parse -import yaml -import os - from abc import ABC, abstractmethod from collections import Counter from collections.abc import Mapping from multiprocessing import shared_memory from typing import Any, AnyStr, Optional, Tuple -import py3nvml.py3nvml as nvml import cv2 import numpy as np import psutil +import py3nvml.py3nvml as nvml import pytz +import yaml from frigate.const import REGEX_HTTP_CAMERA_USER_PASS, REGEX_RTSP_CAMERA_USER_PASS diff --git a/frigate/video.py b/frigate/video.py index 263d8f893f..f7ec08e995 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -10,15 +10,15 @@ import time from collections import defaultdict -import numpy as np import cv2 +import numpy as np from setproctitle import setproctitle from frigate.config import CameraConfig, DetectConfig, PixelFormatEnum from frigate.const import CACHE_DIR -from frigate.object_detection import RemoteObjectDetector from frigate.log import LogPipe from frigate.motion import MotionDetector +from frigate.object_detection import RemoteObjectDetector from frigate.objects import ObjectTracker from frigate.util import ( EventsPerSecond, @@ -30,8 +30,8 @@ intersection, intersection_over_union, listen, - yuv_region_2_rgb, yuv_region_2_bgr, + yuv_region_2_rgb, yuv_region_2_yuv, ) diff --git a/frigate/watchdog.py b/frigate/watchdog.py index 94612e4faf..d1573f0b00 100644 --- a/frigate/watchdog.py +++ b/frigate/watchdog.py @@ -2,10 +2,10 @@ import logging import threading import time +from multiprocessing.synchronize import Event as MpEvent from frigate.object_detection import ObjectDetectProcess from frigate.util import restart_frigate -from multiprocessing.synchronize import Event as MpEvent logger = logging.getLogger(__name__) diff --git a/migrations/001_create_events_table.py b/migrations/001_create_events_table.py index d9b0f08530..9e8ad1b607 100644 --- a/migrations/001_create_events_table.py +++ b/migrations/001_create_events_table.py @@ -21,14 +21,7 @@ """ -import datetime as dt import peewee as pw -from decimal import ROUND_HALF_EVEN - -try: - import playhouse.postgres_ext as pw_pext -except ImportError: - pass SQL = pw.SQL diff --git a/migrations/002_add_clip_snapshot.py b/migrations/002_add_clip_snapshot.py index aaa4a03c98..1431c9c85f 100644 --- a/migrations/002_add_clip_snapshot.py +++ b/migrations/002_add_clip_snapshot.py @@ -21,15 +21,9 @@ """ -import datetime as dt import peewee as pw -from decimal import ROUND_HALF_EVEN -from frigate.models import Event -try: - import playhouse.postgres_ext as pw_pext -except ImportError: - pass +from frigate.models import Event SQL = pw.SQL diff --git a/migrations/003_create_recordings_table.py b/migrations/003_create_recordings_table.py index e13ae657e4..82b15eb9cd 100644 --- a/migrations/003_create_recordings_table.py +++ b/migrations/003_create_recordings_table.py @@ -22,8 +22,6 @@ """ import peewee as pw -from frigate.models import Recordings - SQL = pw.SQL diff --git a/migrations/004_add_bbox_region_area.py b/migrations/004_add_bbox_region_area.py index bcc362aac2..da4ca7ac81 100644 --- a/migrations/004_add_bbox_region_area.py +++ b/migrations/004_add_bbox_region_area.py @@ -21,16 +21,10 @@ """ -import datetime as dt import peewee as pw -from playhouse.sqlite_ext import * -from decimal import ROUND_HALF_EVEN -from frigate.models import Event +from playhouse.sqlite_ext import JSONField -try: - import playhouse.postgres_ext as pw_pext -except ImportError: - pass +from frigate.models import Event SQL = pw.SQL diff --git a/migrations/005_make_end_time_nullable.py b/migrations/005_make_end_time_nullable.py index 5c8cf65054..87d0e3fd45 100644 --- a/migrations/005_make_end_time_nullable.py +++ b/migrations/005_make_end_time_nullable.py @@ -21,16 +21,9 @@ """ -import datetime as dt import peewee as pw -from playhouse.sqlite_ext import * -from decimal import ROUND_HALF_EVEN -from frigate.models import Event -try: - import playhouse.postgres_ext as pw_pext -except ImportError: - pass +from frigate.models import Event SQL = pw.SQL diff --git a/migrations/006_add_motion_active_objects.py b/migrations/006_add_motion_active_objects.py index 6bd564b8c9..2980b441db 100644 --- a/migrations/006_add_motion_active_objects.py +++ b/migrations/006_add_motion_active_objects.py @@ -21,16 +21,9 @@ """ -import datetime as dt import peewee as pw -from playhouse.sqlite_ext import * -from decimal import ROUND_HALF_EVEN -from frigate.models import Recordings -try: - import playhouse.postgres_ext as pw_pext -except ImportError: - pass +from frigate.models import Recordings SQL = pw.SQL diff --git a/migrations/007_add_retain_indefinitely.py b/migrations/007_add_retain_indefinitely.py index a46b72e291..cb5f9da920 100644 --- a/migrations/007_add_retain_indefinitely.py +++ b/migrations/007_add_retain_indefinitely.py @@ -21,16 +21,9 @@ """ -import datetime as dt import peewee as pw -from playhouse.sqlite_ext import * -from decimal import ROUND_HALF_EVEN -from frigate.models import Event -try: - import playhouse.postgres_ext as pw_pext -except ImportError: - pass +from frigate.models import Event SQL = pw.SQL diff --git a/migrations/008_add_sub_label.py b/migrations/008_add_sub_label.py index 00603c63e1..56c4bb75ad 100644 --- a/migrations/008_add_sub_label.py +++ b/migrations/008_add_sub_label.py @@ -21,16 +21,9 @@ """ -import datetime as dt import peewee as pw -from playhouse.sqlite_ext import * -from decimal import ROUND_HALF_EVEN -from frigate.models import Event -try: - import playhouse.postgres_ext as pw_pext -except ImportError: - pass +from frigate.models import Event SQL = pw.SQL diff --git a/migrations/009_add_object_filter_ratio.py b/migrations/009_add_object_filter_ratio.py index cc23e1d86b..e5a00683de 100644 --- a/migrations/009_add_object_filter_ratio.py +++ b/migrations/009_add_object_filter_ratio.py @@ -22,6 +22,7 @@ """ import peewee as pw + from frigate.models import Event SQL = pw.SQL diff --git a/migrations/010_add_plus_image_id.py b/migrations/010_add_plus_image_id.py index 0e239e58b9..6b8c7ccc61 100644 --- a/migrations/010_add_plus_image_id.py +++ b/migrations/010_add_plus_image_id.py @@ -21,16 +21,9 @@ """ -import datetime as dt import peewee as pw -from playhouse.sqlite_ext import * -from decimal import ROUND_HALF_EVEN -from frigate.models import Event -try: - import playhouse.postgres_ext as pw_pext -except ImportError: - pass +from frigate.models import Event SQL = pw.SQL diff --git a/migrations/012_add_segment_size.py b/migrations/012_add_segment_size.py index c5ac1ed93c..7a1c797362 100644 --- a/migrations/012_add_segment_size.py +++ b/migrations/012_add_segment_size.py @@ -21,16 +21,9 @@ """ -import datetime as dt import peewee as pw -from playhouse.sqlite_ext import * -from decimal import ROUND_HALF_EVEN -from frigate.models import Recordings -try: - import playhouse.postgres_ext as pw_pext -except ImportError: - pass +from frigate.models import Recordings SQL = pw.SQL diff --git a/migrations/013_create_timeline_table.py b/migrations/013_create_timeline_table.py index fea7867afe..b96822df16 100644 --- a/migrations/013_create_timeline_table.py +++ b/migrations/013_create_timeline_table.py @@ -21,16 +21,7 @@ """ -import datetime as dt import peewee as pw -from playhouse.sqlite_ext import * -from decimal import ROUND_HALF_EVEN -from frigate.models import Recordings - -try: - import playhouse.postgres_ext as pw_pext -except ImportError: - pass SQL = pw.SQL diff --git a/migrations/014_event_updates_for_fp.py b/migrations/014_event_updates_for_fp.py index 5d075596b4..caa609bfaf 100644 --- a/migrations/014_event_updates_for_fp.py +++ b/migrations/014_event_updates_for_fp.py @@ -21,16 +21,9 @@ """ -import datetime as dt import peewee as pw -from playhouse.sqlite_ext import * -from decimal import ROUND_HALF_EVEN -from frigate.models import Event -try: - import playhouse.postgres_ext as pw_pext -except ImportError: - pass +from frigate.models import Event SQL = pw.SQL diff --git a/migrations/015_event_refactor.py b/migrations/015_event_refactor.py index d8a8a387c0..1bcb9c510e 100644 --- a/migrations/015_event_refactor.py +++ b/migrations/015_event_refactor.py @@ -21,16 +21,10 @@ """ -import datetime as dt import peewee as pw -from playhouse.sqlite_ext import * -from decimal import ROUND_HALF_EVEN -from frigate.models import Event +from playhouse.sqlite_ext import JSONField -try: - import playhouse.postgres_ext as pw_pext -except ImportError: - pass +from frigate.models import Event SQL = pw.SQL diff --git a/migrations/016_sublabel_increase.py b/migrations/016_sublabel_increase.py index b46b9fc79b..536ea0a610 100644 --- a/migrations/016_sublabel_increase.py +++ b/migrations/016_sublabel_increase.py @@ -1,6 +1,5 @@ import peewee as pw -from playhouse.migrate import * -from playhouse.sqlite_ext import * + from frigate.models import Event diff --git a/process_clip.py b/process_clip.py index d8dabbeddd..e122ebf369 100644 --- a/process_clip.py +++ b/process_clip.py @@ -1,8 +1,4 @@ -import sys -from typing_extensions import runtime - -sys.path.append("/lab/frigate") - +import csv import json import logging import multiprocessing as mp @@ -11,21 +7,26 @@ import sys import click -import csv import cv2 import numpy as np -from frigate.config import FrigateConfig -from frigate.object_detection import LocalObjectDetector -from frigate.motion import MotionDetector -from frigate.object_processing import CameraState -from frigate.objects import ObjectTracker -from frigate.util import ( +sys.path.append("/lab/frigate") + +from frigate.config import FrigateConfig # noqa: E402 +from frigate.motion import MotionDetector # noqa: E402 +from frigate.object_detection import LocalObjectDetector # noqa: E402 +from frigate.object_processing import CameraState # noqa: E402 +from frigate.objects import ObjectTracker # noqa: E402 +from frigate.util import ( # noqa: E402 EventsPerSecond, SharedMemoryFrameManager, draw_box_with_label, ) -from frigate.video import capture_frames, process_frames, start_or_restart_ffmpeg +from frigate.video import ( # noqa: E402 + capture_frames, + process_frames, + start_or_restart_ffmpeg, +) logging.basicConfig() logging.root.setLevel(logging.DEBUG) From bc4071dde8380bb98b82bc54851215a9be6050e6 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Tue, 23 May 2023 03:49:39 +0200 Subject: [PATCH 6/7] treewide: apply black --- frigate/events/cleanup.py | 4 +++- frigate/object_processing.py | 6 +----- migrations/013_create_timeline_table.py | 12 +++++++++--- process_clip.py | 1 - 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/frigate/events/cleanup.py b/frigate/events/cleanup.py index 5afd285c49..ada45e6fa6 100644 --- a/frigate/events/cleanup.py +++ b/frigate/events/cleanup.py @@ -90,7 +90,9 @@ def expire(self, media_type: str) -> None: # loop over object types in db for event in distinct_labels: # get expiration time for this label - expire_days = retain_config.objects.get(event.label, retain_config.default) + expire_days = retain_config.objects.get( + event.label, retain_config.default + ) expire_after = ( datetime.datetime.now() - datetime.timedelta(days=expire_days) ).timestamp() diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 8c16ef64ed..f9046d0ab2 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -177,11 +177,7 @@ def update(self, current_frame_time, obj_data): return (thumb_update, significant_change) def to_dict(self, include_thumbnail: bool = False): - ( - self.thumbnail_data["frame_time"] - if self.thumbnail_data is not None - else 0.0 - ) + (self.thumbnail_data["frame_time"] if self.thumbnail_data is not None else 0.0) event = { "id": self.obj_data["id"], "camera": self.camera, diff --git a/migrations/013_create_timeline_table.py b/migrations/013_create_timeline_table.py index b96822df16..7a83d96c72 100644 --- a/migrations/013_create_timeline_table.py +++ b/migrations/013_create_timeline_table.py @@ -30,9 +30,15 @@ def migrate(migrator, database, fake=False, **kwargs): migrator.sql( 'CREATE TABLE IF NOT EXISTS "timeline" ("timestamp" DATETIME NOT NULL, "camera" VARCHAR(20) NOT NULL, "source" VARCHAR(20) NOT NULL, "source_id" VARCHAR(30), "class_type" VARCHAR(50) NOT NULL, "data" JSON)' ) - migrator.sql('CREATE INDEX IF NOT EXISTS "timeline_camera" ON "timeline" ("camera")') - migrator.sql('CREATE INDEX IF NOT EXISTS "timeline_source" ON "timeline" ("source")') - migrator.sql('CREATE INDEX IF NOT EXISTS "timeline_source_id" ON "timeline" ("source_id")') + migrator.sql( + 'CREATE INDEX IF NOT EXISTS "timeline_camera" ON "timeline" ("camera")' + ) + migrator.sql( + 'CREATE INDEX IF NOT EXISTS "timeline_source" ON "timeline" ("source")' + ) + migrator.sql( + 'CREATE INDEX IF NOT EXISTS "timeline_source_id" ON "timeline" ("source_id")' + ) def rollback(migrator, database, fake=False, **kwargs): diff --git a/process_clip.py b/process_clip.py index e122ebf369..9eb6f324b6 100644 --- a/process_clip.py +++ b/process_clip.py @@ -311,7 +311,6 @@ def process(path, label, output, debug_path): for result in results: if count == 0: - # Writing headers of CSV file header = ["file"] + list(result[1].keys()) csv_writer.writerow(header) From b50868bf3015176526d78c045c9c1303f395d63b Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Wed, 24 May 2023 01:58:01 +0200 Subject: [PATCH 7/7] treewide: make regex patterns raw strings This is necessary for escape sequences to be properly recognized. --- frigate/const.py | 6 +++--- frigate/util.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frigate/const.py b/frigate/const.py index cfcdc2a533..df50988537 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -13,9 +13,9 @@ # Regex Consts -REGEX_CAMERA_NAME = "^[a-zA-Z0-9_-]+$" -REGEX_RTSP_CAMERA_USER_PASS = ":\/\/[a-zA-Z0-9_-]+:[\S]+@" -REGEX_HTTP_CAMERA_USER_PASS = "user=[a-zA-Z0-9_-]+&password=[\S]+" +REGEX_CAMERA_NAME = r"^[a-zA-Z0-9_-]+$" +REGEX_RTSP_CAMERA_USER_PASS = r":\/\/[a-zA-Z0-9_-]+:[\S]+@" +REGEX_HTTP_CAMERA_USER_PASS = r"user=[a-zA-Z0-9_-]+&password=[\S]+" # Known Driver Names diff --git a/frigate/util.py b/frigate/util.py index 056c5eb1b2..e624e877ac 100755 --- a/frigate/util.py +++ b/frigate/util.py @@ -863,7 +863,7 @@ def get_bandwidth_stats() -> dict[str, dict]: stats = list(filter(lambda a: a != "", line.strip().split("\t"))) try: if re.search( - "(^ffmpeg|\/go2rtc|frigate\.detector\.[a-z]+)/([0-9]+)/", stats[0] + r"(^ffmpeg|\/go2rtc|frigate\.detector\.[a-z]+)/([0-9]+)/", stats[0] ): process = stats[0].split("/") usages[process[len(process) - 2]] = { @@ -930,7 +930,7 @@ def get_intel_gpu_stats() -> dict[str, str]: # render is used for qsv render = [] - for result in re.findall('"Render/3D/0":{[a-z":\d.,%]+}', reading): + for result in re.findall(r'"Render/3D/0":{[a-z":\d.,%]+}', reading): packet = json.loads(result[14:]) single = packet.get("busy", 0.0) render.append(float(single))