Visualizer (#208)

* Added simple visualization

* Fixed visualizer

* Merge debugger and visualizer

* Merge debugger and visualizer

* Merge debugger and visualizer

---------

Co-authored-by: peter <[email protected]>
Co-authored-by: Pbatch <[email protected]>

.gitignore

@@ -7,4 +7,5 @@ build/
 __pycache__/
 *.pyc
 clashroyalebuildabot/debug
-clashroyalebuildabot/emulator/platform-tools
+clashroyalebuildabot/emulator/platform-tools
+env/

.pylintrc

@@ -573,7 +573,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=
+generated-members=cv2
 
 # Tells whether to warn about missing members when the owner of the attribute
 # is inferred to be None.

clashroyalebuildabot/__init__.py

@@ -1,6 +1,5 @@
 # Exports for clashroyalebuildabot
 from . import constants
-from . import debugger
 from .bot import Bot
 from .detectors import CardDetector
 from .detectors import Detector
@@ -13,9 +12,11 @@
 from .namespaces import Screens
 from .namespaces import State
 from .namespaces import Units
+from .visualizer import Visualizer
 
 __all__ = [
     "constants",
+    "Visualizer",
     "Cards",
     "Units",
     "State",

clashroyalebuildabot/bot/bot.py

@@ -25,13 +25,13 @@
 from clashroyalebuildabot.detectors.detector import Detector
 from clashroyalebuildabot.emulator.emulator import Emulator
 from clashroyalebuildabot.namespaces import Screens
+from clashroyalebuildabot.visualizer import Visualizer
 
 
 class Bot:
-    def __init__(self, actions, auto_start=True, debug=False):
+    def __init__(self, actions, auto_start=True):
         self.actions = actions
         self.auto_start = auto_start
-        self.debug = debug
 
         self._setup_logger()
 
@@ -40,8 +40,13 @@ def __init__(self, actions, auto_start=True, debug=False):
             raise ValueError(f"Must provide 8 cards but was given: {cards}")
         self.cards_to_actions = dict(zip(cards, actions))
 
-        self.detector = Detector(cards=cards, debug=self.debug)
-        self.emulator = Emulator()
+        config_path = os.path.join(SRC_DIR, "config.yaml")
+        with open(config_path, encoding="utf-8") as file:
+            config = yaml.safe_load(file)
+
+        self.visualizer = Visualizer(**config["visuals"])
+        self.emulator = Emulator(**config["adb"])
+        self.detector = Detector(cards=cards)
         self.state = None
 
     @staticmethod
@@ -111,6 +116,7 @@ def get_actions(self):
     def set_state(self):
         screenshot = self.emulator.take_screenshot()
         self.state = self.detector.run(screenshot)
+        self.visualizer.run(screenshot, self.state)
 
     def play_action(self, action):
         card_centre = self._get_card_centre(action.index)

clashroyalebuildabot/config.yaml

@@ -13,3 +13,8 @@ adb:
   # The serial number of your device
   # Use adb devices to obtain it
   device_serial: emulator-5554
+
+visuals:
+  save_labels: False
+  save_images: False
+  show_images: False

clashroyalebuildabot/detectors/detector.py

@@ -3,7 +3,6 @@
 from loguru import logger
 
 from clashroyalebuildabot.constants import MODELS_DIR
-from clashroyalebuildabot.debugger import Debugger
 from clashroyalebuildabot.detectors.card_detector import CardDetector
 from clashroyalebuildabot.detectors.number_detector import NumberDetector
 from clashroyalebuildabot.detectors.screen_detector import ScreenDetector
@@ -14,14 +13,13 @@
 class Detector:
     DECK_SIZE = 8
 
-    def __init__(self, cards, debug=False):
+    def __init__(self, cards):
         if len(cards) != self.DECK_SIZE:
             raise ValueError(
                 f"You must specify all {self.DECK_SIZE} of your cards"
             )
 
         self.cards = cards
-        self.debug = debug
 
         self.card_detector = CardDetector(self.cards)
         self.number_detector = NumberDetector(
@@ -32,10 +30,6 @@ def __init__(self, cards, debug=False):
         )
         self.screen_detector = ScreenDetector()
 
-        self.debugger = None
-        if self.debug:
-            self.debugger = Debugger()
-
     def run(self, image):
         logger.debug("Setting state...")
         cards, ready = self.card_detector.run(image)
@@ -44,7 +38,5 @@ def run(self, image):
         screen = self.screen_detector.run(image)
 
         state = State(allies, enemies, numbers, cards, ready, screen)
-        if self.debugger is not None:
-            self.debugger.run(image, state)
 
         return state

clashroyalebuildabot/emulator/emulator.py

@@ -14,14 +14,12 @@
 import av
 from loguru import logger
 import requests
-import yaml
 
 from clashroyalebuildabot.constants import ADB_DIR
 from clashroyalebuildabot.constants import ADB_PATH
 from clashroyalebuildabot.constants import EMULATOR_DIR
 from clashroyalebuildabot.constants import SCREENSHOT_HEIGHT
 from clashroyalebuildabot.constants import SCREENSHOT_WIDTH
-from clashroyalebuildabot.constants import SRC_DIR
 
 
 class KThread(threading.Thread):
@@ -98,13 +96,9 @@ def ignored(*exceptions):
 
 
 class Emulator:
-    def __init__(self):
-        config_path = os.path.join(SRC_DIR, "config.yaml")
-        with open(config_path, encoding="utf-8") as file:
-            config = yaml.safe_load(file)
-
-        adb_config = config["adb"]
-        self.serial, self.ip = [adb_config[s] for s in ["device_serial", "ip"]]
+    def __init__(self, device_serial, ip):
+        self.device_serial = device_serial
+        self.ip = ip
 
         self.video_socket = None
         self.screenshot_thread = None
@@ -195,7 +189,7 @@ def quit(self):
             kill_pid(self.scrcpy_proc.pid)
 
     def _run_command(self, command):
-        command = [ADB_PATH, "-s", self.serial, *command]
+        command = [ADB_PATH, "-s", self.device_serial, *command]
         logger.debug(" ".join(command))
         try:
             start_time = time.time()
@@ -239,7 +233,7 @@ def _start_scrcpy(self):
         command = [
             ADB_PATH,
             "-s",
-            self.serial,
+            self.device_serial,
             "shell",
             "CLASSPATH=/data/local/tmp/scrcpy-server.jar",
             "app_process",

clashroyalebuildabot/debugger.py → clashroyalebuildabot/visualizer.py

@@ -1,6 +1,9 @@
 from dataclasses import asdict
 import os
 
+import cv2
+from loguru import logger
+import numpy as np
 from PIL import ImageDraw
 from PIL import ImageFont
 
@@ -11,7 +14,7 @@
 from clashroyalebuildabot.namespaces.units import NAME2UNIT
 
 
-class Debugger:
+class Visualizer:
     _COLOUR_AND_RGBA = [
         ["navy", (0, 38, 63, 127)],
         ["blue", (0, 120, 210, 127)],
@@ -30,12 +33,21 @@ class Debugger:
         ["silver", (220, 220, 220, 127)],
     ]
 
-    def __init__(self):
-        os.makedirs(SCREENSHOTS_DIR, exist_ok=True)
-        os.makedirs(LABELS_DIR, exist_ok=True)
+    def __init__(self, save_labels, save_images, show_images):
+        self.save_labels = save_labels
+        self.save_images = save_images
+        self.show_images = show_images
+
         self.font = ImageFont.load_default()
         self.unit_names = [unit["name"] for unit in list(NAME2UNIT.values())]
 
+        os.makedirs(LABELS_DIR, exist_ok=True)
+        os.makedirs(SCREENSHOTS_DIR, exist_ok=True)
+
+        if self.show_images:
+            cv2.namedWindow("Visualizer", cv2.WINDOW_NORMAL)
+            logger.info("Visualizer initialized")
+
     @staticmethod
     def _write_label(image, state, basename):
         labels = []
@@ -75,7 +87,7 @@ def _draw_unit_bboxes(self, d, dets, prefix):
                 d, det.position.bbox, f"{prefix}_{det.unit.name}", rgba
             )
 
-    def _write_image(self, image, state, basename):
+    def _annotate_image(self, image, state):
         d = ImageDraw.Draw(image, "RGBA")
         for det in asdict(state.numbers).values():
             det = NumberDetection(**det)
@@ -89,12 +101,26 @@ def _write_image(self, image, state, basename):
             d.rectangle(tuple(position))
             self._draw_text(d, position, card.name)
 
-        image.save(os.path.join(SCREENSHOTS_DIR, f"{basename}.png"))
+        return image
 
     def run(self, image, state):
         n_screenshots = len(os.listdir(SCREENSHOTS_DIR))
         n_labels = len(os.listdir(LABELS_DIR))
         basename = max(n_labels, n_screenshots) + 1
 
-        self._write_image(image, state, basename)
-        self._write_label(image, state, basename)
+        if self.save_labels:
+            self._write_label(image, state, basename)
+
+        if not self.save_images and not self.show_images:
+            return
+
+        annotated_image = self._annotate_image(image, state)
+
+        if self.save_images:
+            annotated_image.save(
+                os.path.join(SCREENSHOTS_DIR, f"{basename}.png")
+            )
+
+        if self.show_images:
+            cv2.imshow("Visualizer", np.array(annotated_image)[..., ::-1])
+            cv2.waitKey(1)

main.py

@@ -41,7 +41,7 @@ def main():
         MinipekkaAction,
         MusketeerAction,
     }
-    bot = Bot(actions=actions, debug=False)
+    bot = Bot(actions=actions)
     bot.run()
 
 

pyproject.toml

@@ -33,6 +33,7 @@ dev = [
     "flake8==7.0.0",
     "isort==5.13.2",
     "pylint==3.1.0",
+    "opencv-python==4.10.0.84",
 ]
 
 [tool.black]