From fdd85bb5903750ac389c7b611c9d7616855e50af Mon Sep 17 00:00:00 2001 From: Odd Kiva <2375733-oddkiva@users.noreply.gitlab.com> Date: Wed, 27 Dec 2023 03:04:46 +0100 Subject: [PATCH 1/4] WIP: save work. --- .../sara/sfm/robust_global_translations.py | 11 -- ...nference_example.py => run_mobilenetv2.py} | 0 .../coreml/examples/run_yolov4_tiny.py | 124 ++++++++++++++++++ ...etect_object_with_yolov4_tiny_mlpackage.py | 23 ---- 4 files changed, 124 insertions(+), 34 deletions(-) delete mode 100644 python/oddkiva/sara/sfm/robust_global_translations.py rename python/oddkiva/shakti/inference/coreml/examples/{mobilenetv2_inference_example.py => run_mobilenetv2.py} (100%) create mode 100644 python/oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py delete mode 100644 python/oddkiva/shakti/inference/darknet/v4/examples/detect_object_with_yolov4_tiny_mlpackage.py diff --git a/python/oddkiva/sara/sfm/robust_global_translations.py b/python/oddkiva/sara/sfm/robust_global_translations.py deleted file mode 100644 index b38081673..000000000 --- a/python/oddkiva/sara/sfm/robust_global_translations.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Robust Global Translations with 1DSfM, Wilson, Snavely, ECCV 2014.""" -# Reference implementation available in GitHub. - -# Outlier removal using MFAS. - -# Kernel density estimation to sample projections. - -# Combinatorial cleaning: statistical accumulator, threshold tau. - -# Solving the translation problems. -# Levenberg-Marquadt using CERES. diff --git a/python/oddkiva/shakti/inference/coreml/examples/mobilenetv2_inference_example.py b/python/oddkiva/shakti/inference/coreml/examples/run_mobilenetv2.py similarity index 100% rename from python/oddkiva/shakti/inference/coreml/examples/mobilenetv2_inference_example.py rename to python/oddkiva/shakti/inference/coreml/examples/run_mobilenetv2.py diff --git a/python/oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py b/python/oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py new file mode 100644 index 000000000..73899c374 --- /dev/null +++ b/python/oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py @@ -0,0 +1,124 @@ +from collections import namedtuple +from pathlib import Path +from typing import Any + +from PIL import Image + +import numpy as np + +import coremltools as ct + +import oddkiva.shakti.inference.darknet as darknet + + +THIS_FILE = __file__ +SARA_SOURCE_DIR_PATH = Path(THIS_FILE[:THIS_FILE.find('sara') + len('sara')]) +SARA_DATA_DIR_PATH = SARA_SOURCE_DIR_PATH / 'data' +SARA_TRAINED_MODEL_DIR_PATH = SARA_SOURCE_DIR_PATH / 'trained_models' +SARA_YOLOV4_MODEL_DIR_PATH = SARA_SOURCE_DIR_PATH / 'trained_models' + +YOLO_V4_COREML_PATH = SARA_YOLOV4_MODEL_DIR_PATH / 'yolo-v4.mlpackage' +YOLO_V4_COCO_CLASSES_PATH = SARA_YOLOV4_MODEL_DIR_PATH / 'classes.txt' +assert YOLO_V4_COREML_PATH.exists() +YOLO_V4_CFG_PATH = SARA_YOLOV4_MODEL_DIR_PATH / 'yolov4.cfg' +assert YOLO_V4_CFG_PATH.exists() + +DOG_IMAGE_PATH = SARA_DATA_DIR_PATH / 'dog.jpg' +assert DOG_IMAGE_PATH.exists() + + +Box = namedtuple('x', 'y', 'w', 'h', 'p_object', 'class_id', 'p_class') + + +def get_yolo_boxes(yolo_out: np.ndarray, yolo_layers: dict['str': Any], + objectness_thres, + image_ori_sizes, yolo_input_sizes): + B = len(yolo_layers['masks']) + N, C, H, W = yolo_out.shape + out = yolo_out.reshape((N, B, C // B, H, W)) + rel_x = out[:, :, 0] + rel_y = out[:, :, 1] + log_w = out[:, :, 2] + log_h = out[:, :, 3] + p_objectness = out[:, :, 4] + p_classes = out[:, :, 5:] + + yi, xi = np.meshgrid(range(H), range(W), indexing='ij') + + mask = yolo_layers['mask'] + anchors = yolo_layers['anchors'] + w_prior = [anchors[2 * mask[b] + 0] for b in range(B)] + h_prior = [anchors[2 * mask[b] + 1] for b in range(B)] + + sx = yolo_input_sizes[0] / image_ori_sizes[0] + sy = yolo_input_sizes[1] / image_ori_sizes[1] + + x = (rel_x + xi) / W * image_ori_sizes[0] + y = (rel_y + yi) / H * image_ori_sizes[1] + for b in range(B): + w = np.exp(log_w)[:, :, b] * w_prior[b] * sx + h = np.exp(log_h)[:, :, b] * h_prior[b] * sy + + p_class_idx = np.argmax(p_classes, axis=2) + + # Get the 5D indices + object_ids = np.where(p_objectness > objectness_thres) + x = x[object_ids] + y = y[object_ids] + w = w[object_ids] + h = h[object_ids] + p_objectness = p_objectness[object_ids] + class_ids = p_class_idx[object_ids] + p_classes = p_classes[object_ids] + + boxes = np.stack((x, y, w, h, p_objectness, class_ids, p_classes)) + boxes = [Box(b) for b in boxes] + return boxes + +def nms(boxes_ndarr: [Box], iou_thres=0.4): + boxes_sorted = sorted(boxes_ndarr, + cmp=def compare(x, y): x.p_object > y.p_object) + boxes_filtered = [] + for box in boxes_sorted: + if not boxes_filtered: + boxes_filtered.append(box) + continue + x1 = np.array([box.x for box in boxes_filtered]) + y1 = np.array([box.y for box in boxes_filtered]) + w = np.array([box.w for box in boxes_filtered]) + h = np.array([box.h for box in boxes_filtered]) + x2 = x1 + w + y2 = y1 + h + + inter_x1 = np.maximum(x1, box.x) + inter_y1 = np.maximum(y1, box.y) + inter_x2 = np.minimum(x2, box.x + box.w) + inter_y2 = np.minimum(y2, box.y + box.h) + + inter = inter_x1 <= inter_x2 and inter_y1 <= inter_y2 + inter_area = \ + (inter_x2 - inter_x2) * (inter_y2 - inter_y1) * \ + inter.astype(np.float32) + + union_area = w * h + box.w * box.h - inter_area + + iou = inter_area / union_area + + ids = np.where(iou > iou_thres) + if not ids: + boxes_filtered.append(box) + + return boxes_filtered + + +yolo_model = ct.models.CompiledMLModel(str(YOLO_V4_COREML_PATH)) +yolo_cfg = darknet.Config() +yolo_cfg.read(YOLO_V4_CFG_PATH) +yolo_input_sizes = (yolo_cfg._metadata['width'], yolo_cfg._metadata['height']) +yolo_layers = [layer['yolo'] for layer in yolo_cfg._model + if 'yolo' in layer.keys()] + +image = Image.open(DOG_IMAGE_PATH).resize(yolo_input_sizes, + resample=Image.Resampling.LANCZOS) + +yolo_outs = model.predict({'image': image}) diff --git a/python/oddkiva/shakti/inference/darknet/v4/examples/detect_object_with_yolov4_tiny_mlpackage.py b/python/oddkiva/shakti/inference/darknet/v4/examples/detect_object_with_yolov4_tiny_mlpackage.py deleted file mode 100644 index 01f180090..000000000 --- a/python/oddkiva/shakti/inference/darknet/v4/examples/detect_object_with_yolov4_tiny_mlpackage.py +++ /dev/null @@ -1,23 +0,0 @@ -from pathlib import Path - -from PIL import Image - -import coremltools as ct - -YOLO_V4_ML_PATH = Path('/Users/oddkiva/Desktop/yolo-v4-tiny.mlpackage/') - -THIS_FILE = __file__ -SARA_SOURCE_DIR_PATH = Path(THIS_FILE[:THIS_FILE.find('sara') + len('sara')]) -SARA_DATA_DIR_PATH = SARA_SOURCE_DIR_PATH / 'data' -DOG_IMAGE_PATH = SARA_DATA_DIR_PATH / 'dog.jpg' - -assert YOLO_V4_ML_PATH.exists() -assert DOG_IMAGE_PATH.exists() - -model = ct.models.CompiledMLModel(str(YOLO_V4_ML_PATH)) - -image = Image.open(DOG_IMAGE_PATH).resize( - (416, 416), - resample=Image.Resampling.LANCZOS) - -yolo_boxes = model.predict({'image': image}) From 67784944683b1aef62838abeb3f57b2392de6267 Mon Sep 17 00:00:00 2001 From: Odd Kiva <2375733-oddkiva@users.noreply.gitlab.com> Date: Wed, 27 Dec 2023 13:07:51 +0100 Subject: [PATCH 2/4] WIP: save work. --- .../coreml/examples/run_yolov4_tiny.py | 69 +++++++++++-------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/python/oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py b/python/oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py index 73899c374..58c00bda6 100644 --- a/python/oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py +++ b/python/oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py @@ -15,27 +15,29 @@ SARA_SOURCE_DIR_PATH = Path(THIS_FILE[:THIS_FILE.find('sara') + len('sara')]) SARA_DATA_DIR_PATH = SARA_SOURCE_DIR_PATH / 'data' SARA_TRAINED_MODEL_DIR_PATH = SARA_SOURCE_DIR_PATH / 'trained_models' -SARA_YOLOV4_MODEL_DIR_PATH = SARA_SOURCE_DIR_PATH / 'trained_models' +SARA_YOLOV4_MODEL_DIR_PATH = SARA_TRAINED_MODEL_DIR_PATH / 'yolov4-tiny' -YOLO_V4_COREML_PATH = SARA_YOLOV4_MODEL_DIR_PATH / 'yolo-v4.mlpackage' +YOLO_V4_COREML_PATH = SARA_YOLOV4_MODEL_DIR_PATH / 'yolov4-tiny.mlpackage' YOLO_V4_COCO_CLASSES_PATH = SARA_YOLOV4_MODEL_DIR_PATH / 'classes.txt' assert YOLO_V4_COREML_PATH.exists() -YOLO_V4_CFG_PATH = SARA_YOLOV4_MODEL_DIR_PATH / 'yolov4.cfg' +YOLO_V4_CFG_PATH = SARA_YOLOV4_MODEL_DIR_PATH / 'yolov4-tiny.cfg' assert YOLO_V4_CFG_PATH.exists() DOG_IMAGE_PATH = SARA_DATA_DIR_PATH / 'dog.jpg' assert DOG_IMAGE_PATH.exists() -Box = namedtuple('x', 'y', 'w', 'h', 'p_object', 'class_id', 'p_class') +Box = namedtuple('Box', ['x', 'y', 'w', 'h', 'p_object', 'class_id', 'p_class']) def get_yolo_boxes(yolo_out: np.ndarray, yolo_layers: dict['str': Any], objectness_thres, image_ori_sizes, yolo_input_sizes): - B = len(yolo_layers['masks']) - N, C, H, W = yolo_out.shape - out = yolo_out.reshape((N, B, C // B, H, W)) + mask = yolo_layers['mask'] + anchors = yolo_layers['anchors'] + _, B, _, H, W = yolo_out.shape + + out = yolo_out rel_x = out[:, :, 0] rel_y = out[:, :, 1] log_w = out[:, :, 2] @@ -45,39 +47,43 @@ def get_yolo_boxes(yolo_out: np.ndarray, yolo_layers: dict['str': Any], yi, xi = np.meshgrid(range(H), range(W), indexing='ij') - mask = yolo_layers['mask'] - anchors = yolo_layers['anchors'] - w_prior = [anchors[2 * mask[b] + 0] for b in range(B)] - h_prior = [anchors[2 * mask[b] + 1] for b in range(B)] + w_prior = [anchors[mask[b]][0] for b in range(B)] + h_prior = [anchors[mask[b]][1] for b in range(B)] - sx = yolo_input_sizes[0] / image_ori_sizes[0] - sy = yolo_input_sizes[1] / image_ori_sizes[1] + sx = image_ori_sizes[1] / yolo_input_sizes[0] + sy = image_ori_sizes[0] / yolo_input_sizes[1] - x = (rel_x + xi) / W * image_ori_sizes[0] - y = (rel_y + yi) / H * image_ori_sizes[1] + x = (rel_x + xi) / W * image_ori_sizes[1] + y = (rel_y + yi) / H * image_ori_sizes[0] + w = log_w + h = log_h for b in range(B): - w = np.exp(log_w)[:, :, b] * w_prior[b] * sx - h = np.exp(log_h)[:, :, b] * h_prior[b] * sy + w[:, :, b] = np.exp(log_w)[:, :, b] * w_prior[b] * sx + h[:, :, b] = np.exp(log_h)[:, :, b] * h_prior[b] * sy p_class_idx = np.argmax(p_classes, axis=2) - # Get the 5D indices - object_ids = np.where(p_objectness > objectness_thres) + # Get the 4D indices + object_ids = np.nonzero(p_objectness > objectness_thres) x = x[object_ids] y = y[object_ids] w = w[object_ids] h = h[object_ids] p_objectness = p_objectness[object_ids] class_ids = p_class_idx[object_ids] - p_classes = p_classes[object_ids] + ixs = (object_ids[0], object_ids[1], class_ids, object_ids[2], + object_ids[3]) + p_classes = p_classes[ixs] - boxes = np.stack((x, y, w, h, p_objectness, class_ids, p_classes)) - boxes = [Box(b) for b in boxes] + boxes = np.stack((x, y, w, h, p_objectness, class_ids, + p_classes)).transpose().tolist() + boxes = [Box(*b) for b in boxes] return boxes def nms(boxes_ndarr: [Box], iou_thres=0.4): - boxes_sorted = sorted(boxes_ndarr, - cmp=def compare(x, y): x.p_object > y.p_object) + def compare(x: Box, y: Box): + return x.p_object > y.p_object + boxes_sorted = sorted(boxes_ndarr, cmp=compare) boxes_filtered = [] for box in boxes_sorted: if not boxes_filtered: @@ -118,7 +124,16 @@ def nms(boxes_ndarr: [Box], iou_thres=0.4): yolo_layers = [layer['yolo'] for layer in yolo_cfg._model if 'yolo' in layer.keys()] -image = Image.open(DOG_IMAGE_PATH).resize(yolo_input_sizes, - resample=Image.Resampling.LANCZOS) +image_ori = Image.open(DOG_IMAGE_PATH) +image_ori_sizes = np.asarray(image_ori).shape[:2] + +image_resized = image_ori.resize(yolo_input_sizes, + resample=Image.Resampling.LANCZOS) +yolo_outs = yolo_model.predict({'image': image_resized}) +yolo_outs = [yolo_outs[f'yolo_{i}'] for i in range(len(yolo_layers))] + + +yolo_boxes = get_yolo_boxes(yolo_outs[0], yolo_layers[0], 0.4, image_ori_sizes, + yolo_input_sizes) -yolo_outs = model.predict({'image': image}) +import IPython; IPython.embed() From b20bf95bf56660ec258f9d38f87c05d9059f9fc6 Mon Sep 17 00:00:00 2001 From: Odd Kiva <2375733-oddkiva@users.noreply.gitlab.com> Date: Wed, 27 Dec 2023 15:10:54 +0100 Subject: [PATCH 3/4] ENH: finish and fix implementation. --- .../coreml/examples/run_yolov4_tiny.py | 104 ++++++++++++------ 1 file changed, 72 insertions(+), 32 deletions(-) diff --git a/python/oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py b/python/oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py index 58c00bda6..9733c457e 100644 --- a/python/oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py +++ b/python/oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py @@ -8,6 +8,7 @@ import coremltools as ct +import oddkiva.sara as sara import oddkiva.shakti.inference.darknet as darknet @@ -50,25 +51,27 @@ def get_yolo_boxes(yolo_out: np.ndarray, yolo_layers: dict['str': Any], w_prior = [anchors[mask[b]][0] for b in range(B)] h_prior = [anchors[mask[b]][1] for b in range(B)] - sx = image_ori_sizes[1] / yolo_input_sizes[0] - sy = image_ori_sizes[0] / yolo_input_sizes[1] + sx = image_ori_sizes[1] / yolo_input_sizes[1] + sy = image_ori_sizes[0] / yolo_input_sizes[0] x = (rel_x + xi) / W * image_ori_sizes[1] y = (rel_y + yi) / H * image_ori_sizes[0] - w = log_w - h = log_h + w = np.copy(log_w) + h = np.copy(log_h) for b in range(B): - w[:, :, b] = np.exp(log_w)[:, :, b] * w_prior[b] * sx - h[:, :, b] = np.exp(log_h)[:, :, b] * h_prior[b] * sy + w[:, b] = np.exp(log_w)[:, b] * w_prior[b] * sx + h[:, b] = np.exp(log_h)[:, b] * h_prior[b] * sy p_class_idx = np.argmax(p_classes, axis=2) - # Get the 4D indices + # Get the 4D indices object_ids = np.nonzero(p_objectness > objectness_thres) x = x[object_ids] y = y[object_ids] w = w[object_ids] h = h[object_ids] + x -= 0.5 * w + y -= 0.5 * h p_objectness = p_objectness[object_ids] class_ids = p_class_idx[object_ids] ixs = (object_ids[0], object_ids[1], class_ids, object_ids[2], @@ -77,13 +80,15 @@ def get_yolo_boxes(yolo_out: np.ndarray, yolo_layers: dict['str': Any], boxes = np.stack((x, y, w, h, p_objectness, class_ids, p_classes)).transpose().tolist() - boxes = [Box(*b) for b in boxes] + boxes = [Box(*b) for b in boxes] return boxes -def nms(boxes_ndarr: [Box], iou_thres=0.4): +def nms(boxes: [Box], iou_thres=0.4): def compare(x: Box, y: Box): - return x.p_object > y.p_object - boxes_sorted = sorted(boxes_ndarr, cmp=compare) + return y.p_object - x.p_object + from functools import cmp_to_key + boxes_sorted = sorted(boxes, key=cmp_to_key(compare)) + boxes_filtered = [] for box in boxes_sorted: if not boxes_filtered: @@ -93,47 +98,82 @@ def compare(x: Box, y: Box): y1 = np.array([box.y for box in boxes_filtered]) w = np.array([box.w for box in boxes_filtered]) h = np.array([box.h for box in boxes_filtered]) - x2 = x1 + w - y2 = y1 + h + x2 = x1 + w + y2 = y1 + h inter_x1 = np.maximum(x1, box.x) inter_y1 = np.maximum(y1, box.y) inter_x2 = np.minimum(x2, box.x + box.w) inter_y2 = np.minimum(y2, box.y + box.h) - inter = inter_x1 <= inter_x2 and inter_y1 <= inter_y2 + inter = np.logical_and(inter_x1 <= inter_x2, inter_y1 <= inter_y2) inter_area = \ - (inter_x2 - inter_x2) * (inter_y2 - inter_y1) * \ + (inter_x2 - inter_x1) * (inter_y2 - inter_y1) * \ inter.astype(np.float32) union_area = w * h + box.w * box.h - inter_area iou = inter_area / union_area - ids = np.where(iou > iou_thres) - if not ids: + overlap = np.any(iou > iou_thres) + if not overlap: boxes_filtered.append(box) return boxes_filtered -yolo_model = ct.models.CompiledMLModel(str(YOLO_V4_COREML_PATH)) -yolo_cfg = darknet.Config() -yolo_cfg.read(YOLO_V4_CFG_PATH) -yolo_input_sizes = (yolo_cfg._metadata['width'], yolo_cfg._metadata['height']) -yolo_layers = [layer['yolo'] for layer in yolo_cfg._model - if 'yolo' in layer.keys()] +def detect(yolo_model, yolo_layers, image_ori, yolo_input_sizes): + image_ori_sizes = np.asarray(image_ori).shape[:2] + image_resized = image_ori.resize(yolo_input_sizes, + resample=Image.Resampling.LANCZOS) + + yolo_outs = yolo_model.predict({'image': image_resized}) + yolo_outs = [yolo_outs[f'yolo_{i}'] for i in range(len(yolo_layers))] + + yolo_boxes = [get_yolo_boxes(yolo_outs[i], yolo_layers[i], 0.4, + image_ori_sizes, yolo_input_sizes) + for i in range(len(yolo_layers))] + yolo_boxes = sum(yolo_boxes, []) + + yolo_boxes = nms(yolo_boxes) + + return yolo_boxes + + +def draw_detection( + b: Box, + class_name: str, + color: tuple[int, int, int], + font_size: int = 20 +) -> None: + + sara.draw_rect((b.x, b.y), (b.w, b.h), (255, 0, 0), 3) + sara.draw_text((b.x, b.y - 4), class_name, color, font_size, 0, False, True, False) + +def user_main(): + yolo_model = ct.models.CompiledMLModel(str(YOLO_V4_COREML_PATH)) + yolo_cfg = darknet.Config() + yolo_cfg.read(YOLO_V4_CFG_PATH) + yolo_input_sizes = (yolo_cfg._metadata['width'], yolo_cfg._metadata['height']) + yolo_layers = [layer['yolo'] for layer in yolo_cfg._model + if 'yolo' in layer.keys()] + with open(YOLO_V4_COCO_CLASSES_PATH, 'r') as fp: + yolo_classes = [l.strip(' \n') for l in fp.readlines() if l] + print(yolo_classes) + + image_ori = Image.open(DOG_IMAGE_PATH) -image_ori = Image.open(DOG_IMAGE_PATH) -image_ori_sizes = np.asarray(image_ori).shape[:2] + yolo_boxes = detect(yolo_model, yolo_layers, image_ori, yolo_input_sizes) -image_resized = image_ori.resize(yolo_input_sizes, - resample=Image.Resampling.LANCZOS) -yolo_outs = yolo_model.predict({'image': image_resized}) -yolo_outs = [yolo_outs[f'yolo_{i}'] for i in range(len(yolo_layers))] + sara.create_window(*image_ori.size) + sara.set_antialiasing(True) + sara.draw_image(np.asarray(image_ori)) + for b in yolo_boxes: + class_name = yolo_classes[int(b.class_id)] + draw_detection(b, class_name, (191, 0, 0)) + sara.get_key() -yolo_boxes = get_yolo_boxes(yolo_outs[0], yolo_layers[0], 0.4, image_ori_sizes, - yolo_input_sizes) -import IPython; IPython.embed() +if __name__ == '__main__': + sara.run_graphics(user_main) From 5ffdb241309f23a7efe421b5429765d7ca017bed Mon Sep 17 00:00:00 2001 From: Odd Kiva <2375733-oddkiva@users.noreply.gitlab.com> Date: Wed, 27 Dec 2023 15:46:34 +0100 Subject: [PATCH 4/4] MAINT: fix type. --- .../oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py b/python/oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py index 9733c457e..27cdbd20f 100644 --- a/python/oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py +++ b/python/oddkiva/shakti/inference/coreml/examples/run_yolov4_tiny.py @@ -83,7 +83,7 @@ def get_yolo_boxes(yolo_out: np.ndarray, yolo_layers: dict['str': Any], boxes = [Box(*b) for b in boxes] return boxes -def nms(boxes: [Box], iou_thres=0.4): +def nms(boxes: list[Box], iou_thres=0.4): def compare(x: Box, y: Box): return y.p_object - x.p_object from functools import cmp_to_key