add saved_model inferencing script

inferencing/saved_model_config.py

@@ -0,0 +1,34 @@
+# Your Inference Config Class
+# Replace your own config
+# MY_INFERENCE_CONFIG = YOUR_CONFIG_CLASS
+import coco
+class InferenceConfig(coco.CocoConfig):
+    GPU_COUNT = 1
+    IMAGES_PER_GPU = 1
+coco_config = InferenceConfig()
+
+MY_INFERENCE_CONFIG = coco_config
+
+
+# Tensorflow Model server variable
+HOST_NO = 'localhost'
+PORT_NO = 8500
+MODEL_NAME = 'mask'
+
+
+# TF variable name
+OUTPUT_DETECTION = 'mrcnn_detection/Reshape_1'
+OUTPUT_CLASS = 'mrcnn_class/Reshape_1'
+OUTPUT_BBOX = 'mrcnn_bbox/Reshape'
+OUTPUT_MASK = 'mrcnn_mask/Reshape_1'
+INPUT_IMAGE = 'input_image'
+INPUT_IMAGE_META = 'input_image_meta'
+INPUT_ANCHORS = 'input_anchors'
+OUTPUT_NAME = 'predict_images'
+
+
+# Signature name
+SIGNATURE_NAME = 'serving_default'
+
+# GRPC config
+GRPC_MAX_RECEIVE_MESSAGE_LENGTH = 4096 * 4096 * 3 # Max LENGTH the GRPC should handle

inferencing/saved_model_inference.py

@@ -0,0 +1,75 @@
+import cv2, grpc, sys, os
+from tensorflow_serving.apis import prediction_service_pb2_grpc
+from tensorflow_serving.apis import predict_pb2
+import numpy as np
+import tensorflow as tf
+from inferencing import saved_model_config
+from inferencing.saved_model_preprocess import ForwardModel
+
+host = saved_model_config.HOST_NO
+port = saved_model_config.PORT_NO
+
+channel = grpc.insecure_channel(str(host) + ':' + str(port), options=[('grpc.max_receive_message_length', saved_model_config.GRPC_MAX_RECEIVE_MESSAGE_LENGTH)])
+
+stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
+
+
+request = predict_pb2.PredictRequest()
+request.model_spec.name = saved_model_config.MODEL_NAME
+request.model_spec.signature_name = saved_model_config.SIGNATURE_NAME
+
+model_config = saved_model_config.MY_INFERENCE_CONFIG
+preprocess_obj = ForwardModel(model_config)
+
+
+def detect_mask_single_image(image):
+    images = np.expand_dims(image, axis=0)
+    molded_images, image_metas, windows = preprocess_obj.mold_inputs(images)
+    molded_images = molded_images.astype(np.float32)
+    image_metas = image_metas.astype(np.float32)
+    # Validate image sizes
+    # All images in a batch MUST be of the same size
+    image_shape = molded_images[0].shape
+    for g in molded_images[1:]:
+        assert g.shape == image_shape, \
+            "After resizing, all images must have the same size. Check IMAGE_RESIZE_MODE and image sizes."
+
+    # Anchors
+    anchors = preprocess_obj.get_anchors(image_shape)
+    anchors = np.broadcast_to(anchors, (images.shape[0],) + anchors.shape)
+
+    request.inputs[saved_model_config.INPUT_IMAGE].CopyFrom(
+        tf.contrib.util.make_tensor_proto(molded_images, shape=molded_images.shape))
+    request.inputs[saved_model_config.INPUT_IMAGE_META].CopyFrom(
+        tf.contrib.util.make_tensor_proto(image_metas, shape=image_metas.shape))
+    request.inputs[saved_model_config.INPUT_ANCHORS].CopyFrom(
+        tf.contrib.util.make_tensor_proto(anchors, shape=anchors.shape))
+
+    result = stub.Predict(request, 60.)
+    result_dict = preprocess_obj.result_to_dict(images, molded_images, windows, result)[0]
+    return result_dict
+
+
+if __name__ == '__main__':
+    import argparse
+    import os
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-p', '--path', help='Path to Image', required=True)
+    args = vars(parser.parse_args())
+    image_path = args['path']
+
+    if not os.path.exists(image_path):
+        print(f"{image_path}  -- Does not exist")
+        exit()
+
+    image = cv2.imread(image_path)
+    if image is None:
+        print("Image path is not proper")
+        exit()
+
+    result = detect_mask_single_image(image)
+    print("*" * 60)
+    print("RESULTS:")
+    print(result)
+    print("*" * 60)
+

inferencing/saved_model_preprocess.py

@@ -0,0 +1,228 @@
+from inferencing import saved_model_config
+from inferencing import saved_model_utils
+import numpy as np
+import math
+
+
+def compose_image_meta(image_id, original_image_shape, image_shape,
+                       window, scale, active_class_ids):
+    """Takes attributes of an image and puts them in one 1D array.
+    image_id: An int ID of the image. Useful for debugging.
+    original_image_shape: [H, W, C] before resizing or padding.
+    image_shape: [H, W, C] after resizing and padding
+    window: (y1, x1, y2, x2) in pixels. The area of the image where the real
+            image is (excluding the padding)
+    scale: The scaling factor applied to the original image (float32)
+    active_class_ids: List of class_ids available in the dataset from which
+        the image came. Useful if training on images from multiple segmentation_datasets
+        where not all classes are present in all segmentation_datasets.
+    """
+    meta = np.array(
+        [image_id] +  # size=1
+        list(original_image_shape) +  # size=3
+        list(image_shape) +  # size=3
+        list(window) +  # size=4 (y1, x1, y2, x2) in image cooredinates
+        [scale] +  # size=1
+        list(active_class_ids)  # size=num_classes
+    )
+    return meta
+
+
+def mold_image(images, config):
+    """Expects an RGB image (or array of images) and subtracts
+    the mean pixel and converts it to float. Expects image
+    colors in RGB order.
+    """
+    return images.astype(np.float32) - config.MEAN_PIXEL
+
+
+def compute_backbone_shapes(config, image_shape):
+    """Computes the width and height of each stage of the backbone network.
+
+    Returns:
+        [N, (height, width)]. Where N is the number of stages
+    """
+    if callable(config.BACKBONE):
+        return config.COMPUTE_BACKBONE_SHAPE(image_shape)
+
+    # Currently supports ResNet only
+    assert config.BACKBONE in ["resnet50", "resnet101"]
+    return np.array(
+        [[int(math.ceil(image_shape[0] / stride)),
+            int(math.ceil(image_shape[1] / stride))]
+            for stride in config.BACKBONE_STRIDES])
+
+
+class ForwardModel:
+    def __init__(self, config):
+        self.config = config
+        self.outputs = {
+            'detection': 'mrcnn_detection/Reshape_1',
+            'class': 'mrcnn_class/Reshape_1',
+            'box': 'mrcnn_bbox/Reshape',
+            'mask': 'mrcnn_mask/Reshape_1'}
+
+        # self.build_outputs()
+
+    def mold_inputs(self, images):
+        """Takes a list of images and modifies them to the format expected
+        as an input to the neural network.
+        images: List of image matrices [height,width,depth]. Images can have
+            different sizes.
+
+        Returns 3 Numpy matrices:
+        molded_images: [N, h, w, 3]. Images resized and normalized.
+        image_metas: [N, length of meta data]. Details about each image.
+        windows: [N, (y1, x1, y2, x2)]. The portion of the image that has the
+            original image (padding excluded).
+        """
+        molded_images = []
+        image_metas = []
+        windows = []
+        for image in images:
+            # Resize image
+            # TODO: move resizing to mold_image()
+            molded_image, window, scale, padding, crop = saved_model_utils.resize_image(
+                image,
+                min_dim=self.config.IMAGE_MIN_DIM,
+                min_scale=self.config.IMAGE_MIN_SCALE,
+                max_dim=self.config.IMAGE_MAX_DIM,
+                mode=self.config.IMAGE_RESIZE_MODE)
+            molded_image = mold_image(molded_image, self.config)
+            # Build image_meta
+            image_meta = compose_image_meta(
+                0, image.shape, molded_image.shape, window, scale,
+                np.zeros([self.config.NUM_CLASSES], dtype=np.int32))
+            # Append
+            molded_images.append(molded_image)
+            windows.append(window)
+            image_metas.append(image_meta)
+        # Pack into arrays
+        molded_images = np.stack(molded_images)
+        image_metas = np.stack(image_metas)
+        windows = np.stack(windows)
+        return molded_images, image_metas, windows
+
+    def get_anchors(self, image_shape):
+        """Returns anchor pyramid for the given image size."""
+        backbone_shapes = compute_backbone_shapes(self.config, image_shape)
+        # Cache anchors and reuse if image shape is the same
+        if not hasattr(self, "_anchor_cache"):
+            self._anchor_cache = {}
+        if not tuple(image_shape) in self._anchor_cache:
+            # Generate Anchors
+            a = saved_model_utils.generate_pyramid_anchors(
+                self.config.RPN_ANCHOR_SCALES,
+                self.config.RPN_ANCHOR_RATIOS,
+                backbone_shapes,
+                self.config.BACKBONE_STRIDES,
+                self.config.RPN_ANCHOR_STRIDE)
+            # Keep a copy of the latest anchors in pixel coordinates because
+            # it's used in inspect_model notebooks.
+            # TODO: Remove this after the notebook are refactored to not use it
+            self.anchors = a
+            # Normalize coordinates
+            self._anchor_cache[tuple(image_shape)] = saved_model_utils.norm_boxes(a, image_shape[:2])
+        return self._anchor_cache[tuple(image_shape)]
+
+    def unmold_detections(self, detections, mrcnn_mask, original_image_shape,
+                          image_shape, window):
+        """Reformats the detections of one image from the format of the neural
+        network output to a format suitable for use in the rest of the
+        application.
+
+        detections: [N, (y1, x1, y2, x2, class_id, score)] in normalized coordinates
+        mrcnn_mask: [N, height, width, num_classes]
+        original_image_shape: [H, W, C] Original image shape before resizing
+        image_shape: [H, W, C] Shape of the image after resizing and padding
+        window: [y1, x1, y2, x2] Pixel coordinates of box in the image where the real
+                image is excluding the padding.
+
+        Returns:
+        boxes: [N, (y1, x1, y2, x2)] Bounding boxes in pixels
+        class_ids: [N] Integer class IDs for each bounding box
+        scores: [N] Float probability scores of the class_id
+        masks: [height, width, num_instances] Instance masks
+        """
+        # How many detections do we have?
+        # Detections array is padded with zeros. Find the first class_id == 0.
+        zero_ix = np.where(detections[:, 4] == 0)[0]
+        N = zero_ix[0] if zero_ix.shape[0] > 0 else detections.shape[0]
+
+        # Extract boxes, class_ids, scores, and class-specific masks
+        boxes = detections[:N, :4]
+        class_ids = detections[:N, 4].astype(np.int32)
+        scores = detections[:N, 5]
+        masks = mrcnn_mask[np.arange(N), :, :, class_ids]
+
+        # Translate normalized coordinates in the resized image to pixel
+        # coordinates in the original image before resizing
+        window = saved_model_utils.norm_boxes(window, image_shape[:2])
+        wy1, wx1, wy2, wx2 = window
+        shift = np.array([wy1, wx1, wy1, wx1])
+        wh = wy2 - wy1  # window height
+        ww = wx2 - wx1  # window width
+        scale = np.array([wh, ww, wh, ww])
+        # Convert boxes to normalized coordinates on the window
+        boxes = np.divide(boxes - shift, scale)
+        # Convert boxes to pixel coordinates on the original image
+        boxes = saved_model_utils.denorm_boxes(boxes, original_image_shape[:2])
+
+        # Filter out detections with zero area. Happens in early training when
+        # network weights are still random
+        exclude_ix = np.where(
+            (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) <= 0)[0]
+        if exclude_ix.shape[0] > 0:
+            boxes = np.delete(boxes, exclude_ix, axis=0)
+            class_ids = np.delete(class_ids, exclude_ix, axis=0)
+            scores = np.delete(scores, exclude_ix, axis=0)
+            masks = np.delete(masks, exclude_ix, axis=0)
+            N = class_ids.shape[0]
+
+        # Resize masks to original image size and set boundary threshold.
+        full_masks = []
+        for i in range(N):
+            # Convert neural network mask to full size mask
+            full_mask = saved_model_utils.unmold_mask(masks[i], boxes[i], original_image_shape)
+            full_masks.append(full_mask)
+        full_masks = np.stack(full_masks, axis=-1)\
+            if full_masks else np.empty(original_image_shape[:2] + (0,))
+
+        return boxes, class_ids, scores, full_masks
+
+    def format_output(self, result_dict):
+        mask_shape = result_dict.outputs[saved_model_config.OUTPUT_MASK].tensor_shape.dim
+        mask_shape = tuple(d.size for d in mask_shape)
+        mask = np.array(result_dict.outputs[saved_model_config.OUTPUT_MASK].float_val)
+        mask = np.reshape(mask, mask_shape)
+
+        detection_shape = result_dict.outputs[saved_model_config.OUTPUT_DETECTION].tensor_shape.dim
+        detection_shape = tuple(d.size for d in detection_shape)
+        detection = np.array(result_dict.outputs[saved_model_config.OUTPUT_DETECTION].float_val)
+        detection = np.reshape(detection, detection_shape)
+
+        result_dict = {'detection': detection, 'mask': mask}
+
+        return result_dict
+
+    def result_to_dict(self, images, molded_images, windows, result_dict):
+        result_dict = self.format_output(result_dict)
+        results = []
+        for i, image in enumerate(images):
+            # print('detection len',len(result_dict['detection']))
+            # print('mask len ',len(result_dict['mask']))
+            final_rois, final_class_ids, final_scores, final_masks = \
+                self.unmold_detections(result_dict['detection'][i], result_dict['mask'][i],
+                                       image.shape, molded_images[i].shape,
+                                       windows[i])
+            results.append({
+                "rois": final_rois,
+                "class": final_class_ids,
+                "scores": final_scores,
+                "mask": final_masks,
+            })
+            # print('rois:', final_rois.shape)
+            # print('class:', final_class_ids.shape)
+            # print('scores:', final_scores.shape)
+            # print('final mask shaoe:', final_masks.shape)
+        return results