diff --git a/README.md b/README.md index cb89d5f..5c0d7f0 100644 --- a/README.md +++ b/README.md @@ -1 +1,24 @@ -# alpr-jetson \ No newline at end of file +# ALPR-IoT + +ALPR using Nvidia Jetson Nano and Raspberry Pi for (2147336) Internet of Things + +## Components +--- +- Nvidia Jetson Nano +- STM32F411RE +- Raspberry Pi4 x 2 +- Azure VM +- Nvidia CUDA GPU (Tensorflow Serving) + +## Models +--- +- TensorRT SSD MobilenetV2 +- OpenALPR +- TessaractOCR +- License Plate Detector + +## System Diagram +--- +![](docs/iot.jpg) + + diff --git a/docs/iot.jpg b/docs/iot.jpg new file mode 100644 index 0000000..014fdc1 Binary files /dev/null and b/docs/iot.jpg differ diff --git a/raspberry/example.yaml b/raspberry/example.yaml index 76c72c8..a884dc8 100644 --- a/raspberry/example.yaml +++ b/raspberry/example.yaml @@ -1,4 +1,5 @@ default: url : token : - tf_serving_url : \ No newline at end of file + tf_serving_url : + azure_url: \ No newline at end of file diff --git a/raspberry/server.py b/raspberry/server.py index bf363be..36f319f 100644 --- a/raspberry/server.py +++ b/raspberry/server.py @@ -1,32 +1,37 @@ import threading +import cv2 +import requests +import shutil import matplotlib.pyplot as plt + +from dynaconf import settings from datetime import datetime -import cv2 from flask import Response, Flask, request from imutils.video import FPS from imutils.video import WebcamVideoStream -from utils import line_notify,get_plate_rest,lp_mapping +from utils import line_notify global video_frame global fps_i +global thread_lock + fps_i = None video_frame = None - -global thread_lock thread_lock = threading.Lock() +TF_SERVING=settings.TF_SERVING_URL +AZURE=settings.AZURE_URL + app = Flask(__name__) @app.route("/" ,methods = ['POST']) def notify(): file = request.files['media'] lp=request.form['lp'] file.save('tmp.jpg') - img = cv2.imread('tmp.jpg') - img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)/255 - th = threading.Thread(target=post_process,args=(lp,img,'tmp.jpg')) + th = threading.Thread(target=post_process,args=(lp,'tmp.jpg')) th.start() return '200 OK' @@ -34,29 +39,22 @@ def notify(): def streamFrames(): return Response(encodeFrame(), mimetype = "multipart/x-mixed-replace; boundary=frame") -@app.route("/open") -def open_gate(): - print('open gate') - return '200 OK' - -def post_process(lp,img,img_path): - try: - lp_img, cors = get_plate_rest(img_path) - except: - lp_img = None - +def post_process(lp,img_path): if len(lp)==0: - msg="Open Gate: http://guyzsarun.southeastasia.cloudapp.azure.com:5000/open \nLP : Not detected" - else: - msg="Open Gate: http://guyzsarun.southeastasia.cloudapp.azure.com:5000/open \nLP : "+lp - - if lp_img is None: - line_notify(msg,img_path,False) + msg="Open Gate: "+AZURE+"\nLP : Not detected" else: - lp_map_img=lp_mapping(img,lp_img[0]) - plt.imsave('lp.jpg',lp_map_img) - line_notify(msg,'lp.jpg',False) - + msg="Open Gate: "+AZURE+"\nLP : "+lp + try: + r=requests.post(TF_SERVING,files={'media':open(img_path,'rb')}) + if r.status_code==404: + line_notify(msg,img_path,False) + else: + with open('lp.jpg', 'wb') as f: + f.write(r.content) + line_notify(msg,'lp.jpg',False) + except: + print("Error sending request") + pass def encodeFrame(): global thread_lock @@ -71,6 +69,7 @@ def encodeFrame(): if fps_i is None: fps_i=0 + #Detection Zone cv2.rectangle(video_frame,(30,30),(610,450),(0,0,255),2) if fps_i: cv2.putText(video_frame, "FPS: {:.2f}".format(fps_i), (25, 25) , cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,0,255)) @@ -79,11 +78,9 @@ def encodeFrame(): if not return_key: continue - # Output image as a byte array yield(b'--frame\r\n' b'Content-Type: image/jpeg\r\n\r\n' + bytearray(encoded_image) + b'\r\n') - def captureFrames(): global video_frame, thread_lock, fps_i diff --git a/raspberry/utils.py b/raspberry/utils.py index a427834..6398fa6 100644 --- a/raspberry/utils.py +++ b/raspberry/utils.py @@ -9,254 +9,15 @@ from datetime import datetime import numpy as np - URL=settings.URL TOKEN=settings.TOKEN -TF_SERVING=settings.TF_SERVING_URL - -bf=cv2.BFMatcher() -sift=cv2.xfeatures2d.SIFT_create() - -def lp_mapping(img,lp): - lp=np.float32(lp) - img=np.float32(img) - - lp_gray=cv2.cvtColor(np.uint8(lp*255),cv2.COLOR_RGB2GRAY) - img_gray=cv2.cvtColor(np.uint8(img*255),cv2.COLOR_RGB2GRAY) - kp1,des1=sift.detectAndCompute(lp_gray,None) - kp2,des2=sift.detectAndCompute(img_gray,None) - matches = bf.knnMatch(des1,des2, k=2) - - # Apply ratio test - good = [] - for match1,match2 in matches: - #Less distance good match - if match1.distance < 0.70*match2.distance: - good.append([match1]) - - return cv2.drawMatchesKnn(np.uint8(lp*255),kp1,np.uint8(img*255),kp2,good,None,flags=2)/255 +headers = {"Authorization": "Bearer " + TOKEN} def line_notify(msg, payload=None, notifyDisable=False): - headers = {"Authorization": "Bearer " + TOKEN} if msg is None: msg = datetime.now() msg = {"message": msg, "notificationDisabled": notifyDisable} if payload != None: payload = {"imageFile": open(payload, "rb")} - return requests.post(URL, headers=headers, params=msg, files=payload) - -def get_plate_rest(image_path, Dmax=608, Dmin=256): - vehicle = preprocess_image(image_path) - ratio = float(max(vehicle.shape[:2])) / min(vehicle.shape[:2]) - side = int(ratio * Dmin) - bound_dim = min(side, Dmax) - _ , LpImg, _, cor = detect_lp(None, vehicle, bound_dim, lp_threshold=0.5) - return LpImg, cor - -def preprocess_image(image_path,resize=False): - img = cv2.imread(image_path) - img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) - img = img / 255 - if resize: - img = cv2.resize(img, (224,224)) - return img - -def detect_lp(model, I, max_dim, lp_threshold): - min_dim_img = min(I.shape[:2]) - factor = float(max_dim) / min_dim_img - w, h = (np.array(I.shape[1::-1], dtype=float) * factor).astype(int).tolist() - Iresized = cv2.resize(I, (w, h)) - T = Iresized.copy() - T = T.reshape((1, T.shape[0], T.shape[1], T.shape[2])) - data = json.dumps({ - "instances": T.tolist() - }) - r=requests.post(TF_SERVING,data=data) - Yr = np.squeeze(np.array(r.json()['predictions'])) - L, TLp, lp_type, Cor = reconstruct(I, Iresized, Yr, lp_threshold) - return L, TLp, lp_type, Cor - -class Label: - def __init__(self, cl=-1, tl=np.array([0., 0.]), br=np.array([0., 0.]), prob=None): - self.__tl = tl - self.__br = br - self.__cl = cl - self.__prob = prob - - def __str__(self): - return 'Class: %d, top left(x: %f, y: %f), bottom right(x: %f, y: %f)' % ( - self.__cl, self.__tl[0], self.__tl[1], self.__br[0], self.__br[1]) - - def copy(self): - return Label(self.__cl, self.__tl, self.__br) - - def wh(self): return self.__br - self.__tl - - def cc(self): return self.__tl + self.wh() / 2 - - def tl(self): return self.__tl - - def br(self): return self.__br - - def tr(self): return np.array([self.__br[0], self.__tl[1]]) - - def bl(self): return np.array([self.__tl[0], self.__br[1]]) - - def cl(self): return self.__cl - - def area(self): return np.prod(self.wh()) - - def prob(self): return self.__prob - - def set_class(self, cl): - self.__cl = cl - - def set_tl(self, tl): - self.__tl = tl - - def set_br(self, br): - self.__br = br - - def set_wh(self, wh): - cc = self.cc() - self.__tl = cc - .5 * wh - self.__br = cc + .5 * wh - - def set_prob(self, prob): - self.__prob = prob - -class DLabel(Label): - def __init__(self, cl, pts, prob): - self.pts = pts - tl = np.amin(pts, axis=1) - br = np.amax(pts, axis=1) - Label.__init__(self, cl, tl, br, prob) - -def getWH(shape): - return np.array(shape[1::-1]).astype(float) - -def IOU(tl1, br1, tl2, br2): - wh1, wh2 = br1-tl1, br2-tl2 - assert((wh1 >= 0).all() and (wh2 >= 0).all()) - - intersection_wh = np.maximum(np.minimum(br1, br2) - np.maximum(tl1, tl2), 0) - intersection_area = np.prod(intersection_wh) - area1, area2 = (np.prod(wh1), np.prod(wh2)) - union_area = area1 + area2 - intersection_area - return intersection_area/union_area - -def IOU_labels(l1, l2): - return IOU(l1.tl(), l1.br(), l2.tl(), l2.br()) - -def nms(Labels, iou_threshold=0.5): - SelectedLabels = [] - Labels.sort(key=lambda l: l.prob(), reverse=True) - - for label in Labels: - non_overlap = True - for sel_label in SelectedLabels: - if IOU_labels(label, sel_label) > iou_threshold: - non_overlap = False - break - - if non_overlap: - SelectedLabels.append(label) - return SelectedLabels - - - -def find_T_matrix(pts, t_pts): - A = np.zeros((8, 9)) - for i in range(0, 4): - xi = pts[:, i] - xil = t_pts[:, i] - xi = xi.T - - A[i*2, 3:6] = -xil[2]*xi - A[i*2, 6:] = xil[1]*xi - A[i*2+1, :3] = xil[2]*xi - A[i*2+1, 6:] = -xil[0]*xi - - [U, S, V] = np.linalg.svd(A) - H = V[-1, :].reshape((3, 3)) - return H - -def getRectPts(tlx, tly, brx, bry): - return np.matrix([[tlx, brx, brx, tlx], [tly, tly, bry, bry], [1, 1, 1, 1]], dtype=float) - -def normal(pts, side, mn, MN): - pts_MN_center_mn = pts * side - pts_MN = pts_MN_center_mn + mn.reshape((2, 1)) - pts_prop = pts_MN / MN.reshape((2, 1)) - return pts_prop - -# Reconstruction function from predict value into plate crpoped from image -def reconstruct(I, Iresized, Yr, lp_threshold): - # 4 max-pooling layers, stride = 2 - net_stride = 2**4 - side = ((208 + 40)/2)/net_stride - - # one line and two lines license plate size - one_line = (470, 110) - two_lines = (280, 200) - - Probs = Yr[..., 0] - Affines = Yr[..., 2:] - - xx, yy = np.where(Probs > lp_threshold) - # CNN input image size - WH = getWH(Iresized.shape) - # output feature map size - MN = WH/net_stride - - vxx = vyy = 0.5 #alpha - base = lambda vx, vy: np.matrix([[-vx, -vy, 1], [vx, -vy, 1], [vx, vy, 1], [-vx, vy, 1]]).T - labels = [] - labels_frontal = [] - - for i in range(len(xx)): - x, y = xx[i], yy[i] - affine = Affines[x, y] - prob = Probs[x, y] - - mn = np.array([float(y) + 0.5, float(x) + 0.5]) - - # affine transformation matrix - A = np.reshape(affine, (2, 3)) - A[0, 0] = max(A[0, 0], 0) - A[1, 1] = max(A[1, 1], 0) - # identity transformation - B = np.zeros((2, 3)) - B[0, 0] = max(A[0, 0], 0) - B[1, 1] = max(A[1, 1], 0) - - pts = np.array(A*base(vxx, vyy)) - pts_frontal = np.array(B*base(vxx, vyy)) - - pts_prop = normal(pts, side, mn, MN) - frontal = normal(pts_frontal, side, mn, MN) - - labels.append(DLabel(0, pts_prop, prob)) - labels_frontal.append(DLabel(0, frontal, prob)) - - final_labels = nms(labels, 0.1) - final_labels_frontal = nms(labels_frontal, 0.1) - - assert final_labels_frontal, "No License plate is founded!" - - out_size, lp_type = (two_lines, 2) if ((final_labels_frontal[0].wh()[0] / final_labels_frontal[0].wh()[1]) < 1.7) else (one_line, 1) - - TLp = [] - Cor = [] - if len(final_labels): - final_labels.sort(key=lambda x: x.prob(), reverse=True) - for _, label in enumerate(final_labels): - t_ptsh = getRectPts(0, 0, out_size[0], out_size[1]) - ptsh = np.concatenate((label.pts * getWH(I.shape).reshape((2, 1)), np.ones((1, 4)))) - H = find_T_matrix(ptsh, t_ptsh) - Ilp = cv2.warpPerspective(I, H, out_size, borderValue=0) - TLp.append(Ilp) - Cor.append(ptsh) - return final_labels, TLp, lp_type, Cor - + return requests.post(URL, headers=headers, params=msg, files=payload) \ No newline at end of file diff --git a/tf-serving/Tensorflow-serving.ipynb b/tf-serving/Tensorflow-serving.ipynb deleted file mode 100644 index 5f13639..0000000 --- a/tf-serving/Tensorflow-serving.ipynb +++ /dev/null @@ -1,482 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "import json\n", - "import cv2\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "from local_utils import detect_lp\n", - "from os.path import splitext,basename\n", - "from tensorflow.keras.models import model_from_json\n", - "import glob\n", - "import time\n", - "import numpy as np\n", - "import grpc" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "def load_model(path):\n", - " try:\n", - " path = splitext(path)[0]\n", - " with open('%s.json' % path, 'r') as json_file:\n", - " model_json = json_file.read()\n", - " model = model_from_json(model_json, custom_objects={})\n", - " model.load_weights('%s.h5' % path)\n", - " print(\"Loading model successfully...\")\n", - " return model\n", - " except Exception as e:\n", - " print(e)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "# pylint: disable=invalid-name, redefined-outer-name, missing-docstring, non-parent-init-called, trailing-whitespace, line-too-long\n", - "import cv2\n", - "import numpy as np\n", - "\n", - "\n", - "class Label:\n", - " def __init__(self, cl=-1, tl=np.array([0., 0.]), br=np.array([0., 0.]), prob=None):\n", - " self.__tl = tl\n", - " self.__br = br\n", - " self.__cl = cl\n", - " self.__prob = prob\n", - "\n", - " def __str__(self):\n", - " return 'Class: %d, top left(x: %f, y: %f), bottom right(x: %f, y: %f)' % (\n", - " self.__cl, self.__tl[0], self.__tl[1], self.__br[0], self.__br[1])\n", - "\n", - " def copy(self):\n", - " return Label(self.__cl, self.__tl, self.__br)\n", - "\n", - " def wh(self): return self.__br - self.__tl\n", - "\n", - " def cc(self): return self.__tl + self.wh() / 2\n", - "\n", - " def tl(self): return self.__tl\n", - "\n", - " def br(self): return self.__br\n", - "\n", - " def tr(self): return np.array([self.__br[0], self.__tl[1]])\n", - "\n", - " def bl(self): return np.array([self.__tl[0], self.__br[1]])\n", - "\n", - " def cl(self): return self.__cl\n", - "\n", - " def area(self): return np.prod(self.wh())\n", - "\n", - " def prob(self): return self.__prob\n", - "\n", - " def set_class(self, cl):\n", - " self.__cl = cl\n", - "\n", - " def set_tl(self, tl):\n", - " self.__tl = tl\n", - "\n", - " def set_br(self, br):\n", - " self.__br = br\n", - "\n", - " def set_wh(self, wh):\n", - " cc = self.cc()\n", - " self.__tl = cc - .5 * wh\n", - " self.__br = cc + .5 * wh\n", - "\n", - " def set_prob(self, prob):\n", - " self.__prob = prob\n", - "\n", - "class DLabel(Label):\n", - " def __init__(self, cl, pts, prob):\n", - " self.pts = pts\n", - " tl = np.amin(pts, axis=1)\n", - " br = np.amax(pts, axis=1)\n", - " Label.__init__(self, cl, tl, br, prob)\n", - "\n", - "def getWH(shape):\n", - " return np.array(shape[1::-1]).astype(float)\n", - "\n", - "def IOU(tl1, br1, tl2, br2):\n", - " wh1, wh2 = br1-tl1, br2-tl2\n", - " assert((wh1 >= 0).all() and (wh2 >= 0).all())\n", - " \n", - " intersection_wh = np.maximum(np.minimum(br1, br2) - np.maximum(tl1, tl2), 0)\n", - " intersection_area = np.prod(intersection_wh)\n", - " area1, area2 = (np.prod(wh1), np.prod(wh2))\n", - " union_area = area1 + area2 - intersection_area\n", - " return intersection_area/union_area\n", - "\n", - "def IOU_labels(l1, l2):\n", - " return IOU(l1.tl(), l1.br(), l2.tl(), l2.br())\n", - "\n", - "def nms(Labels, iou_threshold=0.5):\n", - " SelectedLabels = []\n", - " Labels.sort(key=lambda l: l.prob(), reverse=True)\n", - " \n", - " for label in Labels:\n", - " non_overlap = True\n", - " for sel_label in SelectedLabels:\n", - " if IOU_labels(label, sel_label) > iou_threshold:\n", - " non_overlap = False\n", - " break\n", - "\n", - " if non_overlap:\n", - " SelectedLabels.append(label)\n", - " return SelectedLabels\n", - "\n", - "\n", - "\n", - "def find_T_matrix(pts, t_pts):\n", - " A = np.zeros((8, 9))\n", - " for i in range(0, 4):\n", - " xi = pts[:, i]\n", - " xil = t_pts[:, i]\n", - " xi = xi.T\n", - " \n", - " A[i*2, 3:6] = -xil[2]*xi\n", - " A[i*2, 6:] = xil[1]*xi\n", - " A[i*2+1, :3] = xil[2]*xi\n", - " A[i*2+1, 6:] = -xil[0]*xi\n", - "\n", - " [U, S, V] = np.linalg.svd(A)\n", - " H = V[-1, :].reshape((3, 3))\n", - " return H\n", - "\n", - "def getRectPts(tlx, tly, brx, bry):\n", - " return np.matrix([[tlx, brx, brx, tlx], [tly, tly, bry, bry], [1, 1, 1, 1]], dtype=float)\n", - "\n", - "def normal(pts, side, mn, MN):\n", - " pts_MN_center_mn = pts * side\n", - " pts_MN = pts_MN_center_mn + mn.reshape((2, 1))\n", - " pts_prop = pts_MN / MN.reshape((2, 1))\n", - " return pts_prop\n", - "\n", - "# Reconstruction function from predict value into plate crpoped from image\n", - "def reconstruct(I, Iresized, Yr, lp_threshold):\n", - " # 4 max-pooling layers, stride = 2\n", - " net_stride = 2**4\n", - " side = ((208 + 40)/2)/net_stride\n", - "\n", - " # one line and two lines license plate size\n", - " one_line = (470, 110)\n", - " two_lines = (280, 200)\n", - "\n", - " Probs = Yr[..., 0]\n", - " Affines = Yr[..., 2:]\n", - "\n", - " xx, yy = np.where(Probs > lp_threshold)\n", - " # CNN input image size \n", - " WH = getWH(Iresized.shape)\n", - " # output feature map size\n", - " MN = WH/net_stride\n", - "\n", - " vxx = vyy = 0.5 #alpha\n", - " base = lambda vx, vy: np.matrix([[-vx, -vy, 1], [vx, -vy, 1], [vx, vy, 1], [-vx, vy, 1]]).T\n", - " labels = []\n", - " labels_frontal = []\n", - "\n", - " for i in range(len(xx)):\n", - " x, y = xx[i], yy[i]\n", - " affine = Affines[x, y]\n", - " prob = Probs[x, y]\n", - "\n", - " mn = np.array([float(y) + 0.5, float(x) + 0.5])\n", - "\n", - " # affine transformation matrix\n", - " A = np.reshape(affine, (2, 3))\n", - " A[0, 0] = max(A[0, 0], 0)\n", - " A[1, 1] = max(A[1, 1], 0)\n", - " # identity transformation\n", - " B = np.zeros((2, 3))\n", - " B[0, 0] = max(A[0, 0], 0)\n", - " B[1, 1] = max(A[1, 1], 0)\n", - "\n", - " pts = np.array(A*base(vxx, vyy))\n", - " pts_frontal = np.array(B*base(vxx, vyy))\n", - "\n", - " pts_prop = normal(pts, side, mn, MN)\n", - " frontal = normal(pts_frontal, side, mn, MN)\n", - "\n", - " labels.append(DLabel(0, pts_prop, prob))\n", - " labels_frontal.append(DLabel(0, frontal, prob))\n", - " \n", - " final_labels = nms(labels, 0.1)\n", - " final_labels_frontal = nms(labels_frontal, 0.1)\n", - "\n", - " #print(final_labels_frontal)\n", - " assert final_labels_frontal, \"No License plate is founded!\"\n", - "\n", - " # LP size and type\n", - " out_size, lp_type = (two_lines, 2) if ((final_labels_frontal[0].wh()[0] / final_labels_frontal[0].wh()[1]) < 1.7) else (one_line, 1)\n", - "\n", - " TLp = []\n", - " Cor = []\n", - " if len(final_labels):\n", - " final_labels.sort(key=lambda x: x.prob(), reverse=True)\n", - " for _, label in enumerate(final_labels):\n", - " t_ptsh = getRectPts(0, 0, out_size[0], out_size[1])\n", - " ptsh = np.concatenate((label.pts * getWH(I.shape).reshape((2, 1)), np.ones((1, 4))))\n", - " H = find_T_matrix(ptsh, t_ptsh)\n", - " Ilp = cv2.warpPerspective(I, H, out_size, borderValue=0)\n", - " TLp.append(Ilp)\n", - " Cor.append(ptsh)\n", - " return final_labels, TLp, lp_type, Cor\n", - "\n", - "def detect_lp2(model, I, max_dim, lp_threshold):\n", - " min_dim_img = min(I.shape[:2])\n", - " factor = float(max_dim) / min_dim_img\n", - " w, h = (np.array(I.shape[1::-1], dtype=float) * factor).astype(int).tolist()\n", - " Iresized = cv2.resize(I, (w, h))\n", - " T = Iresized.copy()\n", - " T = T.reshape((1, T.shape[0], T.shape[1], T.shape[2]))\n", - " data = json.dumps({ \n", - " \"instances\": T.tolist()\n", - " })\n", - " r=requests.post('http://172.28.97.235:8501/v1/models/lp:predict',data=data)\n", - " Yr = np.squeeze(np.array(r.json()['predictions']))\n", - " #print(Yr.shape)\n", - " L, TLp, lp_type, Cor = reconstruct(I, Iresized, Yr, lp_threshold)\n", - " return L, TLp, lp_type, Cor\n", - "\n", - "def detect_lp(model, I, max_dim, lp_threshold):\n", - " min_dim_img = min(I.shape[:2])\n", - " factor = float(max_dim) / min_dim_img\n", - " w, h = (np.array(I.shape[1::-1], dtype=float) * factor).astype(int).tolist()\n", - " Iresized = cv2.resize(I, (w, h))\n", - " T = Iresized.copy()\n", - " T = T.reshape((1, T.shape[0], T.shape[1], T.shape[2]))\n", - " Yr = model.predict(T)\n", - " Yr = np.squeeze(Yr)\n", - " #print(Yr.shape)\n", - " L, TLp, lp_type, Cor = reconstruct(I, Iresized, Yr, lp_threshold)\n", - " return L, TLp, lp_type, Cor" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "def preprocess_image(image_path,resize=False):\n", - " img = cv2.imread(image_path)\n", - " img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n", - " img = img / 255\n", - " if resize:\n", - " img = cv2.resize(img, (224,224))\n", - " return img" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [], - "source": [ - "def get_plate(image_path, Dmax=608, Dmin=256):\n", - " vehicle = preprocess_image(image_path)\n", - " ratio = float(max(vehicle.shape[:2])) / min(vehicle.shape[:2])\n", - " side = int(ratio * Dmin)\n", - " bound_dim = min(side, Dmax)\n", - " _ , LpImg, _, cor = detect_lp(wpod_net, vehicle, bound_dim, lp_threshold=0.5)\n", - " return LpImg, cor" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "def get_plate_rest(image_path, Dmax=608, Dmin=256):\n", - " vehicle = preprocess_image(image_path)\n", - " ratio = float(max(vehicle.shape[:2])) / min(vehicle.shape[:2])\n", - " side = int(ratio * Dmin)\n", - " bound_dim = min(side, Dmax)\n", - " _ , LpImg, _, cor = detect_lp2(None, vehicle, bound_dim, lp_threshold=0.5)\n", - " return LpImg, cor" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "#get_plate('./Car-Number-Plate.jpg')" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": { - "collapsed": true, - "jupyter": { - "outputs_hidden": true - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "([array([[[0.78324142, 0.81412377, 0.86485907],\n", - " [0.78913909, 0.82002145, 0.87075674],\n", - " [0.794447 , 0.82532935, 0.87606464],\n", - " ...,\n", - " [0.92977941, 0.94436275, 0.95281863],\n", - " [0.929856 , 0.94375 , 0.95013787],\n", - " [0.92553615, 0.93922335, 0.94499081]],\n", - " \n", - " [[0.76238894, 0.79057521, 0.85074678],\n", - " [0.76365273, 0.791839 , 0.85201057],\n", - " [0.76479013, 0.79297641, 0.85314798],\n", - " ...,\n", - " [0.95770144, 0.97172181, 0.97848882],\n", - " [0.96830959, 0.98121936, 0.98465456],\n", - " [0.96894914, 0.98130362, 0.98307292]],\n", - " \n", - " [[0.77068015, 0.80009191, 0.85401348],\n", - " [0.77095588, 0.80036765, 0.85428922],\n", - " [0.77123162, 0.80064338, 0.85456495],\n", - " ...,\n", - " [0.88835784, 0.90266544, 0.91029412],\n", - " [0.93137255, 0.94485294, 0.95 ],\n", - " [0.9669424 , 0.97977941, 0.98299632]],\n", - " \n", - " ...,\n", - " \n", - " [[0.19338235, 0.20686275, 0.21936275],\n", - " [0.35453431, 0.37107843, 0.38419118],\n", - " [0.49957108, 0.51887255, 0.53253676],\n", - " ...,\n", - " [0.2372549 , 0.25686275, 0.27205882],\n", - " [0.23823529, 0.25845588, 0.27426471],\n", - " [0.26029412, 0.28327206, 0.29963235]],\n", - " \n", - " [[0.17402344, 0.1845435 , 0.19921492],\n", - " [0.33222656, 0.34707414, 0.36193704],\n", - " [0.47460938, 0.49335172, 0.50838695],\n", - " ...,\n", - " [0.24692862, 0.26653646, 0.28129213],\n", - " [0.25533088, 0.27528722, 0.29090839],\n", - " [0.29558824, 0.31868107, 0.3337163 ]],\n", - " \n", - " [[0.16478248, 0.17602635, 0.19171262],\n", - " [0.28817402, 0.30300245, 0.31868873],\n", - " [0.41156556, 0.42997855, 0.44566483],\n", - " ...,\n", - " [0.27079504, 0.29040288, 0.30445772],\n", - " [0.28823529, 0.30784314, 0.32279412],\n", - " [0.33386949, 0.35678615, 0.37084099]]])],\n", - " [array([[261.13157812, 398.38468585, 395.5580104 , 258.30490267],\n", - " [282.54823849, 285.58059036, 323.55479667, 320.5224448 ],\n", - " [ 1. , 1. , 1. , 1. ]])])" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "get_plate_rest('Car-Number-Plate.jpg')" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.10043734073638916\n" - ] - } - ], - "source": [ - "tic=time.time()\n", - "for i in range(100):\n", - " get_plate('./Plate_examples/china_car_plate.jpg')\n", - "toc=time.time()\n", - "\n", - "print((toc-tic)/100)" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.7757337093353271\n" - ] - } - ], - "source": [ - "tic=time.time()\n", - "for i in range(10):\n", - " get_plate_rest('Car-Number-Plate.jpg')\n", - "toc=time.time()\n", - "\n", - "print((toc-tic)/10)" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "metadata": {}, - "outputs": [], - "source": [ - "r=requests.post('http://127.0.0.1:8501/v1/models/my_model:predict',data=data)" - ] - }, - { - "cell_type": "code", - "execution_count": 96, - "metadata": {}, - "outputs": [], - "source": [ - "ret=np.array(r.json()['predictions'])" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/tf-serving/destination.png b/tf-serving/destination.png new file mode 100644 index 0000000..2744013 Binary files /dev/null and b/tf-serving/destination.png differ diff --git a/tf-serving/docker-compose.yml b/tf-serving/docker-compose.yml index cd6d777..5ae5142 100644 --- a/tf-serving/docker-compose.yml +++ b/tf-serving/docker-compose.yml @@ -1,10 +1,11 @@ -version: "3.2" +version: "2.4" services: tf-serving: image: tensorflow/serving:latest-gpu container_name: tf-serving restart: on-failure + runtime: nvidia volumes: - ./models/lp:/models/lp expose: @@ -12,4 +13,8 @@ services: ports: - 8501:8501 environment: - - MODEL_NAME=lp \ No newline at end of file + - MODEL_NAME=lp + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=all + command: + - '--per_process_gpu_memory_fraction=0.3' diff --git a/tf-serving/main.py b/tf-serving/main.py new file mode 100644 index 0000000..89a3d7f --- /dev/null +++ b/tf-serving/main.py @@ -0,0 +1,28 @@ +import cv2 +import matplotlib.pyplot as plt +import shutil +import uvicorn + +from fastapi import FastAPI,UploadFile,File,Response,status +from utils import get_plate_rest,lp_mapping +from fastapi.responses import FileResponse + +app = FastAPI() + +@app.post("/predict",status_code=200) +async def read_root(response: Response,media: UploadFile=File(...)): + with open("img.jpg", "wb") as buffer: + shutil.copyfileobj(media.file, buffer) + try: + lp_img, cors = get_plate_rest("img.jpg") + except: + response.status_code = 404 + return + img=cv2.cvtColor(cv2.imread('img.jpg'),cv2.COLOR_BGR2RGB)/255 + lp_map_img=lp_mapping(img,lp_img[0]) + plt.imsave('lp.jpg',lp_map_img) + + return FileResponse('lp.jpg') + +if __name__ == '__main__': + uvicorn.run("main:app",host="0.0.0.0",port=8080) \ No newline at end of file diff --git a/tf-serving/models/keras.h5 b/tf-serving/models/keras.h5 new file mode 100644 index 0000000..44de5db Binary files /dev/null and b/tf-serving/models/keras.h5 differ diff --git a/tf-serving/utils.py b/tf-serving/utils.py new file mode 100644 index 0000000..0ff5782 --- /dev/null +++ b/tf-serving/utils.py @@ -0,0 +1,250 @@ +import os +import sys +import json +import time + +from dynaconf import settings +import cv2 +import requests +from datetime import datetime +import numpy as np + +TF_SERVING=settings.TF_SERVING_URL + +bf=cv2.BFMatcher() +sift=cv2.xfeatures2d.SIFT_create() + +def lp_mapping(img,lp): + lp=np.float32(lp) + img=np.float32(img) + + lp_gray=cv2.cvtColor(np.uint8(lp*255),cv2.COLOR_RGB2GRAY) + img_gray=cv2.cvtColor(np.uint8(img*255),cv2.COLOR_RGB2GRAY) + + kp1,des1=sift.detectAndCompute(lp_gray,None) + kp2,des2=sift.detectAndCompute(img_gray,None) + matches = bf.knnMatch(des1,des2, k=2) + + # Apply ratio test + good = [] + for match1,match2 in matches: + #Less distance good match + if match1.distance < 0.70*match2.distance: + good.append([match1]) + + return cv2.drawMatchesKnn(np.uint8(lp*255),kp1,np.uint8(img*255),kp2,good,None,flags=2)/255 + +def get_plate_rest(image_path, Dmax=608, Dmin=256): + vehicle = preprocess_image(image_path) + ratio = float(max(vehicle.shape[:2])) / min(vehicle.shape[:2]) + side = int(ratio * Dmin) + bound_dim = min(side, Dmax) + _ , LpImg, _, cor = detect_lp(None, vehicle, bound_dim, lp_threshold=0.5) + return LpImg, cor + +def preprocess_image(image_path,resize=False): + img = cv2.imread(image_path) + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + img = img / 255 + if resize: + img = cv2.resize(img, (224,224)) + return img + +def detect_lp(model, I, max_dim, lp_threshold): + min_dim_img = min(I.shape[:2]) + factor = float(max_dim) / min_dim_img + w, h = (np.array(I.shape[1::-1], dtype=float) * factor).astype(int).tolist() + Iresized = cv2.resize(I, (w, h)) + T = Iresized.copy() + T = T.reshape((1, T.shape[0], T.shape[1], T.shape[2])) + data = json.dumps({ + "instances": T.tolist() + }) + r=requests.post(TF_SERVING,data=data) + Yr = np.squeeze(np.array(r.json()['predictions'])) + L, TLp, lp_type, Cor = reconstruct(I, Iresized, Yr, lp_threshold) + return L, TLp, lp_type, Cor + +class Label: + def __init__(self, cl=-1, tl=np.array([0., 0.]), br=np.array([0., 0.]), prob=None): + self.__tl = tl + self.__br = br + self.__cl = cl + self.__prob = prob + + def __str__(self): + return 'Class: %d, top left(x: %f, y: %f), bottom right(x: %f, y: %f)' % ( + self.__cl, self.__tl[0], self.__tl[1], self.__br[0], self.__br[1]) + + def copy(self): + return Label(self.__cl, self.__tl, self.__br) + + def wh(self): return self.__br - self.__tl + + def cc(self): return self.__tl + self.wh() / 2 + + def tl(self): return self.__tl + + def br(self): return self.__br + + def tr(self): return np.array([self.__br[0], self.__tl[1]]) + + def bl(self): return np.array([self.__tl[0], self.__br[1]]) + + def cl(self): return self.__cl + + def area(self): return np.prod(self.wh()) + + def prob(self): return self.__prob + + def set_class(self, cl): + self.__cl = cl + + def set_tl(self, tl): + self.__tl = tl + + def set_br(self, br): + self.__br = br + + def set_wh(self, wh): + cc = self.cc() + self.__tl = cc - .5 * wh + self.__br = cc + .5 * wh + + def set_prob(self, prob): + self.__prob = prob + +class DLabel(Label): + def __init__(self, cl, pts, prob): + self.pts = pts + tl = np.amin(pts, axis=1) + br = np.amax(pts, axis=1) + Label.__init__(self, cl, tl, br, prob) + +def getWH(shape): + return np.array(shape[1::-1]).astype(float) + +def IOU(tl1, br1, tl2, br2): + wh1, wh2 = br1-tl1, br2-tl2 + assert((wh1 >= 0).all() and (wh2 >= 0).all()) + + intersection_wh = np.maximum(np.minimum(br1, br2) - np.maximum(tl1, tl2), 0) + intersection_area = np.prod(intersection_wh) + area1, area2 = (np.prod(wh1), np.prod(wh2)) + union_area = area1 + area2 - intersection_area + return intersection_area/union_area + +def IOU_labels(l1, l2): + return IOU(l1.tl(), l1.br(), l2.tl(), l2.br()) + +def nms(Labels, iou_threshold=0.5): + SelectedLabels = [] + Labels.sort(key=lambda l: l.prob(), reverse=True) + + for label in Labels: + non_overlap = True + for sel_label in SelectedLabels: + if IOU_labels(label, sel_label) > iou_threshold: + non_overlap = False + break + + if non_overlap: + SelectedLabels.append(label) + return SelectedLabels + + + +def find_T_matrix(pts, t_pts): + A = np.zeros((8, 9)) + for i in range(0, 4): + xi = pts[:, i] + xil = t_pts[:, i] + xi = xi.T + + A[i*2, 3:6] = -xil[2]*xi + A[i*2, 6:] = xil[1]*xi + A[i*2+1, :3] = xil[2]*xi + A[i*2+1, 6:] = -xil[0]*xi + + [U, S, V] = np.linalg.svd(A) + H = V[-1, :].reshape((3, 3)) + return H + +def getRectPts(tlx, tly, brx, bry): + return np.matrix([[tlx, brx, brx, tlx], [tly, tly, bry, bry], [1, 1, 1, 1]], dtype=float) + +def normal(pts, side, mn, MN): + pts_MN_center_mn = pts * side + pts_MN = pts_MN_center_mn + mn.reshape((2, 1)) + pts_prop = pts_MN / MN.reshape((2, 1)) + return pts_prop + +# Reconstruction function from predict value into plate crpoped from image +def reconstruct(I, Iresized, Yr, lp_threshold): + # 4 max-pooling layers, stride = 2 + net_stride = 2**4 + side = ((208 + 40)/2)/net_stride + + # one line and two lines license plate size + one_line = (470, 110) + two_lines = (280, 200) + + Probs = Yr[..., 0] + Affines = Yr[..., 2:] + + xx, yy = np.where(Probs > lp_threshold) + # CNN input image size + WH = getWH(Iresized.shape) + # output feature map size + MN = WH/net_stride + + vxx = vyy = 0.5 #alpha + base = lambda vx, vy: np.matrix([[-vx, -vy, 1], [vx, -vy, 1], [vx, vy, 1], [-vx, vy, 1]]).T + labels = [] + labels_frontal = [] + + for i in range(len(xx)): + x, y = xx[i], yy[i] + affine = Affines[x, y] + prob = Probs[x, y] + + mn = np.array([float(y) + 0.5, float(x) + 0.5]) + + # affine transformation matrix + A = np.reshape(affine, (2, 3)) + A[0, 0] = max(A[0, 0], 0) + A[1, 1] = max(A[1, 1], 0) + # identity transformation + B = np.zeros((2, 3)) + B[0, 0] = max(A[0, 0], 0) + B[1, 1] = max(A[1, 1], 0) + + pts = np.array(A*base(vxx, vyy)) + pts_frontal = np.array(B*base(vxx, vyy)) + + pts_prop = normal(pts, side, mn, MN) + frontal = normal(pts_frontal, side, mn, MN) + + labels.append(DLabel(0, pts_prop, prob)) + labels_frontal.append(DLabel(0, frontal, prob)) + + final_labels = nms(labels, 0.1) + final_labels_frontal = nms(labels_frontal, 0.1) + + assert final_labels_frontal, "No License plate is founded!" + + out_size, lp_type = (two_lines, 2) if ((final_labels_frontal[0].wh()[0] / final_labels_frontal[0].wh()[1]) < 1.7) else (one_line, 1) + + TLp = [] + Cor = [] + if len(final_labels): + final_labels.sort(key=lambda x: x.prob(), reverse=True) + for _, label in enumerate(final_labels): + t_ptsh = getRectPts(0, 0, out_size[0], out_size[1]) + ptsh = np.concatenate((label.pts * getWH(I.shape).reshape((2, 1)), np.ones((1, 4)))) + H = find_T_matrix(ptsh, t_ptsh) + Ilp = cv2.warpPerspective(I, H, out_size, borderValue=0) + TLp.append(Ilp) + Cor.append(ptsh) + return final_labels, TLp, lp_type, Cor + diff --git a/thread-test.py b/thread-test.py deleted file mode 100644 index 414fbf5..0000000 --- a/thread-test.py +++ /dev/null @@ -1,49 +0,0 @@ -import time - -import threading -import cv2 -''' -A Dummy function that accepts 2 arguments i.e. Filename and encryption type -and sleeps for 5 seconds in a loop while printing few lines. -This is to simulate a heavey function that takes 10 seconds to complete -''' -def loadContents(fileName, encryptionType): - print('Started loading contents from file : ', fileName) - print('Encryption Type : ', encryptionType) - for i in range(5): - print('Loading ... ') - time.sleep(1) - print('Finished loading contents from file : ', fileName) - -def check_thread_alive(thr): - thr.join(timeout=0.0) - return thr.is_alive() - -def main(): - cap = cv2.VideoCapture(0) - # Create a thread from a function with arguments - th = threading.Thread(target=loadContents, args=('users.csv','ABC' )) - # Start the thread - # print some lines in main thread - th.start() - while(1): - ret_val, img = cap.read() - - if not check_thread_alive(th): - print('THread DEAD') - th = threading.Thread(target=loadContents, args=('users.csv','ABC' )) - th.start() - else: - pass - - cv2.imshow("CSI Camera", img) - if cv2.waitKey(1) & 0xFF == ord('q'): - break - - # When everything done, release the capture - cap.release() - cv2.destroyAllWindows() - # Wait for thread to finish - -if __name__ == '__main__': - main() \ No newline at end of file