updated color map

README.md

@@ -21,14 +21,16 @@ The results have only 3 classes marked on them.
  - vehicle (Blue)
 
 ## Video results
-[Part 1](https://youtu.be/D68uL3aEu5o)
-[Part 2](https://youtu.be/YwalCv13E0Q)
+[Part 1](https://youtu.be/ehy4yoVLuvM)
+[Part 2](https://youtu.be/Q7Z_4USTNuU)
+[Part 3](https://youtu.be/x5JNB3NrNfY)
+[Part 4](https://youtu.be/YwalCv13E0Q)
 
 ## Image results
-![image 1](outputs/results/1264.png)
-![image 1](outputs/results/1356.png)
-![image 1](outputs/results/1351.png)
-![image 1](outputs/results/1743.png)
+![image 1](outputs2/1.png)
+![image 1](outputs2/3.png)
+![image 1](outputs2/2.png)
+![image 1](outputs2/4.png)
 
 ## To-do
 - [ ] Hyperparameter tuning

inference.ipynb

@@ -0,0 +1,344 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import random\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "from tensorflow.python.keras import backend as K\n",
+    "from tensorflow.python.keras.layers import Input, Conv2D, MaxPooling2D, concatenate, UpSampling2D\n",
+    "from tensorflow.python.keras.optimizers import Adadelta, Nadam\n",
+    "from tensorflow.python.keras.models import Model, load_model\n",
+    "from tensorflow.python.keras.utils import multi_gpu_model, plot_model\n",
+    "from tensorflow.python.keras.callbacks import TensorBoard, ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\n",
+    "from tensorflow.python.keras.preprocessing import image\n",
+    "import tensorflow as tf\n",
+    "from tensorflow.python.keras.losses import binary_crossentropy\n",
+    "import cv2\n",
+    "from multiclassunet import Unet\n",
+    "from dilatednet import DilatedNet\n",
+    "import tqdm\n",
+    "import time\n",
+    "from tensorflow.python.keras.utils import Sequence\n",
+    "from moviepy.editor import VideoFileClip, ImageSequenceClip"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "WARNING:tensorflow:From /home/mia/tf/lib/python3.6/site-packages/tensorflow/python/keras/initializers.py:104: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with distribution=normal is deprecated and will be removed in a future version.\n",
+      "Instructions for updating:\n",
+      "`normal` is a deprecated alias for `truncated_normal`\n"
+     ]
+    }
+   ],
+   "source": [
+    "unet = Unet(256, 256, 8, 64)\n",
+    "# unet = DilatedNet(256, 256, 8, True, True)\n",
+    "# unet.load_weights('dilated.h5')\n",
+    "p_unet = multi_gpu_model(unet, 4)\n",
+    "# unet.load_weights('unet1m.h5')\n",
+    "p_unet.load_weights('models-dr/unet50k.h5')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# s = time.time()\n",
+    "# for i in tqdm.tqdm(range(len(image_list))):\n",
+    "#     img = image.load_img(f'{test_dir}/{image_list[i]}')\n",
+    "#     x = image.img_to_array(img)/255.\n",
+    "#     img = image.img_to_array(img)\n",
+    "#     _img = img.copy()\n",
+    "#     dims = x.shape\n",
+    "#     x = cv2.resize(x, (256, 256))\n",
+    "    \n",
+    "#     z = unet.predict(np.expand_dims(x, axis=0))\n",
+    "#     z = np.squeeze(z)\n",
+    "#     z = z.reshape(256, 256, 8)\n",
+    "#     z = cv2.resize(z, (dims[1], dims[0]))\n",
+    "#     y = np.argmax(z, axis=2)\n",
+    "#     y = np.argmax(z, axis=2)\n",
+    "    \n",
+    "#     construction = np.zeros_like(y)\n",
+    "#     human = np.zeros_like(y)\n",
+    "#     vehicle = np.zeros_like(y)\n",
+    "    \n",
+    "#     construction[y==2] = 255.\n",
+    "#     human[y==6] = 255.\n",
+    "#     vehicle[y==7] = 255.\n",
+    "\n",
+    "# #     fig = plt.figure(figsize=(40, 10))\n",
+    "# #     ax = fig.add_subplot(1,2,1)\n",
+    "# #     ax.set_title(f'Image')\n",
+    "# #     ax.imshow(img/255.)\n",
+    "    \n",
+    "#     result = img.copy()\n",
+    "#     alpha = 0.3\n",
+    "#     img[:,:,1] = construction\n",
+    "#     img[:,:,2] = vehicle \n",
+    "#     img[:,:,0] = human\n",
+    "\n",
+    "\n",
+    "#     result = cv2.addWeighted(img, alpha, result, 1-alpha, 0, result)\n",
+    "# #     ax3 = fig.add_subplot(1,2,2)\n",
+    "# #     ax3.set_title(f'Generated_Mask {i}')\n",
+    "# #     ax3.imshow(result/255.)\n",
+    "    \n",
+    "#     cv2.imwrite(f'outputs/stuggart_02/{image_list[i]}', cv2.cvtColor(result, cv2.COLOR_RGB2BGR))\n",
+    "# #     cv2.imwrite(f'outputs/images/{i}.png', cv2.cvtColor(_img, cv2.COLOR_RGB2BGR))\n",
+    "\n",
+    "# #     plt.close(fig)\n",
+    "    \n",
+    "# e = time.time()\n",
+    "# t = str(e-s)\n",
+    "# o = str((e-s)/len(image_list))\n",
+    "# print(f'total time taken = {t}')\n",
+    "# print(f'time taken per image = {o}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "clip = VideoFileClip('test4k.mp4',target_resolution=(288, 512))\n",
+    "# res = clip.fl_image(pipeline)\n",
+    "res.write_videofile('0.mp4', audio=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%HTML\n",
+    "<video width=\"512\" height=\"288\" controls>\n",
+    "  <source src=\"outputs/s0.mp4\" type=\"video/mp4\">\n",
+    "</video>"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def pp(image):\n",
+    "    alpha = 0.6\n",
+    "    dims = image.shape\n",
+    "    x = cv2.resize(image, (256, 256))\n",
+    "    x = np.float32(x)/255.\n",
+    "    z = unet.predict(np.expand_dims(x, axis=0))\n",
+    "    z = np.squeeze(z)\n",
+    "    z = z.reshape(256, 256, 8)\n",
+    "    z = cv2.resize(z, (dims[1], dims[0]))\n",
+    "    y = np.argmax(z, axis=2)\n",
+    "    \n",
+    "    img_color = image.copy()   \n",
+    "    for i in range(dims[0]):\n",
+    "        for j in range(dims[1]):\n",
+    "            img_color[i, j] = color_map[str(y[i, j])]\n",
+    "    cv2.addWeighted(image, alpha, img_color, 1-alpha, 0, img_color)\n",
+    "    return img_color"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "i = 9\n",
+    "test = image.load_img(f'{test_dir}/{image_list[i]}')\n",
+    "test = image.img_to_array(test)\n",
+    "result = pp(test)\n",
+    "plt.imshow(result/255.)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1200 frames found\n"
+     ]
+    }
+   ],
+   "source": [
+    "test_dir = 'validate/stuttgart_02/'\n",
+    "image_list = os.listdir(test_dir)\n",
+    "print(f'{len(image_list)} frames found')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "color_map = {\n",
+    "'0': [0, 0, 0],\n",
+    " '1': [153, 153, 0],\n",
+    " '2': [255, 204, 204],\n",
+    " '3': [255, 0, 127],\n",
+    " '4': [0, 255, 0],\n",
+    " '5': [0, 204, 204],\n",
+    " '6': [255, 0, 0],\n",
+    " '7': [0, 0, 255]\n",
+    "}\n",
+    "\n",
+    "s = time.time()\n",
+    "for t in tqdm.tqdm(range(len(image_list))):\n",
+    "    img = image.load_img(f'{test_dir}/{image_list[t]}')\n",
+    "    img = image.img_to_array(img)\n",
+    "    alpha = 0.6\n",
+    "    dims = img.shape\n",
+    "    x = cv2.resize(img, (256, 256))\n",
+    "    x = np.float32(x)/255.\n",
+    "    z = unet.predict(np.expand_dims(x, axis=0))\n",
+    "    z = np.squeeze(z)\n",
+    "    z = z.reshape(256, 256, 8)\n",
+    "    z = cv2.resize(z, (dims[1], dims[0]))\n",
+    "    y = np.argmax(z, axis=2)\n",
+    "    \n",
+    "    img_color = img.copy()   \n",
+    "    for i in range(dims[0]):\n",
+    "        for j in range(dims[1]):\n",
+    "            img_color[i, j] = color_map[str(y[i, j])]\n",
+    "    cv2.addWeighted(img, alpha, img_color, 1-alpha, 0, img_color)\n",
+    "    cv2.imwrite(f'outputs/stuggart02/{image_list[t]}', cv2.cvtColor(img_color, cv2.COLOR_RGB2BGR))   \n",
+    "e = time.time()\n",
+    "t = str(e-s)\n",
+    "o = str((e-s)/len(image_list))\n",
+    "print(f'total time taken = {t}')\n",
+    "print(f'time taken per image = {o}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading video stuggart00\n",
+      "Successfully loaded video stuggart00\n",
+      "[MoviePy] >>>> Building video outputs/videos/stuggart00.mp4\n",
+      "[MoviePy] Writing video outputs/videos/stuggart00.mp4\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 600/600 [00:26<00:00, 22.90it/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[MoviePy] Done.\n",
+      "[MoviePy] >>>> Video ready: outputs/videos/stuggart00.mp4 \n",
+      "\n",
+      "Successfully created video stuggart00\n",
+      "loading video stuggart01\n",
+      "Successfully loaded video stuggart01\n",
+      "[MoviePy] >>>> Building video outputs/videos/stuggart01.mp4\n",
+      "[MoviePy] Writing video outputs/videos/stuggart01.mp4\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 1101/1101 [00:50<00:00, 21.88it/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[MoviePy] Done.\n",
+      "[MoviePy] >>>> Video ready: outputs/videos/stuggart01.mp4 \n",
+      "\n",
+      "Successfully created video stuggart01\n",
+      "loading video stuggart02\n",
+      "Successfully loaded video stuggart02\n",
+      "[MoviePy] >>>> Building video outputs/videos/stuggart02.mp4\n",
+      "[MoviePy] Writing video outputs/videos/stuggart02.mp4\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 1201/1201 [00:52<00:00, 22.88it/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[MoviePy] Done.\n",
+      "[MoviePy] >>>> Video ready: outputs/videos/stuggart02.mp4 \n",
+      "\n",
+      "Successfully created video stuggart02\n"
+     ]
+    }
+   ],
+   "source": [
+    "for folder in ['stuggart00', 'stuggart01', 'stuggart02']:\n",
+    "    print(f'loading video {folder}')\n",
+    "    clip = ImageSequenceClip(f'outputs/{folder}/', fps=18, load_images=True)\n",
+    "    print(f'Successfully loaded video {folder}')\n",
+    "    clip.write_videofile(f'outputs/videos/{folder}.mp4')\n",
+    "    print(f'Successfully created video {folder}')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}