Merge pull request #40 from jaelgu/main

Add showcase in reverse_video_search notebook
towhee-io · May 31, 2022 · 49ae03e · 49ae03e
2 parents 7966921 + 47b5600
commit 49ae03e
Showing 1 changed file with 97 additions and 20 deletions.
diff --git a/video/reverse_video_search/reverse_video_search.ipynb b/video/reverse_video_search/reverse_video_search.ipynb
@@ -37,8 +37,10 @@
     "| -- |\n",
     "| pymilvus |\n",
     "| towhee |\n",
+    "| towhee.models |\n",
     "| torch |\n",
-    "| torchvision |"
+    "| torchvision |\n",
+    "| gradio |"
    ]
   },
   {
@@ -149,7 +151,6 @@
    "metadata": {},
    "source": [
     "For later steps to easier get videos & measure results, we build some helpful functions in advance:\n",
-    "- **read_videos:** get video paths by video ids;\n",
     "- **ground_truth:** get ground-truth video ids for the query video by its path"
    ]
   },
@@ -168,13 +169,6 @@
     "# print(id_video)\n",
     "# print(label_ids)\n",
     "\n",
-    "def read_videos(results):\n",
-    "    videos = []\n",
-    "    for re in results:\n",
-    "        path = id_video[re.id]\n",
-    "        videos.append(path)\n",
-    "    return videos\n",
-    "\n",
     "def ground_truth(path):\n",
     "    label = path.split('/')[-2]\n",
     "    return label_ids[label]"
@@ -266,7 +260,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Total insert time: 93.22s\n",
+      "Total insert time: 95.91s\n",
       "Total number of inserted data is 200.\n"
      ]
     }
@@ -324,7 +318,7 @@
     "\n",
     "Now all embeddings of candidate videos have been inserted into Milvus collection, we can query embeddings across the collection for nearest neighbors.\n",
     "\n",
-    "To get query embeddings, we should go through same pre-insert steps for each input video. Because Milvus returns video ids and vector distances, we use the `read_videos` function to get corresponding video paths based on ids."
+    "To get query embeddings, we should go through same pre-insert steps for each input video. Because Milvus returns video ids and vector distances, we use the `id_video` dictionary to get corresponding video paths based on ids."
    ]
   },
   {
@@ -352,7 +346,7 @@
     "        .video_classification['frames', 'vec'].pytorchvideo(\n",
     "            model_name='x3d_m', predict=False, skip_preprocess=True)\n",
     "        .milvus_search['vec', 'result'](collection=collection, limit=10)\n",
-    "        .runas_op['result', 'res_path'](func=read_videos)\n",
+    "        .runas_op['result', 'res_path'](func=lambda res: [id_video[x.id] for x in res])\n",
     "        .select['res_path']().to_list()[0].res_path\n",
     ")"
    ]
@@ -497,7 +491,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Total search time: 10.94s\n"
+      "Total search time: 10.65s\n"
      ]
     }
    ],
@@ -555,7 +549,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Total insert time: 92.52s\n",
+      "Total insert time: 92.18s\n",
       "Total number of inserted data is 200.\n"
      ]
     },
@@ -613,7 +607,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Total search time: 10.78s\n"
+      "Total search time: 11.75s\n"
      ]
     }
    ],
@@ -698,7 +692,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Total insert time: 151.25s\n",
+      "Total insert time: 141.29s\n",
       "Total number of inserted data is 200.\n"
      ]
     },
@@ -756,7 +750,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Total search time: 24.10s\n"
+      "Total search time: 17.90s\n"
      ]
     }
    ],
@@ -814,13 +808,96 @@
     "Switching to MVIT model increases the mHR to 0.79 and mAP to 0.86, which are much better than X3D model. However, both insert and search time have increased. It's time for you to make trade-off between latency and accuracy. You're always encouraged to play around with this tutorial."
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "607783a1",
+   "metadata": {},
+   "source": [
+    "## Release a Showcase\n",
+    "\n",
+    "We've learnt how to build a reverse video search engine. Now it's time to add some interface and release a showcase. Towhee provides `towhee.api()` to wrap the data processing pipeline as a function with `.as_function()`. So we can build a quick demo with this `milvus_search_function` with [Gradio](https://gradio.app/)."
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "id": "a78c9ba6",
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using cache found in /home/mengjia.gu/.cache/torch/hub/facebookresearch_pytorchvideo_main\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on local URL:  http://127.0.0.1:7860/\n",
+      "Running on public URL: https://53758.gradio.app\n",
+      "\n",
+      "This share link expires in 72 hours. For free permanent hosting, check out Spaces (https://huggingface.co/spaces)\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "        <iframe\n",
+       "            width=\"900\"\n",
+       "            height=\"500\"\n",
+       "            src=\"https://53758.gradio.app\"\n",
+       "            frameborder=\"0\"\n",
+       "            allowfullscreen\n",
+       "            \n",
+       "        ></iframe>\n",
+       "        "
+      ],
+      "text/plain": [
+       "<IPython.lib.display.IFrame at 0x7fb5c95fd8e0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "(<gradio.routes.App at 0x7fb5ccb552e0>,\n",
+       " 'http://127.0.0.1:7860/',\n",
+       " 'https://53758.gradio.app')"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import gradio\n",
+    "\n",
+    "with towhee.api() as api:\n",
+    "    milvus_search_function = (\n",
+    "         api.video_decode.ffmpeg(\n",
+    "                sample_type='uniform_temporal_subsample', args={'num_samples': 32})\n",
+    "            .video_classification.pytorchvideo(\n",
+    "                model_name='mvit_base_32x3', predict=False, skip_preprocess=True)\n",
+    "            .tensor_normalize()\n",
+    "            .milvus_search(collection='mvit_base', limit=5)\n",
+    "            .runas_op(func=lambda res: [id_video[x.id] for x in res])\n",
+    "            .as_function()\n",
+    "    )\n",
+    "    \n",
+    "\n",
+    "interface = gradio.Interface(milvus_search_function, \n",
+    "                             inputs=gradio.Video(source='upload'),\n",
+    "                             outputs=[gradio.Video(format='mp4') for _ in range(5)]\n",
+    "                            )\n",
+    "\n",
+    "interface.launch(inline=True, share=True)"
+   ]
   }
  ],
  "metadata": {