TensorFlow Serving Prediction Example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Start your TensorFlow Serving instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n!docker run -p 8500:8500 -p 8501:8501 --mount type=bind,source=/tmp,target=/models/movie -e MODEL_NAME=movie -t tensorflow/serving\\n'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "!docker run -p 8500:8500 -p 8501:8501 \\\n",
    "--mount type=bind,source=/tmp,target=/models/movie \\\n",
    "-e MODEL_NAME=movie -t tensorflow/serving\n",
    "\n",
    "You can check your model with \n",
    "\n",
    "!saved_model_cli show --dir /path/to/the/model --tag_set serve --signature_def serving_default\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Some Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXTS = (\"\"\"Quantum of Solace continues the adventures of James Bond after Casino Royale. Betrayed by Vesper, the woman he loved, 007 fights the urge to make his latest mission personal. Pursuing his determination to uncover the truth, \n",
    "                    Bond and M interrogate Mr. White, who reveals that the organization that blackmailed Vesper is far more complex and dangerous than anyone had imagined.\"\"\",\n",
    "#          \"\"\"Monty Python and the Holy Grail loosely follows the legend of King Arthur. Arthur along with his squire, Patsy, recruits his\n",
    "#     Knights of the Round Table, including Sir Bedevere the Wise, Sir Lancelot the Brave, Sir Robin the\n",
    "#     Not-Quite-So-Brave-As-Sir-Lancelot and Sir Galahad the Pure. On the way Arthur battles the Black Knight who, despite having had\n",
    "#     all his limbs chopped off, insists he can still fight. They reach Camelot, but Arthur decides not to enter, as \"it is a silly\n",
    "#     place\".\"\"\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# REST Prediction Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rest_request():\n",
    "    url = 'http://localhost:8501/v1/models/movie:predict'\n",
    "    payload = json.dumps({\"instances\": [TEXTS[0]]})\n",
    "    r = requests.post(url, payload)\n",
    "    return r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The slowest run took 55.64 times longer than the fastest. This could mean that an intermediate result is being cached.\n",
      "63.6 ms ± 134 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit -n 1\n",
    "\n",
    "rs_rest = rest_request()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'predictions': [{'classes': ['0', '1', '2', '3', '4', '5', '6', '7', '8'],\n",
       "   'scores': [0.401,\n",
       "    0.0962357,\n",
       "    0.262375,\n",
       "    0.392996,\n",
       "    0.0621801,\n",
       "    0.0822788,\n",
       "    0.296542,\n",
       "    0.0819803,\n",
       "    0.567294]}]}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rs_rest = rest_request()\n",
    "rs_rest.json()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# gRPC Predict Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import grpc\n",
    "from tensorflow_serving.apis import predict_pb2\n",
    "from tensorflow_serving.apis import prediction_service_pb2_grpc\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grpc_request():\n",
    "    hostport = 'localhost:8500'\n",
    "    \n",
    "    channel = grpc.insecure_channel(hostport)\n",
    "    stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)\n",
    "    request = predict_pb2.PredictRequest()\n",
    "    request.model_spec.name = 'movie'\n",
    "    request.model_spec.signature_name = 'serving_default'\n",
    "    \n",
    "    request.inputs['inputs'].CopyFrom(tf.contrib.util.make_tensor_proto(TEXTS[0], shape=[1,1]))\n",
    "    result_future = stub.Predict.future(request, 10.25) \n",
    "    return result_future"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.52 ms ± 413 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit -n 1\n",
    "rs_grpc = grpc_request()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "outputs {\n",
       "  key: \"classes\"\n",
       "  value {\n",
       "    dtype: DT_STRING\n",
       "    tensor_shape {\n",
       "      dim {\n",
       "        size: 1\n",
       "      }\n",
       "      dim {\n",
       "        size: 9\n",
       "      }\n",
       "    }\n",
       "    string_val: \"0\"\n",
       "    string_val: \"1\"\n",
       "    string_val: \"2\"\n",
       "    string_val: \"3\"\n",
       "    string_val: \"4\"\n",
       "    string_val: \"5\"\n",
       "    string_val: \"6\"\n",
       "    string_val: \"7\"\n",
       "    string_val: \"8\"\n",
       "  }\n",
       "}\n",
       "outputs {\n",
       "  key: \"scores\"\n",
       "  value {\n",
       "    dtype: DT_FLOAT\n",
       "    tensor_shape {\n",
       "      dim {\n",
       "        size: 1\n",
       "      }\n",
       "      dim {\n",
       "        size: 9\n",
       "      }\n",
       "    }\n",
       "    float_val: 0.4009998142719269\n",
       "    float_val: 0.0962357223033905\n",
       "    float_val: 0.2623746991157532\n",
       "    float_val: 0.3929961621761322\n",
       "    float_val: 0.06218007206916809\n",
       "    float_val: 0.08227881789207458\n",
       "    float_val: 0.2965419888496399\n",
       "    float_val: 0.08198034763336182\n",
       "    float_val: 0.5672940611839294\n",
       "  }\n",
       "}\n",
       "model_spec {\n",
       "  name: \"movie\"\n",
       "  version {\n",
       "    value: 1556583584\n",
       "  }\n",
       "  signature_name: \"serving_default\"\n",
       "}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = grpc_request()\n",
    "a.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}