samples/local-examples.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "lesbian-springer",
   "metadata": {},
   "source": [
    "## Seldon V2 Non Kubernetes Local Examples\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b1925df",
   "metadata": {},
   "source": [
    "### SKLearn Model\n",
    "\n",
    "We use a simple sklearn iris classification model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8b4be911",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: iris\r\n",
      "spec:\r\n",
      "  storageUri: \"gs://seldon-models/scv2/samples/mlserver_1.3.5/iris-sklearn\"\r\n",
      "  requirements:\r\n",
      "  - sklearn\r\n",
      "  memory: 100Ki\r\n"
     ]
    }
   ],
   "source": [
    "!cat ./models/sklearn-iris-gs.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26c11833",
   "metadata": {},
   "source": [
    "Load the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "checked-cream",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model load -f ./models/sklearn-iris-gs.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccd73e1c",
   "metadata": {},
   "source": [
    "Wait for the model to be ready"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "interracial-secret",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model status iris -w ModelAvailable | jq -M ."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83a3583c",
   "metadata": {},
   "source": [
    "Do a REST inference call"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "upper-cholesterol",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "\t\"model_name\": \"iris_1\",\r\n",
      "\t\"model_version\": \"1\",\r\n",
      "\t\"id\": \"983bd95f-4b4d-4ff1-95b2-df9d6d089164\",\r\n",
      "\t\"parameters\": {},\r\n",
      "\t\"outputs\": [\r\n",
      "\t\t{\r\n",
      "\t\t\t\"name\": \"predict\",\r\n",
      "\t\t\t\"shape\": [\r\n",
      "\t\t\t\t1,\r\n",
      "\t\t\t\t1\r\n",
      "\t\t\t],\r\n",
      "\t\t\t\"datatype\": \"INT64\",\r\n",
      "\t\t\t\"parameters\": {\r\n",
      "\t\t\t\t\"content_type\": \"np\"\r\n",
      "\t\t\t},\r\n",
      "\t\t\t\"data\": [\r\n",
      "\t\t\t\t2\r\n",
      "\t\t\t]\r\n",
      "\t\t}\r\n",
      "\t]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer iris \\\n",
    "  '{\"inputs\": [{\"name\": \"predict\", \"shape\": [1, 4], \"datatype\": \"FP32\", \"data\": [[1, 2, 3, 4]]}]}' "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58057385",
   "metadata": {},
   "source": [
    "Do a gRPC inference call"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "right-talent",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "  \"modelName\": \"iris_1\",\r\n",
      "  \"modelVersion\": \"1\",\r\n",
      "  \"outputs\": [\r\n",
      "    {\r\n",
      "      \"name\": \"predict\",\r\n",
      "      \"datatype\": \"INT64\",\r\n",
      "      \"shape\": [\r\n",
      "        \"1\",\r\n",
      "        \"1\"\r\n",
      "      ],\r\n",
      "      \"parameters\": {\r\n",
      "        \"content_type\": {\r\n",
      "          \"stringParam\": \"np\"\r\n",
      "        }\r\n",
      "      },\r\n",
      "      \"contents\": {\r\n",
      "        \"int64Contents\": [\r\n",
      "          \"2\"\r\n",
      "        ]\r\n",
      "      }\r\n",
      "    }\r\n",
      "  ]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer iris --inference-mode grpc \\\n",
    "   '{\"model_name\":\"iris\",\"inputs\":[{\"name\":\"input\",\"contents\":{\"fp32_contents\":[1,2,3,4]},\"datatype\":\"FP32\",\"shape\":[1,4]}]}' | jq -M ."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d95a5f03",
   "metadata": {},
   "source": [
    "Unload the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ceramic-illness",
   "metadata": {},
   "outputs": [],
   "source": [
    "!seldon model unload iris"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30431134",
   "metadata": {},
   "source": [
    "### Tensorflow Model\n",
    "\n",
    "We run a simple tensorflow model. Note the requirements section specifying `tensorflow`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "34165288",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: tfsimple1\r\n",
      "spec:\r\n",
      "  storageUri: \"gs://seldon-models/triton/simple\"\r\n",
      "  requirements:\r\n",
      "  - tensorflow\r\n",
      "  memory: 100Ki\r\n"
     ]
    }
   ],
   "source": [
    "!cat ./models/tfsimple1.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "275a8445",
   "metadata": {},
   "source": [
    "Load the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "726f2dd0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model load -f ./models/tfsimple1.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b141e73",
   "metadata": {},
   "source": [
    "Wait for the model to be ready."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2d9b631f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model status tfsimple1 -w ModelAvailable | jq -M ."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88301952",
   "metadata": {},
   "source": [
    "Get model metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f812f531",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "\t\"name\": \"tfsimple1_1\",\r\n",
      "\t\"versions\": [\r\n",
      "\t\t\"1\"\r\n",
      "\t],\r\n",
      "\t\"platform\": \"tensorflow_graphdef\",\r\n",
      "\t\"inputs\": [\r\n",
      "\t\t{\r\n",
      "\t\t\t\"name\": \"INPUT0\",\r\n",
      "\t\t\t\"datatype\": \"INT32\",\r\n",
      "\t\t\t\"shape\": [\r\n",
      "\t\t\t\t-1,\r\n",
      "\t\t\t\t16\r\n",
      "\t\t\t]\r\n",
      "\t\t},\r\n",
      "\t\t{\r\n",
      "\t\t\t\"name\": \"INPUT1\",\r\n",
      "\t\t\t\"datatype\": \"INT32\",\r\n",
      "\t\t\t\"shape\": [\r\n",
      "\t\t\t\t-1,\r\n",
      "\t\t\t\t16\r\n",
      "\t\t\t]\r\n",
      "\t\t}\r\n",
      "\t],\r\n",
      "\t\"outputs\": [\r\n",
      "\t\t{\r\n",
      "\t\t\t\"name\": \"OUTPUT0\",\r\n",
      "\t\t\t\"datatype\": \"INT32\",\r\n",
      "\t\t\t\"shape\": [\r\n",
      "\t\t\t\t-1,\r\n",
      "\t\t\t\t16\r\n",
      "\t\t\t]\r\n",
      "\t\t},\r\n",
      "\t\t{\r\n",
      "\t\t\t\"name\": \"OUTPUT1\",\r\n",
      "\t\t\t\"datatype\": \"INT32\",\r\n",
      "\t\t\t\"shape\": [\r\n",
      "\t\t\t\t-1,\r\n",
      "\t\t\t\t16\r\n",
      "\t\t\t]\r\n",
      "\t\t}\r\n",
      "\t]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model metadata tfsimple1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77f85f7d",
   "metadata": {},
   "source": [
    "Do a REST inference call."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7abd240a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "  \"model_name\": \"tfsimple1_1\",\r\n",
      "  \"model_version\": \"1\",\r\n",
      "  \"outputs\": [\r\n",
      "    {\r\n",
      "      \"name\": \"OUTPUT0\",\r\n",
      "      \"datatype\": \"INT32\",\r\n",
      "      \"shape\": [\r\n",
      "        1,\r\n",
      "        16\r\n",
      "      ],\r\n",
      "      \"data\": [\r\n",
      "        2,\r\n",
      "        4,\r\n",
      "        6,\r\n",
      "        8,\r\n",
      "        10,\r\n",
      "        12,\r\n",
      "        14,\r\n",
      "        16,\r\n",
      "        18,\r\n",
      "        20,\r\n",
      "        22,\r\n",
      "        24,\r\n",
      "        26,\r\n",
      "        28,\r\n",
      "        30,\r\n",
      "        32\r\n",
      "      ]\r\n",
      "    },\r\n",
      "    {\r\n",
      "      \"name\": \"OUTPUT1\",\r\n",
      "      \"datatype\": \"INT32\",\r\n",
      "      \"shape\": [\r\n",
      "        1,\r\n",
      "        16\r\n",
      "      ],\r\n",
      "      \"data\": [\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0,\r\n",
      "        0\r\n",
      "      ]\r\n",
      "    }\r\n",
      "  ]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer tfsimple1 \\\n",
    "    '{\"inputs\":[{\"name\":\"INPUT0\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]},{\"name\":\"INPUT1\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]}]}' | jq -M ."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34a1f004",
   "metadata": {},
   "source": [
    "Do a gRPC inference call"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f86c494a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "  \"modelName\": \"tfsimple1_1\",\r\n",
      "  \"modelVersion\": \"1\",\r\n",
      "  \"outputs\": [\r\n",
      "    {\r\n",
      "      \"name\": \"OUTPUT0\",\r\n",
      "      \"datatype\": \"INT32\",\r\n",
      "      \"shape\": [\r\n",
      "        \"1\",\r\n",
      "        \"16\"\r\n",
      "      ],\r\n",
      "      \"contents\": {\r\n",
      "        \"intContents\": [\r\n",
      "          2,\r\n",
      "          4,\r\n",
      "          6,\r\n",
      "          8,\r\n",
      "          10,\r\n",
      "          12,\r\n",
      "          14,\r\n",
      "          16,\r\n",
      "          18,\r\n",
      "          20,\r\n",
      "          22,\r\n",
      "          24,\r\n",
      "          26,\r\n",
      "          28,\r\n",
      "          30,\r\n",
      "          32\r\n",
      "        ]\r\n",
      "      }\r\n",
      "    },\r\n",
      "    {\r\n",
      "      \"name\": \"OUTPUT1\",\r\n",
      "      \"datatype\": \"INT32\",\r\n",
      "      \"shape\": [\r\n",
      "        \"1\",\r\n",
      "        \"16\"\r\n",
      "      ],\r\n",
      "      \"contents\": {\r\n",
      "        \"intContents\": [\r\n",
      "          0,\r\n",
      "          0,\r\n",
      "          0,\r\n",
      "          0,\r\n",
      "          0,\r\n",
      "          0,\r\n",
      "          0,\r\n",
      "          0,\r\n",
      "          0,\r\n",
      "          0,\r\n",
      "          0,\r\n",
      "          0,\r\n",
      "          0,\r\n",
      "          0,\r\n",
      "          0,\r\n",
      "          0\r\n",
      "        ]\r\n",
      "      }\r\n",
      "    }\r\n",
      "  ]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer tfsimple1 --inference-mode grpc \\\n",
    "    '{\"model_name\":\"tfsimple1\",\"inputs\":[{\"name\":\"INPUT0\",\"contents\":{\"int_contents\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},\"datatype\":\"INT32\",\"shape\":[1,16]},{\"name\":\"INPUT1\",\"contents\":{\"int_contents\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},\"datatype\":\"INT32\",\"shape\":[1,16]}]}' | jq -M ."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d32d3c0",
   "metadata": {},
   "source": [
    "Unload the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f7405940",
   "metadata": {},
   "outputs": [],
   "source": [
    "!seldon model unload tfsimple1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "amber-inclusion",
   "metadata": {},
   "source": [
    "### Experiment\n",
    "\n",
    "We will use two SKlearn Iris classification models to illustrate an experiment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "dfc5a4fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: iris\r\n",
      "spec:\r\n",
      "  storageUri: \"gs://seldon-models/mlserver/iris\"\r\n",
      "  requirements:\r\n",
      "  - sklearn\r\n"
     ]
    }
   ],
   "source": [
    "!cat ./models/sklearn1.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8837f315",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: iris2\r\n",
      "spec:\r\n",
      "  storageUri: \"gs://seldon-models/mlserver/iris\"\r\n",
      "  requirements:\r\n",
      "  - sklearn\r\n"
     ]
    }
   ],
   "source": [
    "!cat ./models/sklearn2.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "359b456e",
   "metadata": {},
   "source": [
    "Load both models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "elect-samba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\n",
      "{}\n"
     ]
    }
   ],
   "source": [
    "!seldon model load -f ./models/sklearn1.yaml\n",
    "!seldon model load -f ./models/sklearn2.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1aaa2b8f",
   "metadata": {},
   "source": [
    "Wait for both models to be ready."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "floral-moisture",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"modelName\": \"iris\",\n",
      "  \"versions\": [\n",
      "    {\n",
      "      \"version\": 1,\n",
      "      \"serverName\": \"mlserver\",\n",
      "      \"kubernetesMeta\": {},\n",
      "      \"modelReplicaState\": {\n",
      "        \"0\": {\n",
      "          \"state\": \"Available\",\n",
      "          \"lastChangeTimestamp\": \"2023-06-29T14:01:41.362720538Z\"\n",
      "        }\n",
      "      },\n",
      "      \"state\": {\n",
      "        \"state\": \"ModelAvailable\",\n",
      "        \"availableReplicas\": 1,\n",
      "        \"lastChangeTimestamp\": \"2023-06-29T14:01:41.362720538Z\"\n",
      "      },\n",
      "      \"modelDefn\": {\n",
      "        \"meta\": {\n",
      "          \"name\": \"iris\",\n",
      "          \"kubernetesMeta\": {}\n",
      "        },\n",
      "        \"modelSpec\": {\n",
      "          \"uri\": \"gs://seldon-models/mlserver/iris\",\n",
      "          \"requirements\": [\n",
      "            \"sklearn\"\n",
      "          ]\n",
      "        },\n",
      "        \"deploymentSpec\": {\n",
      "          \"replicas\": 1\n",
      "        }\n",
      "      }\n",
      "    }\n",
      "  ]\n",
      "}\n",
      "{\n",
      "  \"modelName\": \"iris2\",\n",
      "  \"versions\": [\n",
      "    {\n",
      "      \"version\": 1,\n",
      "      \"serverName\": \"mlserver\",\n",
      "      \"kubernetesMeta\": {},\n",
      "      \"modelReplicaState\": {\n",
      "        \"0\": {\n",
      "          \"state\": \"Available\",\n",
      "          \"lastChangeTimestamp\": \"2023-06-29T14:01:41.362845079Z\"\n",
      "        }\n",
      "      },\n",
      "      \"state\": {\n",
      "        \"state\": \"ModelAvailable\",\n",
      "        \"availableReplicas\": 1,\n",
      "        \"lastChangeTimestamp\": \"2023-06-29T14:01:41.362845079Z\"\n",
      "      },\n",
      "      \"modelDefn\": {\n",
      "        \"meta\": {\n",
      "          \"name\": \"iris2\",\n",
      "          \"kubernetesMeta\": {}\n",
      "        },\n",
      "        \"modelSpec\": {\n",
      "          \"uri\": \"gs://seldon-models/mlserver/iris\",\n",
      "          \"requirements\": [\n",
      "            \"sklearn\"\n",
      "          ]\n",
      "        },\n",
      "        \"deploymentSpec\": {\n",
      "          \"replicas\": 1\n",
      "        }\n",
      "      }\n",
      "    }\n",
      "  ]\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "!seldon model status iris | jq -M .\n",
    "!seldon model status iris2 | jq -M ."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf1ac158",
   "metadata": {},
   "source": [
    "Create an experiment that modifies the iris model to add a second model splitting traffic 50/50 between the two."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "10fe4447",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Experiment\r\n",
      "metadata:\r\n",
      "  name: experiment-sample\r\n",
      "spec:\r\n",
      "  default: iris\r\n",
      "  candidates:\r\n",
      "  - name: iris\r\n",
      "    weight: 50\r\n",
      "  - name: iris2\r\n",
      "    weight: 50\r\n"
     ]
    }
   ],
   "source": [
    "!cat ./experiments/ab-default-model.yaml "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c9b215b",
   "metadata": {},
   "source": [
    "Start the experiment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "differential-array",
   "metadata": {},
   "outputs": [],
   "source": [
    "!seldon experiment start -f ./experiments/ab-default-model.yaml "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "727e4fd4",
   "metadata": {},
   "source": [
    "Wait for the experiment to be ready."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "overhead-banks",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "  \"experimentName\": \"experiment-sample\",\r\n",
      "  \"active\": true,\r\n",
      "  \"candidatesReady\": true,\r\n",
      "  \"mirrorReady\": true,\r\n",
      "  \"statusDescription\": \"experiment active\",\r\n",
      "  \"kubernetesMeta\": {}\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon experiment status experiment-sample -w | jq -M ."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efe293b0",
   "metadata": {},
   "source": [
    "Run a set of calls and record which route the traffic took. There should be roughly a 50/50 split."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "finite-significance",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success: map[:iris2_1::57 :iris_1::43]\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer iris -i 100 \\\n",
    "  '{\"inputs\": [{\"name\": \"predict\", \"shape\": [1, 4], \"datatype\": \"FP32\", \"data\": [[1, 2, 3, 4]]}]}' "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30fe1b9f",
   "metadata": {},
   "source": [
    "Run one more request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "85357805",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "\t\"model_name\": \"iris_1\",\r\n",
      "\t\"model_version\": \"1\",\r\n",
      "\t\"id\": \"fa425bdf-737c-41fe-894d-58868f70fe5d\",\r\n",
      "\t\"parameters\": {},\r\n",
      "\t\"outputs\": [\r\n",
      "\t\t{\r\n",
      "\t\t\t\"name\": \"predict\",\r\n",
      "\t\t\t\"shape\": [\r\n",
      "\t\t\t\t1,\r\n",
      "\t\t\t\t1\r\n",
      "\t\t\t],\r\n",
      "\t\t\t\"datatype\": \"INT64\",\r\n",
      "\t\t\t\"parameters\": {\r\n",
      "\t\t\t\t\"content_type\": \"np\"\r\n",
      "\t\t\t},\r\n",
      "\t\t\t\"data\": [\r\n",
      "\t\t\t\t2\r\n",
      "\t\t\t]\r\n",
      "\t\t}\r\n",
      "\t]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer iris \\\n",
    "  '{\"inputs\": [{\"name\": \"predict\", \"shape\": [1, 4], \"datatype\": \"FP32\", \"data\": [[1, 2, 3, 4]]}]}' "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7ca1665",
   "metadata": {},
   "source": [
    "Use sticky session key passed by last infer request to ensure same route is taken each time. \n",
    "We will test REST and gRPC."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1ffb00b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success: map[:iris_1::50]\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer iris -s -i 50 \\\n",
    "  '{\"inputs\": [{\"name\": \"predict\", \"shape\": [1, 4], \"datatype\": \"FP32\", \"data\": [[1, 2, 3, 4]]}]}' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "81a10627",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success: map[:iris_1::50]\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer iris --inference-mode grpc -s -i 50\\\n",
    "   '{\"model_name\":\"iris\",\"inputs\":[{\"name\":\"input\",\"contents\":{\"fp32_contents\":[1,2,3,4]},\"datatype\":\"FP32\",\"shape\":[1,4]}]}' "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12ad0016",
   "metadata": {},
   "source": [
    "Stop the experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "downtown-construction",
   "metadata": {},
   "outputs": [],
   "source": [
    "!seldon experiment stop experiment-sample"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5264ed4",
   "metadata": {},
   "source": [
    "Show the requests all go to original model now."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b085568e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success: map[:iris_1::100]\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer iris -i 100 \\\n",
    "  '{\"inputs\": [{\"name\": \"predict\", \"shape\": [1, 4], \"datatype\": \"FP32\", \"data\": [[1, 2, 3, 4]]}]}' "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9da648e8",
   "metadata": {},
   "source": [
    "Unload both models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "077bbce8",
   "metadata": {},
   "outputs": [],
   "source": [
    "!seldon model unload iris\n",
    "!seldon model unload iris2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3720658f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}