mlflow/tf_mlflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0201e149-696b-439d-8bc9-837af2a9d23c",
   "metadata": {},
   "source": [
    "# Introduct\n",
    "\n",
    "Tests the MLflow tracking from KF notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2288cb08-9f43-4fd4-9ff5-56261902010a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fff9d263-f31c-4299-a795-7c6fd1bd7e6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# install the mlflow==2.10.0 sdk, since the backend is 2.10.0 from helm chart 0.7.1\n",
    "# !{sys.executable} -m pip install --user --upgrade mlflow==2.10.0 python-dotenv==1.0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3012916a-9712-4e62-8363-ec3f223b1966",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-27 18:06:55.165399: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2024-01-27 18:06:55.165447: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.9.3\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a743a9c-6ff0-40c4-91d1-c1a2257a05d7",
   "metadata": {},
   "source": [
    "## Using remote tracking env variable\n",
    "```python\n",
    "os.environ[\"MLFLOW_TRACKING_URI\"]=\"https://mymlflowhost.example.com\"\n",
    "os.environ[\"MLFLOW_TRACKING_USERNAME\"]=\"user\"\n",
    "os.environ[\"MLFLOW_TRACKING_PASSWORD\"]=\"\"\n",
    "```\n",
    "\n",
    "* https://mlflow.org/docs/latest/tracking/tutorials/remote-server.html\n",
    "* https://www.mlflow.org/docs/latest/auth/index.html#authenticating-to-mlflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0dcb9e7c-668c-4d2d-9be2-0217e92b912b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%writefile .mlflow_env\n",
    "## environment variables for ssh\n",
    "#MLFLOW_TRACKING_URI=\"https://mymlflowhost.example.com\"\n",
    "#MLFLOW_TRACKING_USERNAME=\"user\"\n",
    "#MLFLOW_TRACKING_PASSWORD=\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "73320096-2157-423b-95a8-33de2745b7e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dotenv import load_dotenv\n",
    "import os\n",
    "\n",
    "mlflow_env_file=\".mlflow_env\"\n",
    "load_dotenv(dotenv_path=mlflow_env_file, override=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b29fc540-1f2b-471e-ab32-0cf0b492c691",
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_mlflow_env():\n",
    "    print(f\"mlflow env:\\n\\\n",
    "{os.environ['MLFLOW_TRACKING_URI']}\\n\\\n",
    "{os.environ['MLFLOW_TRACKING_USERNAME']}\\n\\\n",
    "{os.environ['MLFLOW_TRACKING_PASSWORD']}\\n\")\n",
    "    \n",
    "# print_mlflow_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0f5a739c-f64b-43da-8439-72d9eb266151",
   "metadata": {},
   "outputs": [],
   "source": [
    "MLFLOW_EXPERIMENT_NAME=\"kubeflow_notebook\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d36da2b8-0971-4f80-bfd9-d43e7ba260c8",
   "metadata": {},
   "source": [
    "## Boto3 on mlflow server SSL verification\n",
    "\n",
    "Note: \n",
    "* the boto3 client can only verify \"sub1.subdomain.domain.com\"\n",
    "* But not verify host \"sub2.sub1.subdomain.domain.com\", with SSL wildcard certificate \"*.subdomain.domain.com\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3ce1a375-cc20-4ed6-a266-8a41500ed6e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mlflow verison: 2.10.0\n",
      "Loading data...\n",
      "8982 train sequences\n",
      "2246 test sequences\n",
      "46 classes\n",
      "Vectorizing sequence data...\n",
      "x_train shape: (8982, 1000)\n",
      "x_test shape: (2246, 1000)\n",
      "Convert class vector to binary class matrix (for use with categorical_crossentropy)\n",
      "y_train shape: (8982, 46)\n",
      "y_test shape: (2246, 46)\n",
      "Building model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-27 18:07:01.637657: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2024-01-27 18:07:01.637715: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2024-01-27 18:07:01.637749: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:163] no NVIDIA GPU device is present: /dev/nvidia0 does not exist\n",
      "2024-01-27 18:07:01.638074: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "253/253 [==============================] - 7s 26ms/step - loss: 1.4133 - accuracy: 0.6828 - val_loss: 1.0896 - val_accuracy: 0.7664\n",
      "Epoch 2/5\n",
      "253/253 [==============================] - 6s 24ms/step - loss: 0.7795 - accuracy: 0.8155 - val_loss: 0.9173 - val_accuracy: 0.7875\n",
      "Epoch 3/5\n",
      "253/253 [==============================] - 6s 24ms/step - loss: 0.5484 - accuracy: 0.8660 - val_loss: 0.8531 - val_accuracy: 0.8031\n",
      "Epoch 4/5\n",
      "253/253 [==============================] - 6s 23ms/step - loss: 0.4075 - accuracy: 0.9025 - val_loss: 0.8967 - val_accuracy: 0.8065\n",
      "Epoch 5/5\n",
      "253/253 [==============================] - 6s 23ms/step - loss: 0.3388 - accuracy: 0.9145 - val_loss: 0.8808 - val_accuracy: 0.8020\n",
      "1/1 [==============================] - 0s 63ms/step\n",
      "INFO:tensorflow:Assets written to: /tmp/tmp7iw4ivwt/model/data/model/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/01/27 18:07:39 WARNING mlflow.utils.autologging_utils: MLflow autologging encountered a warning: \"/opt/conda/lib/python3.8/site-packages/_distutils_hack/__init__.py:33: UserWarning: Setuptools is replacing distutils.\"\n",
      "2024/01/27 18:07:43 WARNING mlflow.utils.autologging_utils: Encountered unexpected error during tensorflow autologging: 413 Client Error: Request Entity Too Large for url: https://kindfor-dagster.med.uni-muenchen.de/api/2.0/mlflow-artifacts/artifacts/1/80d7197bc315417c9ec3771189b007a1/artifacts/model/data/model/variables/variables.data-00000-of-00001. Response text: <html>\n",
      "<head><title>413 Request Entity Too Large</title></head>\n",
      "<body>\n",
      "<center><h1>413 Request Entity Too Large</h1></center>\n",
      "<hr><center>nginx</center>\n",
      "</body>\n",
      "</html>\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "71/71 [==============================] - 0s 1ms/step - loss: 0.8643 - accuracy: 0.8023\n",
      "Test score: 0.8643499612808228\n",
      "Test accuracy: 0.8023152351379395\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Trains and evaluate a simple MLP\n",
    "on the Reuters newswire topic classification task.\n",
    "\"\"\"\n",
    "import numpy as np\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.datasets import reuters\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Dropout, Activation\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "\n",
    "# The following import and function call are the only additions to code required\n",
    "# to automatically log metrics and parameters to MLflow.\n",
    "import mlflow\n",
    "import time\n",
    "\n",
    "print(f\"mlflow verison: {mlflow.__version__}\")\n",
    "\n",
    "# Set the run name to timestamp\n",
    "# run_name = str(time.time())\n",
    "# Set the run name to time string\n",
    "run_name = time.strftime(\"%Y-%m-%d_%H-%M-%S\")\n",
    "\n",
    "# Create the experiment\n",
    "# default experiment id is 0\n",
    "# this will be shown in the remote mlflow server as experiment name\n",
    "experiment_name = MLFLOW_EXPERIMENT_NAME\n",
    "# search_pattern = f\"name LIKE '{experiment_name}'\"\n",
    "search_pattern = f\"name = '{experiment_name}'\"\n",
    "# search the experiment with the name, if doesn't exist will return an empty list\n",
    "experiments = mlflow.search_experiments(filter_string=search_pattern)\n",
    "if len(experiments) == 0:\n",
    "   experiment_id = mlflow.create_experiment(name=experiment_name)\n",
    "   print(f\"experiment with string id {experiment_id} is created.\")\n",
    "\n",
    "mlflow.tensorflow.autolog()\n",
    "mlflow.set_experiment(experiment_name=experiment_name)\n",
    "mlflow.set_tag(\"mlflow.runName\", run_name)\n",
    "\n",
    "max_words = 1000\n",
    "batch_size = 32\n",
    "epochs = 5\n",
    "\n",
    "print(\"Loading data...\")\n",
    "(x_train, y_train), (x_test, y_test) = reuters.load_data(num_words=max_words, test_split=0.2)\n",
    "\n",
    "print(len(x_train), \"train sequences\")\n",
    "print(len(x_test), \"test sequences\")\n",
    "\n",
    "num_classes = np.max(y_train) + 1\n",
    "print(num_classes, \"classes\")\n",
    "\n",
    "print(\"Vectorizing sequence data...\")\n",
    "tokenizer = Tokenizer(num_words=max_words)\n",
    "x_train = tokenizer.sequences_to_matrix(x_train, mode=\"binary\")\n",
    "x_test = tokenizer.sequences_to_matrix(x_test, mode=\"binary\")\n",
    "print(\"x_train shape:\", x_train.shape)\n",
    "print(\"x_test shape:\", x_test.shape)\n",
    "\n",
    "print(\"Convert class vector to binary class matrix (for use with categorical_crossentropy)\")\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "print(\"y_train shape:\", y_train.shape)\n",
    "print(\"y_test shape:\", y_test.shape)\n",
    "\n",
    "print(\"Building model...\")\n",
    "model = Sequential()\n",
    "model.add(Dense(512, input_shape=(max_words,)))\n",
    "model.add(Activation(\"relu\"))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(num_classes))\n",
    "model.add(Activation(\"softmax\"))\n",
    "\n",
    "model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
    "\n",
    "history = model.fit(\n",
    "    x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_split=0.1\n",
    ")\n",
    "score = model.evaluate(x_test, y_test, batch_size=batch_size, verbose=1)\n",
    "print(\"Test score:\", score[0])\n",
    "print(\"Test accuracy:\", score[1])\n",
    "\n",
    "# Log the model\n",
    "# mlruns/0/run_id/artifacts/my_models/\n",
    "# otherwise the autolog() is saving the model at\n",
    "# mlruns/0/run_id/artifacts/model/\n",
    "# for mlflow 2.5.0\n",
    "# mlflow.tensorflow.log_model(model, artifact_path=\"my_models\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9310bebb-3891-4a4b-a922-a1af3ef33fa1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}