02-gradient-decent.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 梯度下降\n",
    "\n",
    "- 优化问题 - Optimization Problem 求解最小值的问题\n",
    "- 鞍点 - 梯度为 $0$ 的点\n",
    "- (mini) batch - 批量随机梯度下降中的批量s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict before training: 4.0\n",
      "Epoch: 0, Cost: 11.00, Gradient: -22.00, w: 1.22\n",
      "Epoch: 1, Cost: 6.69, Gradient: -17.16, w: 1.39\n",
      "Epoch: 2, Cost: 4.07, Gradient: -13.38, w: 1.53\n",
      "Epoch: 3, Cost: 2.48, Gradient: -10.44, w: 1.63\n",
      "Epoch: 4, Cost: 1.51, Gradient: -8.14, w: 1.71\n",
      "Epoch: 5, Cost: 0.92, Gradient: -6.35, w: 1.77\n",
      "Epoch: 6, Cost: 0.56, Gradient: -4.95, w: 1.82\n",
      "Epoch: 7, Cost: 0.34, Gradient: -3.86, w: 1.86\n",
      "Epoch: 8, Cost: 0.21, Gradient: -3.01, w: 1.89\n",
      "Epoch: 9, Cost: 0.13, Gradient: -2.35, w: 1.92\n",
      "Epoch: 10, Cost: 0.08, Gradient: -1.83, w: 1.93\n",
      "Epoch: 11, Cost: 0.05, Gradient: -1.43, w: 1.95\n",
      "Epoch: 12, Cost: 0.03, Gradient: -1.12, w: 1.96\n",
      "Epoch: 13, Cost: 0.02, Gradient: -0.87, w: 1.97\n",
      "Epoch: 14, Cost: 0.01, Gradient: -0.68, w: 1.98\n",
      "Epoch: 15, Cost: 0.01, Gradient: -0.53, w: 1.98\n",
      "Epoch: 16, Cost: 0.00, Gradient: -0.41, w: 1.99\n",
      "Epoch: 17, Cost: 0.00, Gradient: -0.32, w: 1.99\n",
      "Epoch: 18, Cost: 0.00, Gradient: -0.25, w: 1.99\n",
      "Epoch: 19, Cost: 0.00, Gradient: -0.20, w: 1.99\n",
      "Epoch: 20, Cost: 0.00, Gradient: -0.15, w: 1.99\n",
      "Epoch: 21, Cost: 0.00, Gradient: -0.12, w: 2.00\n",
      "Epoch: 22, Cost: 0.00, Gradient: -0.09, w: 2.00\n",
      "Epoch: 23, Cost: 0.00, Gradient: -0.07, w: 2.00\n",
      "Epoch: 24, Cost: 0.00, Gradient: -0.06, w: 2.00\n",
      "Epoch: 25, Cost: 0.00, Gradient: -0.04, w: 2.00\n",
      "Epoch: 26, Cost: 0.00, Gradient: -0.03, w: 2.00\n",
      "Epoch: 27, Cost: 0.00, Gradient: -0.03, w: 2.00\n",
      "Epoch: 28, Cost: 0.00, Gradient: -0.02, w: 2.00\n",
      "Epoch: 29, Cost: 0.00, Gradient: -0.02, w: 2.00\n",
      "Epoch: 30, Cost: 0.00, Gradient: -0.01, w: 2.00\n",
      "Epoch: 31, Cost: 0.00, Gradient: -0.01, w: 2.00\n",
      "Epoch: 32, Cost: 0.00, Gradient: -0.01, w: 2.00\n",
      "Epoch: 33, Cost: 0.00, Gradient: -0.01, w: 2.00\n",
      "Epoch: 34, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 35, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 36, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 37, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 38, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 39, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 40, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 41, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 42, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 43, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 44, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 45, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 46, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 47, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 48, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 49, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 50, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 51, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 52, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 53, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 54, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 55, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 56, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 57, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 58, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 59, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 60, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 61, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 62, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 63, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 64, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 65, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 66, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 67, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 68, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 69, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 70, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 71, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 72, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 73, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 74, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 75, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 76, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 77, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 78, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 79, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 80, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 81, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 82, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 83, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 84, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 85, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 86, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 87, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 88, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 89, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 90, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 91, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 92, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 93, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 94, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 95, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 96, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 97, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 98, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Epoch: 99, Cost: 0.00, Gradient: -0.00, w: 2.00\n",
      "Predict after training: 7.999999999935208\n"
     ]
    }
   ],
   "source": [
    "x_data = [float(i) for i in range(1, 6)]\n",
    "y_data = [2 * i for i in x_data]\n",
    "w = 1.0\n",
    "\n",
    "def forward(x):\n",
    "    return x * w\n",
    "\n",
    "def cost(x_data, y_data):\n",
    "    cost = 0\n",
    "    for x, y in zip(x_data, y_data):\n",
    "        y_pred = forward(x)\n",
    "        cost += (y_pred - y) ** 2\n",
    "    return cost / len(x_data)\n",
    "\n",
    "def gradient(x_data, y_data):\n",
    "    grad = 0\n",
    "    for x, y in zip(x_data, y_data):\n",
    "        grad += 2 * (forward(x) - y) * x\n",
    "    return grad / len(x_data)\n",
    "\n",
    "print(f\"Predict before training: {forward(4)}\")\n",
    "for epoch in range(100):\n",
    "    cost_val = cost(x_data, y_data)\n",
    "    grad_val = gradient(x_data, y_data)\n",
    "    w -= 0.01 * grad_val\n",
    "    print(f\"Epoch: {epoch}, Cost: {cost_val:.2f}, Gradient: {grad_val:.2f}, w: {w:.2f}\")\n",
    "print(f\"Predict after training: {forward(4)}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "vpteam",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "0.0.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}