Simple_3_L_ReLU_Neural_Network.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Simple 3-L ReLU Neural Network.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMMxrsynthWrgRpnF8qOS5P",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/oObelix/DataScience/blob/master/Simple_3_L_ReLU_Neural_Network.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X0g7EVTUixOf"
      },
      "source": [
        "# Simple custom 2 layer Neural Network\n",
        "# ReLU"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ly3Dl-hij2yg"
      },
      "source": [
        "import numpy as np"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LeTErz13l8QI"
      },
      "source": [
        "np.random.seed(1)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "86kCyccHliKk"
      },
      "source": [
        "# layer_0 = features[0]\n",
        "# layer_1 = relu(np.dot(layer_0, weights_0_1))\n",
        "# layer_1 = np.dot(layer_1, weights_1_2)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eF-HbKILqiTX",
        "outputId": "909ac6a5-55bf-487a-fe4b-ef63b36d0964"
      },
      "source": [
        "relu = lambda x: (x > 0) * x\n",
        "relu_deriv = lambda x: x > 0\n",
        "\n",
        "alpha = 0.005\n",
        "hidden_size = 4\n",
        "\n",
        "features = np.array([[1, 0, 1],\n",
        "                     [0, 1, 1],\n",
        "                     [0, 0, 1],\n",
        "                     [1, 1, 1]])\n",
        "values = np.array([1, 1, 0, 0]).T\n",
        "\n",
        "weights_0_1 = 2 * np.random.random((3, hidden_size)) - 1\n",
        "weights_1_2 = 2 * np.random.random((hidden_size, 1)) - 1\n",
        "\n",
        "for iter in range(50):\n",
        "  for i in range(len(features)):\n",
        "    layer_0 = features[i:i+1]\n",
        "    layer_1 = relu(\n",
        "              np.dot(layer_0, weights_0_1)\n",
        "              )\n",
        "    layer_2 = np.dot(layer_1, weights_1_2)\n",
        "\n",
        "    error = (layer_2 - values[i:i+1]) ** 2\n",
        "\n",
        "    layer_2_delta = layer_2 - values[i:i+1]\n",
        "    layer_1_delta = layer_2_delta.dot(weights_1_2.T) * relu_deriv(layer_1)\n",
        "\n",
        "    weights_1_2 -= alpha * layer_1.T.dot(layer_2_delta)\n",
        "    weights_0_1 -= alpha * layer_0.T.dot(layer_1_delta)\n",
        "\n",
        "  # if (iter % 10 == 0):\n",
        "  print(\"Error:\", error)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Error: [[0.62951773]]\n",
            "Error: [[0.61615536]]\n",
            "Error: [[0.60357258]]\n",
            "Error: [[0.59171263]]\n",
            "Error: [[0.58052386]]\n",
            "Error: [[0.56995915]]\n",
            "Error: [[0.55997547]]\n",
            "Error: [[0.55053344]]\n",
            "Error: [[0.54159696]]\n",
            "Error: [[0.53313288]]\n",
            "Error: [[0.52511073]]\n",
            "Error: [[0.51750242]]\n",
            "Error: [[0.51028207]]\n",
            "Error: [[0.50342576]]\n",
            "Error: [[0.49691139]]\n",
            "Error: [[0.49071847]]\n",
            "Error: [[0.48482801]]\n",
            "Error: [[0.47922238]]\n",
            "Error: [[0.47388521]]\n",
            "Error: [[0.46880125]]\n",
            "Error: [[0.46395629]]\n",
            "Error: [[0.45933709]]\n",
            "Error: [[0.4549313]]\n",
            "Error: [[0.45072736]]\n",
            "Error: [[0.44671447]]\n",
            "Error: [[0.44288252]]\n",
            "Error: [[0.43922203]]\n",
            "Error: [[0.43572412]]\n",
            "Error: [[0.43238043]]\n",
            "Error: [[0.42918314]]\n",
            "Error: [[0.42612488]]\n",
            "Error: [[0.42319872]]\n",
            "Error: [[0.42039813]]\n",
            "Error: [[0.41771695]]\n",
            "Error: [[0.4151494]]\n",
            "Error: [[0.41268999]]\n",
            "Error: [[0.41033357]]\n",
            "Error: [[0.40807524]]\n",
            "Error: [[0.40591041]]\n",
            "Error: [[0.40383469]]\n",
            "Error: [[0.40184396]]\n",
            "Error: [[0.39993431]]\n",
            "Error: [[0.39810202]]\n",
            "Error: [[0.39634359]]\n",
            "Error: [[0.39465568]]\n",
            "Error: [[0.39303512]]\n",
            "Error: [[0.39147891]]\n",
            "Error: [[0.3899842]]\n",
            "Error: [[0.38854827]]\n",
            "Error: [[0.38716854]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AGGjsNiSx9b6"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}