tensorflow2/010-popular-activation-functions/popular-activation-functions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Popular Activation functions\n",
    "\n",
    "This is a notebook for the medium article [7 popular activation functions you should know in Deep Learning](https://bindichen.medium.com/7-popular-activation-functions-you-should-know-in-deep-learning-and-how-to-use-them-with-keras-and-27b4d838dfe6)\n",
    "\n",
    "Please check out article for instructions\n",
    "\n",
    "**License**: [BSD 2-Clause](https://opensource.org/licenses/BSD-2-Clause)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "z = np.linspace(-7, 7, 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def derivative(f, z, eps=0.000001):\n",
    "    return (f(z + eps) - f(z - eps))/(2 * eps)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Sigmoid (Logistic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(z):\n",
    "    return 1 / (1 + np.exp(-z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApMAAAEJCAYAAADIA6xFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZgU1fX/8fdhAEV2EVABxSiouAPuGkBNgitZUPSnuESDGNHENQqKikCMW6IBt6BB44KKiV9c4s5IFlzABREV0KCAyiagiCzDnN8ft8dpmplhprumq5fP63nq6arqmqrTPdNnTlfVvdfcHRERERGRdDSIOwARERERyV8qJkVEREQkbSomRURERCRtKiZFREREJG0qJkVEREQkbSomRURERCRtKiazzMxKzWxM3HFA7WIxs5lmdm2WQko+7ngzezoLx+ltZm5m22ThWIPM7DMzK4/jPU2J5UwzWxVnDCJSP7L5+U7kz/7ZOJbkLlM/k9Exs7bAdcAxwHbACmAmcIO7v5jYZmtgvbt/E1ugCbWJxcxmAhPd/dp6iqE3MBlo6+5Lk9a3JPx9rojwWPOAMe5+c9K6xsDWwCKvxw+DmbUGFgMXAxOBb9w9a8keONHdJyatawI0d/fF2YhBRCqZ2XjgjMRiGbAceJ+QG+5x9/UZ7j/yz3ci5m3c/biU9dsCy919bVTHkvzTMO4ACswTwFbA2cBcoB3QC2hTsYG7fxVPaJvKpVhSufvKLB1nHfBlFg61I+Hz9rS7f5GF49XI3b8Dvos7DpEi9hIwECgB2gJHEE5GDDSzI93923R2amaNsvn5dvds5E/Jde6uKYIJaAU4cNRmtislnB2rWG4PTCJ88D8FziKczbw2aRsHzgP+D1gNzAb6AB2B54FvgXeA7inH+jnwHrAWmA8MI3E2uppY2iWOURHLL1NjqeL17Jz4mS8TcbwFHJeyTWNgdGKfa4FPgAuBzonXljyNT/zMeELhBTAIWASUpOz3YWBSbeJIvNaNjpVY3zuxvE0d3rd5wFXA3cDXwALgshreozOreJ2dgWuBmVVsuypp+drE7+Bk4GPgG+DJ5HgT252RFPMi4P6kWJOPO6+q4yTWnUv4ErQu8firlOc98bt4PPEefwKcFvdnT5OmfJuS81vK+j0Tn7/rEsuNgT8kcsxq4E3gJ0nbV+SvY4A3Ej97XPLnG+ia2GavlGMNApYCjQgF7b3A/wj5fw5wOdAgse21VeSw3onnHOifmP8vcEvKcVok9vnz2rwmTfk56Z7J6KxKTCeY2ZZ1+Ln7CWetjgD6AaclllNdBUwA9gGmJebvBe4A9gM+JyQoAMysB+Gf/t+BvYArgCuBITXEMh7YBTgK+ClwOqHoqUkz4J/AjxKxPQH83cx2S3mNpxMu8e5OOHO7glCo/SKxzR6EWwN+U8UxHgdaJo5R8fqaEd6vB2sZx88JyWtE4jjbVfVi6vC+XUQo3roTEuONZnZwVfsEHgX6JuYPSBx7fjXbVqUzMAD4GfBjwu97VFLM5xIK278CexP+scxMPL1/4vFXieNWLG/EzH4GjAH+RPiHdhtwh5kdn7LpcELRvk/idd1nZjvU4bWISDXcfSbwHJV58a+Eq1v/j/C5vB94ysz2SfnRPxD+R+wGvJ6yz9mEgu3UlJ85FXjMwyX1BsBC4CRCjh4GDCWc3AC4GXiMcDa1In/+t4qX8CBwspkl1xa/ANYAz9TxNUk+ibuaLaSJ8KH5ivDBmUr4AB6Ysk0pibOBwK6Eb3UHJT3fCdjApmcmf5+0vGdi3cVJ63qTdIYNeAh4JeXY1wILqoml4tvroUnP75gaSy3fh9eAqxLzXRL77VvNthvFnbR+PEnf3AnF3d+Slk8DVgJb1iaOxPI84NKajl/L920e8EjKNnOSj1VFLD0Tx+mcst/anJlcA7RMWjcMmJu0vIBwX251x/7+zEENx/kPcF8Vv4N/1/B32JBwZkFnJzVpqsOUmt9Snrsh8bnaGSgHdkh5/kngjsR8Rf76Rco2qZ/vCwlXhiraSeyQ2PchNcR4A/DS5mJm4zOTbQhnR49Mev4lwn2g1OY1acrPSWcmI+TuTwDbA8cTzpIdArxmZkOr+ZHdCB+saUn7mE84y5hqRtL8osTje1Wsa5d43J1QICT7N9DBzFpUsf/dE7G8kRTLp9XE8j0za2pmN5rZLDNbnmhB2JOQrCCcRSsnNLLJxIPAT81sq8TyqcAT7r6mlnHUVm3ftxkp23xO5XsftU9943tIvz+WmbUDOgAvZ3iM6l53t5R1379udy8DllB/r1ukGBmhQOuemJ9lZqsqJuBYQlGWbBo1m0D433R4YvkU4H/u/v3ZRTMbbGbTzGxJ4jgXUcf86e7LCGdWT03sc3vCLVkVV5Dq8pokj6gBTsQSxc2LiWmEmY0DrjWzmz009khXcus+r2Fdbb4geJrPVeVmwiXcSwln51YDDxDui4nSM4RWj/3M7GXCpfifZDmO5PcmtbWlU/eutsoJiTVZoyq2i+JY6Ur9e4gzFpFi0I1wP3IDwudrfzb93KU2rqmxsY67LzazFwlF3pTE40MVz5vZAMItLpcSLl9/DZxPuLWmrh4E/mJmvybc6z0f+Ffiubq8Jskj+idQ/2YRivaq7qP8kPA76FGxwsw6Er5BZuoD4NCUdYcRLtdW1RVQRSwHJMWyQy1iOQx4wN2fcPcZhEuuyd8w30nst081P19RYJfUdBAP3U48TkiCAwgNbUrrEEfFsWo8DnV/3zKxBGhvZskF5b512YGHrj8WAkfWsNl60n/ds+oSj4ikz8z2JHwpngi8Tfiyua27z02ZFqax+weBExP3he9F5dlCCJ/11919jLu/5e5zSS9/QmhQCqEh0KnAw+5e8aU06tckOULFZETMrI2ZvWJmp5nZ3ma2k5mdSGgR97K7f536M+7+EaE19l1mdpCZ7Uu4OXk1dT9DmOoWoJeZXWtmXc3sVOAS4MaqNk7E8hxwt5kdnIhlPJv/tjgb+JmZdTezigT1feHs4ebvx4BxZvaLxPtyuJkNTGzyKeG1HmtmbRMNa6rzIOFs5GDCPYvltY0jYR5wuJl1qKGT8jq9bxkqJfRxOdTMdjazs4F0Ov8dBfzWzC5KxLyvmV2S9Pw84Egz2zbR32VVbiJ0SXK+mXUxswsI/wjq43WLCGyR+Exub2b7mNnFhJwwHbg5kTsfAsabWX8z+4GZ9TSzS83s52kc70nClY97gTcT+68wG+huZkcnPv9XExrJJJsH7Glmu5rZNmZW1VWUiqtzTxAaBHUnqWith9ckOULFZHRWERp8/AZ4ldAB7WhC9zUDavi5Mwln0UoJ3+geInRuvSaTYNz9LeBEQqOgmYSbqW8gtNitKZb/Aa8ATyVin7eZQ12ciPdfhPtEX6PykkaF0xP7up1wBnQ8oXU2iW+j1xAKokWbie9fhLNw3dj4W3Vt4xhOaOD0MeGs4CbSfN/S4u4fELp8GkS4F/FHhL+Zuu7nTsIlqV8RYn6O0Dq+wiWEM8PzCWcGqtrHk8AFhPukZhH+jn/t7k/VNR4RqZWjgC+Azwj3PJ9AaHD3Q6/sY/IswgmGGwm582ngh4Qv4XXi7quBfxB6YkjNn3cTvvQ/TGj53ZnwxTrZXwhXMKYR8mfqlYxkDyaO87a7p17diOw1Se7QCDg5JnHG7HPglESDHhEREZGcpQY4MTOzI4DmhJbZ7Qhn6JYSzi6JiIiI5LRILnOb2X1mttjCOM5VPX+qmc0ws/fM7L/qnHQjjYCRhGLyKcL9ksmXOUSkAClvikihiOQyt5n9kHDP4APuvmcVzx8CfODuy83saEIn2AdmfGARkTylvCkihSKSy9zuPsXMOtfwfPKwS68RxpQWESlaypsiUijiuGfybEJr202Y2SBCy1aaNGnSo1OnTpEdtLy8nAYNcrfxuuLLzOzZs+natWvcYVQr19+/Yopv9uzZS929bSQ7y55q8yYodyq+9Cl3pi+XY4Po46sxd0Y1LiOhK4GZm9mmD6FrgTab21+PHj08SpMnT450f1FTfJkJf8q5K9ffv2KKD5jmOTCWrddD3nTlzpyT6/Epd6Yvl2Nzjz6+mnJn1s5MmtnewDjgaA/jd4qISA2UN0UkH2Tl/GxiWL6/AwN94173RUSkCsqbIpIvIjkzaWaPAL2BbcxsAWFEk0YA7n4XYeSRNsAdiWGIy9y9ZxTHFhHJR8qbIlIoomrNfcpmnj8HOCeKY4mIFALlTREpFLnbDElEREREcp6KSRERERFJm4pJEREREUmbikkRERERSZuKSRERERFJm4pJEREREUmbikkRERERSZuKSRERERFJm4pJEREREUmbikkRERERSZuKSRERERFJm4pJEREREUmbikkRERERSZuKSRERERFJm4pJEREREUmbikkRERERSZuKSRERERFJm4pJEREREUmbikkRERERSZuKSRERERFJm4pJEREREUmbikkRERERSVskxaSZ3Wdmi81sZjXPm5ndbmZzzWyGmXWP4rgiIvlKeVNECkVUZybHA31reP5ooEtiGgTcGdFxRUTy1XiUN0WkADSMYifuPsXMOtewST/gAXd34DUza2Vm27n7F1EcX0TylzuUlcF338HatbB+fViueCwrg7lzm9Gs2abPpT5u2ADl5WGqbj5XKG+KSKGIpJishQ7A/KTlBYl1GyVFMxtE+AZO06ZNMbPvn7vrrrsAGDx48PfrzjjjDM4880z69+/PsmXLAOjSpQv33HMPN998M88888z3244fP57Ro0czbNiw79ddfPHFHH/88fTp0+f7dQcffDCjR49m6NChTJ069fv1kydP5qmnnuLWW2/9ft2oUaPo2rUrJ5544vfrjj32WC699FIGDRrEnDlzAGjTpg0TJ05k/Pjx3H///ZG9pscff5zZs2dn5TUNHjyYrl275vRrMrOc/T3NmDFjo/jz+2+vAePGPcl7733Bbbc9CLQAWnDkkT9n55334557Hkmsa87WW3dm55334cMPP+Wbb9YDWwJNaNu2E998s4E1axxoApRQs56beb4g1Spvwsa5s3379pSWlkYWxKpVqyLdX9QUX+ZyOb5cfv9yOTbIbnwWvvRGsKPwDftpd9+ziueeBm5w938nll8Gfufu06rbX8+ePX3atGqfrrPS0lJ69+4d2f6ipvgyY2ZE9bdcH3L9/SstLeXgg3uzcCEsWADz51c+LlwIS5aEaelSWL48nE2MUsOG0KQJbLEFNGoUlpMf16xZRevWzTZZn/zYsCGUlISpQYPKKXV5zBib7u45UZ1GnTdBuTPX5Hp8yp3py+XYIPr4zKrPndk6M7kQ6JS03DGxTkSyaP16+PhjmD0bPvqocnr//UNYvrx2+zCDNm1gm23CY6tW0KJF5dS8+cbLTZuGQnHLLcNjVfMNN5OJSkunRZYUx4yJZDfZoLwpInkhW8XkJGCImU0ADgRW6r4fkfq1bh3MnAnTp1dOM2aE9ZtqTEkJbL89dOwInTpVPnboAO3bQ9u2oYBs3XrzxZ9EQnlTRPJCJP8SzOwRoDewjZktAK4BGgG4+13As8AxwFxgNXBWFMcVkUrffgv//S+UlsKrr8Kbb1ZdOO64I+y668bTsmVT+cUvDlaRmEXKmyJSKKJqzX3KZp534PwojiUilebMgaeegkmT4D//CS2aK5iFQrF7d+jRI0z77hsuS6cqLV2rQjLLlDdFpFDo34dInpk1Cx58EP7xD/jww8r1DRpAz57Quzf06gWHHVZ14SgiIhIlFZMieWDJEpgwAR54AJIb6rZqBcccAyecAD/5iYpHERHJPhWTIjnsnXfgj3+ERx4JLbEBWraEAQPg5JPD2cdGjeKNUUREipuKSZEcU14OzzwTisjJk8O6Bg3g2GPhjDPg+ONDdzoiIiK5QMWkSI5wD0Xk1VeHM5IQ+mw8+2y48ELYaad44xMREamKikmRHDB1Klx8Mbz2Wljefnu45JJQSLZsGW9sIiIiNVExKRKjzz+H3/0utM6G0DH40KFw7rlhdBgREZFcp2JSJAbl5XD33XD55bBqVRiT+tJL4YoroFmzuKMTERGpPRWTIlk2d264fD1lSlju1w9uvRV+8IN44xIREUmHikmRLHEP/USef34Y+rBdOxg7Fvr3jzsyERGR9KmYFMmC1atLGDgQHnooLA8YEArJNm3ijUtERCRTKiZF6tmcOfDrX3fn009hq61gzBg488wwdraIiEi+UzEpUo+efz6MVLNiRVP22AMmToTddos7KhERkeg0iDsAkUJ1xx1h3OwVK+DQQ5cydaoKSRERKTw6MykSMXcYPhxGjgzLV10FffrMpHnz3rHGJSIiUh90ZlIkQhs2hA7HR44M42mPGwfXXx/mRURECpHOTIpEZMOG0LDmwQdhyy3h0UfhhBPijkpERKR+qZgUiUByIdm0Kfzzn3D44XFHJSIiUv9UTIpkqLwczjqrspB87jk47LC4oxIREckO3cklkgF3+O1v4W9/UyEpIiLFScWkSAZ+/3v485+hcWOYNEmFpIiIFB8VkyJpuvdeGDYsjGTz0ENwxBFxRyQiIpJ9kRSTZtbXzD4ys7lmdkUVz+9gZpPN7G0zm2Fmx0RxXJG4vPJK6AIIwhjb/fvHG4/kJ+VOESkEGReTZlYCjAWOBroBp5hZt5TNrgIec/f9gJOBOzI9rkhc5s4NxeOGDXD55XDeeXFHJPlIuVNECkUUZyYPAOa6+yfuvg6YAPRL2caBFon5lsDnERxXJOtWroTjj4fly8Pj6NFxRyR5TLlTRAqCuXtmOzDrD/R193MSywOBA919SNI22wEvAK2BpsBR7j69in0NAgYBtG/fvseECRMyii3ZqlWraNasWWT7i5riy0yfPn2YPHlyvR6jvByuumpPpk7dhp12WsWYMW+z1VYbavWzuf7+FVN8ffr0me7uPSPZWQaUO6Oh+DKTjdyZiVx+/3I5Nog+vhpzp7tnNAH9gXFJywOBMSnbXAxckpg/GJgFNKhpvz169PAoTZ48OdL9RU3xZSb8Kdev3//eHdxbt3b/5JO6/Wyuv3/FFB8wzTPMe1FMyp3RUHyZyUbuzEQuv3+5HJt79PHVlDujuMy9EOiUtNwxsS7Z2cBjieJ1KrAlsE0ExxbJitLS0HIbQp+SO+0UazhSGJQ7RaQgRFFMvgl0MbOdzKwx4SbxSSnbfAYcCWBmuxMS4pIIji1S7778Ek45JVzmvvJKOPbYuCOSAqHcKSIFIeNi0t3LgCHA88AHhJaH75vZCDM7IbHZJcCvzOxd4BHgzMQpU5GcVl4OZ5wRCspevWDEiLgjkkKh3CkihSKSsbnd/Vng2ZR1w5PmZwGHRnEskWz685/hhRegTRt4+GFoqNHsJULKnSJSCDQCjkg13nsPfve7MD9uHGy/fbzxiIiI5CIVkyJVWLMGTj0V1q6Fc86Bn/407ohERERyk4pJkSpcd104M7nLLvDHP8YdjYiISO5SMSmSYto0uOkmMIP774cc7pNWREQkdiomRZKsWwe//GUYd/u3v4VDDok7IhERkdymYlIkyahR4fL2zjvDyJFxRyMiIpL7VEyKJLz/PoweHebvvRe22ireeERERPKBikkRwB3OOw/KymDw4NBBuYiIiGyeikkRwnjb//oXtG1beXZSRERENk/FpBS9r76CSy8N8zffDK1bxxuPiIhIPlExKUVv2DBYsgR++EMYODDuaERERPKLikkpam+8AXffHcbcvuOO0LekiIiI1J6KSSlaGzaERjfucMklsMcecUckIiKSf1RMStH6y1/grbdghx3g6qvjjkZERCQ/qZiUorRyJQwfHuZvuQWaNo03HhERkXylYlKK0qhRodHN4YfDL34RdzQiIiL5S8WkFJ1PPoHbbgvzt96qRjciIiKZUDEpRefyy2HdOjj9dOjZM+5oRERE8puKSSkqU6bAE0+Ecbc10o2IiEjmVExK0Sgvh4suCvOXXw4dOsQbj4iISCFQMSlF429/C10BdehQOXyiiIiIZEbFpBSFtWvhmmvC/OjR6gpIREQkKpEUk2bW18w+MrO5ZnZFNducZGazzOx9M3s4iuOK1NZf/gKffhpGuTn11LijEQmUO0WkEDTMdAdmVgKMBX4ELADeNLNJ7j4raZsuwJXAoe6+3MzaZXpckdr69lsYOTLMjxwJJSXxxiMCyp0iUjiiODN5ADDX3T9x93XABKBfyja/Asa6+3IAd18cwXFFamXMGFi0CPbfH/ql/mWKxEe5U0QKQhTFZAdgftLygsS6ZF2Brmb2HzN7zcz6RnBckc1auRL+8IcwP2qUOiiXnKLcKSIFIePL3HU4ThegN9ARmGJme7n7iuSNzGwQMAigffv2lJaWRhbAqlWrIt1f1BRf5qqK7777OrN8eWf23Xc5DRu+S1wvIdffP8WXs5Q7N0PxZS6X48vl9y+XY4Msx+fuGU3AwcDzSctXAlembHMXcFbS8svA/jXtt0ePHh6lyZMnR7q/qCm+zIQ/5Y0tXuzerJk7uP/nPzEElSTX379iig+Y5hnmvSgm5c5oKL7MVJU7c0kuv3+5HJt79PHVlDujuMz9JtDFzHYys8bAycCklG2eJHyzxsy2IVy6+SSCY4tU64YbYNUqOPZYOOSQuKMR2YRyp4gUhIyLSXcvA4YAzwMfAI+5+/tmNsLMTkhs9jywzMxmAZOBy9x9WabHFqnOggUwdmyYr2jJLZJLlDtFpFBEcs+kuz8LPJuybnjSvAMXJyaRenf99aGj8pNOgn33jTsakaopd4pIIdAIOFJw5s6F++6DBg1gxIi4oxERESlsKial4Fx7LZSVwZlnwq67xh2NiIhIYVMxKQVl5kx4+GFo1AiGD9/89iIiIpIZFZNSUK6+Gtxh8GDYcce4oxERESl8KialYLzxBjz5JGy1FQwdGnc0IiIixUHFpBSMYcPC44UXwrbbxhuLiIhIsVAxKQWiNy+9BC1bwmWXxR2LiIhI8VAxKXkvjDI3CgiF5NZbxxqOiIhIUVExKXnvmWcADqFtW/jNb+KORkREpLiomJS8Vl4OV10V5ocOhWbN4o1HRESk2KiYlLz2+OPw7rsA8xk8OO5oREREio+KSclbZWXJHZOPYMst44xGRESkOKmYlLz1wAMwezbssgvA+JijERERKU4qJiUvrV0bxuAGGDECoCzGaERERIqXiknJS3ffDfPnw157wYABcUcjIiJSvFRMSt759lsYFbqVZORIaKC/YhERkdjo37Dkndtvh8WL4cAD4fjj445GRESkuKmYlLyyYgXceGOYHzUKzOKNR0REpNipmJS8cvPNoaA84gg48si4oxEREREVk5I3Fi2CP/0pzFfcMykiIiLxUjEpeeP660PjmxNOgIMOijsaERERARWTkic++SR0B9SgAYweHXc0IiIiUiGSYtLM+prZR2Y218yuqGG7X5iZm1nPKI4rxePqq8PwiaefDnvsEXc0ItFQ7hSRQpBxMWlmJcBY4GigG3CKmXWrYrvmwG+A1zM9phSXd96Bhx+Gxo0rR70RyXfKnSJSKKI4M3kAMNfdP3H3dcAEoF8V210P/AFYE8ExpYgMHRoezz8fdtwx3lhEIqTcKSIFIYpisgMwP2l5QWLd98ysO9DJ3Z+J4HhSRF59Ff75T2jevLKoFCkQyp0iUhAa1vcBzKwBcCtwZi22HQQMAmjfvj2lpaWRxbFq1apI9xc1xbcpdxgyZD+gJf37/4+ZMz+tcXu9f+lTfLlHubN2FF/mcjm+XH7/cjk2yHJ87p7RBBwMPJ+0fCVwZdJyS2ApMC8xrQE+B3rWtN8ePXp4lCZPnhzp/qKm+Db1j3+4g3u7du7ffFPztuFPOXfp95uZKOMDpnmGeS+KSbkzGoovM8qd6cvl2Nyjj6+m3BnFZe43gS5mtpOZNQZOBiYlFasr3X0bd+/s7p2B14AT3H1aBMeWAlVWVnlZ++qroVmzeOMRqQfKnSJSEDIuJt29DBgCPA98ADzm7u+b2QgzOyHT/UtxeuAB+OAD2GknGDQo7mhEoqfcKSKFIpJ7Jt39WeDZlHXDq9m2dxTHlMK1ahVcdVWYv/760CWQSCFS7hSRQqARcCTn3HgjfPEF7L8/nHJK3NGIiIhITVRMSk757DO46aYw/8c/huETRUREJHfpX7XklCuvhDVr4KST4NBD445GRERENkfFpOSM118PwyZusQX84Q9xRyMiIiK1oWJScoI7XHRRmL/4YujcOdZwREREpJZUTEpOePRRmDoV2rcPl7pFREQkP6iYlNitWgWXXRbmR44M43CLiIhIflAxKbEbMQIWLICePeGss+KORkREROpCxaTEaubM0AWQGdx5J5SUxB2RiIiI1IWKSYmNO/z612Ec7vPOC2cmRUREJL+omJTYPPAA/Otf0K4djBoVdzQiIiKSDhWTEouvvqpsdHPLLdCqVbzxiIiISHpUTEoshg6FJUugVy849dS4oxEREZF0qZiUrJsyBe6+Gxo2hDvuCI1vREREJD+pmJSsWr0azj47zA8dCt26xRuPiIiIZEbFpGTV1VfD3Lmw554wbFjc0YiIiEimGsYdgBSPqVNDn5IlJfDXv0LjxnFHJCJSnNasge++A9gSd91uJJlRMSlZ8e23cOaZoW/Jyy5Tn5IiItngDu+9B//+d5hmzIDPP4flyyu2+I4GDaBDB9hjj3DVqFcvOPJIaNo0zsgln6iYlKy45BKYPTskq2uuiTsaEZHCNmcOPPggPPQQfPzxps83agRbbQUrV64BtmThQli4EF54AW69FbbcEo44Igxx269f2F6kOiompd5NmhRabzduDA8/HJKUiIhE71//ghtugGefrVy37bbhTONhh8H++8MOO0CbNtCgAZg1oazMmTcvDG/71lvw3HPwxhthH88+C9tvD+eeCxdcAK1bx/bSJIepmJR69eWXla23f/972HvveOMRESlEU6fC5ZeHS9kATZrASSfBwIHQu3e4V706JSWw885h6tcPrrsu5O7HHoM774QPPwxXlP74R7j0UrjwQmjePCsvS/KEWnNLvdmwISSypUvhqKPgt7+NOyIRkcIybx6cfDIcckgoJFu3huHD4UZiUL8AABWPSURBVNNPYfz4cEaypkKyOttuG4rGWbPg5ZfDJe8VK+Cqq2CXXcJwuO5RvxrJVyompd6MGAEvvQRt24ak1kB/bSIikSgrgxtvDH31PvpouH1o2LBQRF53Xci7UTALheTLL4fpoINg8WI44wz44Q9D4x6RSP69m1lfM/vIzOaa2RVVPH+xmc0ysxlm9rKZ7RjFcSV3vfACXH99SEQPPxxaCorIxpQ7JR3vvAMHHAC/+13o3ufkk+Gjj2DkyPq9/HzEEfDf/8L990O7duFMaI8eMHp0KG6leGVcTJpZCTAWOBroBpxiZqnjmrwN9HT3vYGJwI2ZHldy1/z5Ybxtd7j22nCJW0Q2ptwpdeUOf/pTKCTffht23DE0kHnkkdCoJhvM4PTTQ/F67rmwfn04I3rooeHeSilOUZyZPACY6+6fuPs6YALQL3kDd5/s7qsTi68BHSM4ruSgb78NN3AvXQo//nG4v0ZEqqTcKbW2ZAkcdxxcdFEo4M47L7S+PvroeOJp1Qruuguefx46dgytv/fbLxS75eXxxCTxMc/wDloz6w/0dfdzEssDgQPdfUg1248BvnT3kVU8NwgYBNC+ffseEyZMyCi2ZKtWraJZs2aR7S9qhRBfeTlcd90eTJnSlu23/44775xOixbZufbRp08fJk+enJVjpaMQfr9xijK+Pn36THf32LvNV+6MRjHE99ZbrRg9eneWLduC5s3Xc/nlH3HYYUsjiS+K3LlqVUPGjt2Z557bDoCDDlrGFVd8SMuW6zOOL5d/v7kcG0QfX425090zmoD+wLik5YHAmGq2PY3w7XqLze23R48eHqXJkydHur+oFUJ8w4e7g3uLFu6zZtV/TMnCn3LuKoTfb5yijA+Y5hnmvSgm5c5oFHJ869a5Dx3qbhZy6+GHu3/2WXSxuUebO//xD/fWrUOsHTq4v/pq5vvM5d9vLsfmHn18NeXOKC5zLwQ6JS13TKzbiJkdBQwDTnD3tREcV3LI+PGh9XaDBqFl4e67xx2RSM5T7pRqzZsXhjUcPTrcp3jNNfDKK9Cp02Z/NDY//WloHHTIIWE0nT59QkPMDRvijkzqWxTF5JtAFzPbycwaAycDk5I3MLP9gLsJyXBxBMeUHPL003DOOWH+ttugb9944xHJE8qdUqWJE2HffUNH5B06hCLy2muhYR4MM7LDDlBaCkOHhgZDw4eH++e/+CLuyKQ+ZVxMunsZMAR4HvgAeMzd3zezEWZ2QmKzm4BmwONm9o6ZTapmd5Jn/vvfMMrChg2hRd+QKu/2EpFUyp2SavXq0EL6xBNh5Uo44QR4991whjKfNGoEo0aFxjnt2oVieJ99wrIUpki+57j7s8CzKeuGJ82rc5gCNH06HHts6OfsnHPC5QwRqT3lTqkwc2boL/L996FxY7jlFjj//HCJO1/96EehGD7ttNDhed++YcjH668Pr1EKh8YkkbRMnx76j1yxAn7+8zB+az4nPRGROLiHLnb23z8UkrvuGrrZGTKkMHLqttuGM5IjR4ZhHW+8EQ47DD7+OO7IJEoqJqXOkgvJn/0sdJibD/fyiIjkkqVLw5fx886DNWvgl78M+XWffeKOLFolJeE2qClTQkfrb74Z+qR8+OG4I5OoqJiUOpk8ObTQqygkJ0zQ5QoRkbp68UXYe2948klo0SJ8Kb/3XmjaNO7I6s8hh4TW3v37wzffhJHSzjoLVq2KOzLJlIpJqbXHHw/3vHzzTWh0o0JSRKRu1q6FSy6pbOF82GHhvsKTT447suxo1QoeewzuuQeaNAndynXvDm+9FXdkkgkVk7JZ7jBxYkcGDIB16+CCC8K3aBWSIiK1N2tWGFf71lvDpd+RI0M3Op07xx1ZdpnBr34F06bBnnvCnDlw0EGhBXhZdgZNk4ipmJQarV0b7uMZO3YX3EPyu+220Dm5iIhs3vr18PvfhzNwM2bAzjvDf/4T7iMsKYk7uvh06xYaG51/fniPrroqFJXvvx93ZFJXKgmkWgsXQu/e4TLEFltsYMKEkPwKoYWhiEg2zJ3bjAMPDJ14r10LZ58Nb78NBx4Yd2S5oUkTGDMGXnopdHg+fXooum+4IRSYkh9UTEqVnnoqtCh87bUwfNef//w2AwbEHZWISH5YvTqcaRs8uDtvvx1aMb/wAowbB82bxx1d7jnySHjvPRg0KNxOdeWV0KMHvPdei7hDk1pQMSkbWb0afvObMPLCsmXwk5+E+1q6dFFzOxGRzXGHJ56A3XcP9wCWlxsXXBA6Jf/Rj+KOLre1aAF33x36pdxpp1BcXnhhd84+O3SjJLlLxaR879VXw9nI228P/UbedBM8+2wYDktERGo2a1YoGPv3h88+C13/3Hbb29x+OzRrFnd0+ePHPw73TV59NTRqVM5994XO3G+/PZy1lNyjYlJYtgwGDw73R86dG1rXTZ0Kl16qhjYiIpuzYEEYU3vvvcOwga1bw9ix4f6/vfb6Ou7w8lKTJjBiBIwb9yZHHglffRWumu2+Ozz6KJSXxx2hJFOpUMTKykLC69IlXFpo2BCuuSYkwJ49445ORCS3LV4MF10Eu+wS+k10D1/M58yBX/9aI4NFYYcdvuPFF0Pn7rvtBp98EvrkPPDAcDncPe4IBVRMFqXy8tBp7F57hfFfly8PNz+/8w5ce636jxQRqcn8+XDZZfCDH8Cf/hRaaZ90Urg0e+ed0KZN3BEWFjPo1y/cQ3nPPbDdduFe/r59w4mPJ57Qmcq4qZgsImVloYjs3h0GDIAPPwzJ8O9/D0N77bFH3BGKiOSud96BgQND3rz5Zvj2Wzj22DB6y6OPhjNnUn8aNgydnc+ZE7oOat8+vPf9+4f/X+PGhUakkn0qJovAypVwyy2ho9wBA8LQXR06wF13wQcfhDG21XekiMimvvsOHnww3FO+335hvrw85NI33oCnnw7rJXuaNoXf/Q7+979wq9aOO4aTI7/6FWy/fbi38oMP4o6yuKiYLFDu8Prr4b6djh1DY5rPPgv3R44dGxranHuuLmmLiKRyD5dRhwwJl1QHDgy9XTRtGgqVjz+GCRNg//3jjrS4NWkS/sfNmQN/+1sYPWflytDqu1s36NUrXBb/6qu4Iy18uj24wPzvf/Dww/DAAzB7duX6I44IN4ofc4xaaIuIpKooIB9/HCZODLm0wv77wznnhIYfLdSHds5p1AhOOy1Mb78drro99BBMmRKmIUNCn8mnnBL+B7ZqFXfEhUfFZJ4rL4c334RJk8KoNe+9V/ncttvCqafCGWeExjYiIlJp5Up45RV47rkwffZZ5XPbbhsa1Zx9dujyR/LDfvuF3kluuim0B3jkkTBU49NPh6mkBA47DI47Ltzvuttuus0rCiom84x7uBektDRMr74auqeo0Lx5+JCcfjocdZS6phARqbBiRehD99//Dmespk6FDRsqn99uu9CYo39/OPTQUHhIfmrRAs48M0yLFoXGp088EX73r74apssuC7/zXr3CPbG9eoXO0VVc1p1KjRzmHrqgeOut0Pfj9OnhMsySJRtvt+OOcPzxYQjEXr10H6SIyMqVobHhO++Eadq0MKRhcr+EFWepfvKT0M1M9+66DagQtW8PF1wQphUrwhjpTz8dzkZ/8UW4/3XChLBt69bh76BHj8rpBz9Qgbk5KiZzQHk5fPnlljz/PHz0UeX07rtVj0e63XaV36J694auXfWHLiLFxx2WLm1MaWlohDFnTrhXfMaMje95rNC4ceiX8LDDwpnHXr2gZcushy0xatUq3L5w0kmVV/pefTVc6ZsyBb78Moxi9PLLlT/TsmUYeWfXXcNl8YrHdev0j7eCisl6tn59GK5w0aIw5Nb8+Zs+zp8Pa9YcVOXPt2698Tek7t31LUlECt/69aEV7qJFsHBhyJfJ08KFMG8efPvtIVX+/BZbhHvF9923curRA7bcMruvQ3KXWWj13a0bnHdeKC4XLqy8ElgxLVoEr70Wpo31on172GEH6NQpPFbMb789tGsHbduGS+6F/j87kmLSzPoCtwElwDh3vyHl+S2AB4AewDJggLvPi+LY9WnDhtDH2DffwKpV4TF5Pvlx+fJwFrFiWrYsPH5dy2FZt956LXvttQW77hq+9XTtGsbI3nHHwv8jFClWhZw7v/226lyZuu6rryrz5bJlldPKlbU7VosW6+nWrRFdu4auz7p0Cblz1111z7jUjVnoSq9jxzDiDoQCc9GicLXwww8rHz/8ED791Fm0yFi0KDSErU7jxqGorJjatAkFZsuWYUqdb9ECttoqdH2UPDVqlJ33IR0Zf9TMrAQYC/wIWAC8aWaT3H1W0mZnA8vdfRczOxn4AzCgpv0uWQJjxoSkVDGVlW28XNW66rb54otujBkThr1asyY8VkypyxXrkm/MTldJCWy9dfgD6tgxfGNJfezUCaZPn0rv3r0zP6CI5IX6zJ1//vPmc2Vt133xRTduv33THFld7ly7Nprc2aBBuDLTrl3lP/iKqUOH8LjDDvDuu/9R7pR6YxZa9m+7bbgtItnLL09h11178dlnoSeA+fP5fv7LL8NncfHi8MVq4cIwZaKkpLKwTC42GzWqnBo2rJxfsWIPttuu+ucbNQr7LCkJn7fNPdYkiu9tBwBz3f0TADObAPQDkhNiP+DaxPxEYIyZmXv1Q7R/9lm4WTY67er8E2bhF9W8eZiaNav+sXXr8G1jm23CVDHfsqVu6BaRKtVb7rzwwijDTC93Nm1amR9rkzsrcmbFfKtWyp2S20pK/PsvOIdUfbcFEK5wLllSOS1fHs68V0xff73x/Ndfh59ZvTo8VkwbNlSe3a+dtlG8zFqJopjsAMxPWl4AHFjdNu5eZmYrgTbARs1LzGwQMCgs7UD40r4hMZUlzWeybi2wJvGYOr/xsnsZq1eHX+iiRem9OZI9pvsBJL/UY+4cQ+a5MnldGRvnyapyZ+U69w11/KcncVLuzBcNgSbAVonHiqlR0tSwhuWqnmtIGAyxpBaPNUeWM9z9HuAegJ49e/q0aedHtu/S0tKcvhSi+DJjZtRwsiZ2uf7+FVN8hfiPc9PcOSSyfRfT30Z9yPX4lDvTl8uxQfTxmf262ueiuIiwEOiUtNwxsa7KbcysIdCScDO5iEixUu4UkYIQRTH5JtDFzHYys8bAycCklG0mAWck5vsDr9R0z4+ISBFQ7hSRgpDxZe7EfTxDgOcJF9Xvc/f3zWwEMM3dJwH3An8zs7nAV4SkKSJStJQ7RaRQRHLPpLs/Czybsm540vwa4MQojiUiUiiUO0WkEKjjBRERERFJm4pJEREREUmbikkRERERSZuKSRERERFJm4pJEREREUmbikkRERERSZuKSRERERFJm4pJEREREUmbikkRERERSZuKSRERERFJm4pJEREREUmbikkRERERSZuKSRERERFJm4pJEREREUmbikkRERERSZuKSRERERFJm4pJEREREUmbikkRERERSZuKSRERERFJm4pJEREREUmbikkRERERSZuKSRERERFJW0bFpJltbWYvmtmcxGPrKrbZ18ymmtn7ZjbDzAZkckwRkXyn3CkihSTTM5NXAC+7exfg5cRyqtXA6e6+B9AX+JOZtcrwuCIi+Uy5U0QKRqbFZD/g/sT8/cBPUzdw99nuPicx/zmwGGib4XFFRPKZcqeIFIyGGf58e3f/IjH/JdC+po3N7ACgMfBxNc8PAgYlFleZ2UcZxpdsG2BphPuLmuLLzDZmltPxkePvH8UT344R7ScTyp3RUXyZUe5MXy7HBtHHV23uNHev8SfN7CVg2yqeGgbc7+6tkrZd7u6b3PuTeG47oBQ4w91fq0XQkTKzae7eM9vHrS3FlxnFlxnFFz3lzuxQfJlRfOnL5dggu/Ft9sykux9V3XNmtsjMtnP3LxIJb3E127UAngGGxZEMRUSyTblTRIpFpvdMTgLOSMyfAfxf6gZm1hj4B/CAu0/M8HgiIoVAuVNECkamxeQNwI/MbA5wVGIZM+tpZuMS25wE/BA408zeSUz7ZnjcdNwTwzHrQvFlRvFlRvFll3JndBRfZhRf+nI5NshifJu9Z1JEREREpDoaAUdERERE0qZiUkRERETSVnTFpJldYGYfJoYouzHueKpiZpeYmZvZNnHHkszMbkq8dzPM7B+5MBqHmfU1s4/MbK6ZVTWKSGzMrJOZTTazWYm/t9/EHVNVzKzEzN42s6fjjiWVmbUys4mJv7sPzOzguGMqVsqd6VPurBvlzsxlO3cWVTFpZn0II0/skxii7OaYQ9qEmXUCfgx8FncsVXgR2NPd9wZmA1fGGYyZlQBjgaOBbsApZtYtzphSlAGXuHs34CDg/ByLr8JvgA/iDqIatwHPuftuwD7kbpwFTbkzY8qddaPcmbms5s6iKiaB84Ab3H0tgLtX2bdbzP4IXA7kXMsod3/B3csSi68BHeOMBzgAmOvun7j7OmAC4R9eTnD3L9z9rcT8N4QPc4d4o9qYmXUEjgXGbW7bbDOzloTWzPcCuPs6d18Rb1RFS7kzA8qddaPcmZk4cmexFZNdgcPN7HUze9XM9o87oGRm1g9Y6O7vxh1LLfwS+GfMMXQA5ictLyDHEk4FM+sM7Ae8Hm8km/gT4R9wedyBVGEnYAnw18SlpHFm1jTuoIqUcmd0lDvrQLkzLVnPnZmOzZ1zNjOEWUNga8Jp8/2Bx8zsB57F/pE2E99QwmWa2NQUn7v/X2KbYYTLEA9lM7Z8ZWbNgCeA37r713HHU8HMjgMWu/t0M+sddzxVaAh0By5w99fN7DbgCuDqeMMqTMqdmVHujJ5yZ9qynjsLrpjczBBm5wF/TyTAN8ysnDAQ+pK44zOzvQjfJt41MwiXQd4yswPc/cu446tgZmcCxwFHZvMfSTUWAp2Sljsm1uUMM2tESIYPufvf444nxaHACWZ2DLAl0MLMHnT302KOq8ICYIG7V5yRmEhIiFIPlDvrJ74Kyp11o9yZkaznzmK7zP0k0AfAzLoCjYGlsUaU4O7vuXs7d+/s7p0Jfwzds5kMN8fM+hJO65/g7qvjjgd4E+hiZjtZGHruZMIwdTnBwn+2e4EP3P3WuONJ5e5XunvHxN/bycArOZQMSfztzzezXROrjgRmxRhSMVPuzIByZ90od2YmjtxZcGcmN+M+4D4zmwmsA87IgW+I+WQMsAXwYuIMwGvuPjiuYNy9zMyGAM8DJcB97v5+XPFU4VBgIPCemb2TWDfU3Z+NMaZ8cwHwUOIf3ifAWTHHU6yUOzOj3Fk3yp2Zy2ru1HCKIiIiIpK2YrvMLSIiIiIRUjEpIiIiImlTMSkiIiIiaVMxKSIiIiJpUzEpIiIiImlTMSkiIiIiaVMxKSIiIiJp+/9GWA9pZtX+2wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(11,4))\n",
    "\n",
    "plt.subplot(1,2,1)\n",
    "# coordinate \n",
    "plt.plot([-7, 7], [0, 0], 'k-', linewidth=1)\n",
    "plt.plot([-5, 5], [-1, -1], 'k--', linewidth=1)\n",
    "plt.plot([0, 0], [-2.2, 3.2], 'k-', linewidth=1)\n",
    "plt.plot([-7,7], [1,1], 'k--', linewidth=1)\n",
    "# Plot sigmoid\n",
    "plt.plot(z, sigmoid(z), \"b-\", linewidth=2, label=\"Sigmoid\")\n",
    "plt.grid(True)\n",
    "plt.title(\"Sigmoid activation function\", fontsize=14)\n",
    "plt.axis([-7, 7, -0.2, 1.2])\n",
    "\n",
    "plt.subplot(1,2,2)\n",
    "plt.plot([-7, 7], [0, 0], 'k-', linewidth=1)\n",
    "plt.plot([-5, 5], [-1, -1], 'k--', linewidth=1)\n",
    "plt.plot([0, 0], [-2.2, 3.2], 'k-', linewidth=1)\n",
    "plt.plot(z, derivative(sigmoid, z), \"b-\", linewidth=2, label=\"Sigmoid\")\n",
    "plt.grid(True)\n",
    "plt.title(\"Derivative\", fontsize=14)\n",
    "plt.axis([-7, 7, -0.2, 1.2])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to use it with Keras and TensorFlow 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.core.Dense at 0x120ed2b50>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "Dense(10, activation='sigmoid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.26894143, 0.5       , 0.76852477], dtype=float32)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.activations import sigmoid\n",
    "z = tf.constant([-20, -1, 0, 1.2], dtype=tf.float32)\n",
    "output = sigmoid(z)\n",
    "output.numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Tanh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApMAAAEJCAYAAADIA6xFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de7xNdf7H8dfHLaLCqeRWumCkK0ZDTQ5qUqRCVxVTjS4aFU39QlOZSXfdjESJRrqgknSZklNTiahUNKGSS+SS5M7h+/vju/c523HO4ezb2pf38/FYj73W2uus7+drH9/z2d+1vt9lzjlERERERKJRLugARERERCR9KZkUERERkagpmRQRERGRqCmZFBEREZGoKZkUERERkagpmRQRERGRqCmZzGBm5sysW9BxAJhZnpkNTUI5Pc1sQ6LLCZV1p5n9HPp37pmMMvcQy9dBxiAiqSHJ7WDK/J2R4CiZDEjoP2Bpy+igY4xGKY1YF+C2OJdVXCP2InBEPMspoexjgDuAa4DaoXITzswahOrdoshbDwJtkhGDiMTGzEZHtPXbzWylmU0zs95mVjEORcS9HQzF/Hoxb9UGJsezLEk/FYIOIIvVjljvBIwssm9zcsNJLOfcL0kqZzPJ+bc7KvT6qkuBmf+dcxuApPREiEhcvAtcBpQHDgLaAXcBl5lZe+fcxmhOamYVk9gO4pxbkYxyJLWpZzIgzrkV4QX4NXIfUBV41sxWmNlGM/vMzDpF/ryZLTKzgWb2pJn9ZmZLzexvxRRV08zGh87zvZldWlpcZvZ7M/uPma0OnfdDM2tV5JgDzOwJM1tuZlvM7Bszu9DMcoFngKoR37rvDP1MwWVuMxtsZrOLKftjM3tsb+Iws0Wh1fGhchaF9u/WM2pmV5vZQjPbFnr9S5H3nZn12tt/p1CdXglt7jQzF9q/2zf3opefw8eY2Q1mtszM1prZM2a2b8QxZmb9zGyBmW0Nfbb3hN7+IfT6aSjuvBLKKWdmt5vZktA5vjKzcyLeD/dwdjWzd8xsk5nNM7PTS6q3iMTV1lCbv8w594VzbgiQCzQDbgEws0pmdl+oDdhkZp+a2RnhE5hZbuj/8VlmNtPMtgFnRLaDZtYodMyxkYWH2rzVZlbRzMqb2dNm9oOZbQ61PbeYWbnQsXcCPYCOEW17bui9gitEoTb8oSLl7B86Z5e9qZOkJyWTqaka8CZwOnA8MBF42cx+V+S4m4Cv8I3PfcD9RRM/4O/ApNB5XgRGmdmhpZS9H/Bv4I9AS+AL4A0zywGf6ABv4C+p/hk4GugLbAM+Bm4ENuF7WWvjL78WNRZoFlkfMzsCaBV6b49xAL8Pvf4lVE54exdmdh4wFHgEOAZ4FBhmZmcXObQs/04Phsolop5l8cdQLKcBFwLnATdEvD8YuB24B2gKnA8sCb3XMvTaIVRulxLKuAH4G3ArcCw++X3ZzE4octzdwGP4en8KvGBm1cpYHxGJA+fc18BbQNfQrmfwbe0l+DZjDDDZzI4v8qP3AQOB3wEzipxzPv7/dvciP9MdeMk5tx2fCywDLgCaAAOA/vg2Hnyb9xK+NzXc5n1cTBXGAheFk9CQrsAWYEoZ6yTpxDmnJeAF6OY/ilKP+QQYGLG9CHi+yDELihzjgHsitivgE71LyxCbAcvDP4NPcHcCTUo4viewoZj9ecDQiO3PgH9EbA8Evt3bOCLq16208oGPgFFFjhkNfBjLv1Nxn1novK8X2Xcn8HWRY5YA5SP2jQTeDa1Xwze815RQboNQvC32UM4y4O/FfAZji5zn6oj364b2nRL0/wktWjJ5Ka6tiHjv3lD7c2SorT20yPuvAsNC67mh/7NdixxTtB3sA/wIWGj70NC5W5cS473hdqm0mCPbYSAH37HQPuL9d4ERofU91klLei7qmUxBZlbVzO4PXXZcG7pc0QLfAET6ssj2T8DBJR3jnMsHVhVzTGTZB5u/dD7fzNYB60PHh8s+EVjunPumzBXb1Vj8N9Ow7sBzZYhjbzXBJ5SRPsT3qEYq079TjOY553ZEbEd+bkcD+wBToz25me0P1KGM9Q7FAYmrt4jsmeETtGah9XlmtiG8AB3xSVmkWXs45wv4NuGPoe2LgR+ccwW9i2Z2jZnNMrNVoXJuooztrXNuDb5ntXvonHWAthRecSpLnSSNaABOanoQfxnzZnxv4ybgWaBSkeO2F9l27H7rwt4cE2kMUAvfkCwCtuITm6Jlx+p5Ci/Lb8Vfnhkb8X6i4yg6aKas/07F2YlvKCMVNzIzHmVFq8R6O+ecv4tBXzJFAnQ08D3+/6HD38JTtM0oOrim1ME6zrmVZvYOPsn7gN2/vF+IvxXoZvzl69+A3vhbcMpqLDDSzK4DLsJfiflv6L2y1EnSiJLJ1HQK8KxzbiKAmVXGf2ubn6Sy+zjnpoTKrsWu9wR+DtQ2syYl9E5uw49OLJVzbrmZvYdv1LYC051z35chDvCN0Z7K+gY4GXi6yLnn7SnGKKwCit6TWHR7T77B/3u0x3+RKGpb6LXEejvnfjOzn/D1juzhTFS9RSQOzE851gH4J76tNeAQ59y0OJx+LDDUzEbg76OOnFbtFGCGc65gLmAzK9pTuFdtO/Aa/tadTvj2fZxzLvwlNt51khShZDI1zQfOM7NJ+ITpDqByEsu+1Mxm4EeV309hAgM+OZkBTDSzm0LHHwVUdc69iu9FrBwaFfw5sMk5t6mEssYCD4XOf3cZ4yBUVnszex8/MnJtMWU8gB/xPRv4D76h7k7JA1di8R5wi5ldgf/23wWf0C3d2xM459ab2aPAPWa2NXSeHKC5c+4JYCX+G/wZ5kewb3HOrSvmVA8Ag8xsATAbuBR/iatZtJUTkbjax8wOwffWHYT/Atkf///1QefcRjN7DhhtZv3w95nXxN8n+b1z7uUylvcq8CT+i/Wnzg/MCZsP9DSzM4GF+B7FNkBkm7oIONPMGgNrgHXOD97ZhXNui5lNxN8Hfzx++qPwe/PjXCdJEbqclZr64pOG/+JHdX9C4WWCRLsCPwhkNv4+m1H4RgQA59xO4Ez8/Xhj8T1pjxK6/By6B2c4/jL2KkJTXJTgZWBffENadNLvUuMI6Ye/H2cJPnHdTSjB/Sv+cvk8/Cjn65xzcZ9k1zn3Nn6euLtDcTcAhkVxqtvwozNvx//7TgTqhcrIx99MfxX+HsdJJZzjMXxCeT/wNf5yVVfn3Jwo4hGR+DsNP6hwMf5Lemf8QLpTXeEck3/Gj36+H/gf8DpwKn4wTZmEvtS/gk/wxhZ5+0n8aO1x+JHfDfBf9CONxLdHs/Bt+8mlFDc2VM7nzrmiV0PiVidJHVbY+ywiIiIiUjbqmRQRERGRqCmZFBEREZGoKZkUERERkagpmRQRERGRqKXs1EAHHniga9CgQdLK27hxI1WrVk1aecmW6fWbPXs2zZs3DzqMhMn0zy+Z9Zs9e/Zq59xBSSksAGo74yvT66e2M30lu26ltZ0pO5q7RYsWbtasPT0hKn7y8vLIzc1NWnnJlun1MzNS9Xc5HjL980tm/cxstnOuRVIKC4DazvjK9Pqp7Uxfya5baW2nLnOLiIiISNSUTIqIiIhI1JRMioiIiEjUlEyKiIiISNSUTIqIiIhI1JRMioiIiEjUlEyKiIiISNSUTIqIiIhI1JRMioiIiEjUlEyKiIiISNSUTIqIiIhI1JRMioiIiEjUlEyKiIiISNTikkya2SgzW2lmX5fwvpnZY2a20My+NLNm8ShXRCRdqd0UkUwRr57J0UCHUt4/E2gYWnoBT8SpXBGRdDUatZsikgEqxOMkzrkPzKxBKYecAzzrnHPAJ2ZW3cxqO+eWx6N8EYmdc7B1K2zeDNu2FS7bt5e+np8PO3fu/eLc7vsWLKjPjBmF70fGVDTGWLZTidpNCcqWLTB5MqxZA9CLefPg6KODjkrSWVySyb1QF1gSsb00tG+XRtHMeuG/gYe3ufvuu2nUqBHnn39+wXEdO3bk5ptvplevXixYsACAnJwcJkyYwOjRoxkzZkzBscOHDwfgmmuuKdjXo0cPevbsSbdu3Vjj/zdx5JFH8tRTT/Hggw8yZcqUgmPHjx/P/PnzGTBgQMG+vn37cvbZZ9O2bduCfa1atWLw4MH079+f6dOnF+yfNm0akydPZsiQIQX7klWnhg0bMmLEiKyoE8DgwYMzqk7Rf04HAnUZPPhZPvjga956ayaQA9TkuONyMcthzpzvgKpAVfbZpyYVKhzAxo2QvCahqCMDKjel7VW7Cbu2nbVq1SIvLy8Z8QGwYcOGpJaXbJlUvx074KWX6jN+fH3Wrq0U2vskTZvCSSetoVev7zniiI2BxhhvmfT5FZVKdTMXp6/uoW/YrzvnjinmvdeBe51zH4a2pwK3OudmlXS+Fi1auFmzSnw77vLy8sjNzU1aecmW6fUzM+L1u5yKIj+/jRth/nz49lu/zJ8PS5bA0qXw00++dzFaFSvCvvvCPvv49UqV/FLSeqVKUL68X8qVK30xK/m9ZcsWc9hhhxYcE8ksvtuDBtls51yL6P+V4ife7Sao7Yy3TKnfpk1wySUwaZLfPv54+MMf4Mknx1G58iVs2QLVqsHEifCnPwUbazxlyudXnGTXzazktjNZ3RDLgPoR2/VC+0SkFNu3w6efwoQJ9Xj6aZg9G/73v9Iv31avDvXqQd26cNBBULPmrkuNGrDfflC1auFSrZp/rVgxeXWLlJf3Pbm5hyalrEGDklJMPKjdlLjYuNEniB9/7P////vfcNZZ/ovWk092Z8mSS+jTB55/Hjp2hHHjIOKCjMgeJSuZfA243sxeAE4C1um+H5HizZsHb70FU6fCBx/Ahg0ARxW8X6ECNGoEjRv710aN4PDDffJYt65PCiUjqN2UmDkHV1/tE8n69eHtt6FJk12POfBAGDvWfwl94AHo0cO3L8cdF0zMkn7ikkya2fNALnCgmS0F7gAqAjjnhgNvAGcBC4FNwJ/jUa5Ippg/H1580S9z5+76XuPGcOSRyzn77No0bw7HHguVKwcTp8SP2k1JhieegOee87evvPnm7olkWLlycN99sHIljBkDXbvCrFlwwAHJjVfSU7xGc1+8h/cd0DseZYlkiu3b/f1JjzwCM2YU7q9RAzp1gtNOg/btfW9jXt635ObWDi5YiTu1m5Jo33wDN97o10eOhKZNSz/eDIYNg88/hy+/hD59fGIpsidBDd0UyVq//uob9scf9wNnAPbfH849Fy680CeRlSqVfg4RkdI4B717+y+tV1zhB9/sjX33hfHj/SXuZ5/1P9umTWJjlfSnxymKJMnmzf5+pMMPh1tu8Ylk48YwfDgsX+57AM46S4mkiMTu+edh2jTIyYH77y/bzzZqBLfd5tevu84npCKlUTIpkgRTpvhJgW+5xfdMtmnj982b52+O33ffoCMUkUyxYQP06+fX77/fJ5RldeutcOSRvo167LH4xieZR8mkSAKtXu2n2OjUCRYt8oNn3nrL9xicddbucyqKiMTqscdgxQpo2RJ69ozuHJUrw6OP+vV77oHffotbeJKB9KdMJEHeftsnjxMm+HkchwyBzz6DM87YfSJtEZF4+PVXfzsNwODBsX1hPessOPlk/9jFcGIpUhwlkyJxtnOnnxj7zDN978Af/whffQU33eTniBQRSZSHH/YJZW4utGsX27nM4J//9OsPPQRr18YcnmQoJZMicbR5M3TrBnfc4bfvustf0m7QINCwRCQLrF3rk0mAf/wjPldAcnP9FGXr1hWeW6QoJZMicbJmjZ/W55VX/CMNp0yBv//dP7daRCTRhg+H9et98nfKKfE7b/jL8dCh4SdyiexKyaRIHKxc6b/Bhx9Z9uGH/jK3iEgybNlSeF/jLbfE99ynnAKtWvmez1Gj4ntuyQxKJkVitHKl7wn4+mv/qLLp0/f8pAkRkXgaOxZ+/hmOPx5OPz2+5zaDv/3Nrw8ZAvn58T2/pD8lkyIx+O036NChMJGcNs0//lBEJFl27oQHH/Trt9ySmNkiOnf2k5n/+KN/Qo5IJCWTIlHatg26dvXPsT3qKJ9I1qoVdFQikm2mToVvv4V69fy8tolQvjz07evXH388MWVI+lIyKRIF5+Daa+Hdd+Hgg/2ckkokRSQITzzhX6++GipWTFw5l14KBxzgb+X5/PPElSPpR8mkSBSeeMLfiF6lCrzxBhxxRNARiUg2WrYMXnvNz2F75ZWJLatq1cIn6vzrX4ktS9KLkkmRMvroI7jhBr/+9NPQvHmw8YhI9nrqKdixA849F2rXTnx5113nX8eN0yTmUkjJpEgZrF0LF1/sRzP27evXRUSCkJ8PI0f69WuvTU6ZjRr50eKbN8Po0ckpU1KfkkmRveScvydpyRI46SS4996gIxKRbDZ5sr/M3bgxtG2bvHLDvZPDhvmR5CJKJkX20rPP+ikxqlWD555L7I3uIiJ7Eh54c801iZkOqCSdOvmHMyxcCO+8k7xyJXUpmRTZC8uXw403+vWhQ+HII4ONR0Sy24IFPpGrXBl69Ehu2RUq+AQWNBBHPCWTInvhr3+FX3/1j0i8/PKgoxGRbDdihH+96CKoUSP55V91FVSqBK+/DosWJb98SS1KJkX24NVXYeJEf3l7+PDkXk4SESlqyxZ45hm/nqyBN0UdfLCfIN25wsRWspeSSZFSbN5ceHn7nnvg0EODjUdEZPx4WLMGmjWD3/8+uDjCl7pHjfJPBJPspWRSpBQPPOCfRXv88cH1AIiIRAoPvLn22mCvlJx8MjRtCj//DJMmBReHBE/JpEgJFi8unP7nscf8s2lFRII0Z45/nOEBBwQ/z62Zny4N4Mkng41FgqVkUqQEAwf6y9wXXACnnhp0NCIi/r5t8AMBq1YNNhaAyy7zj5WdOtWPMJfspGRSpBhffQVjx/q5JDU5uYikgvXrfbsEhT2CQate3Y8oBw3EyWZKJkWK0b+/H6V47bVw+OFBRyMi4hPJDRv8lZKmTYOOplB4IM4zz/iR5pJ9lEyKFPHRR37utKpVYcCAoKMREfFfbiMH3qSS3/8eTjjBjzCfODHoaCQISiZFirjrLv/at6+fS01EJGgff+xvvzn4YOjSJehodmVW2DupgTjZScmkSIQZM/wjyvbbr3B+SRGRoIUH3lx5pX/yTKq55BL/YIf//hfmzg06Gkk2JZMiEf75T//auzfUrBlsLCIiAKtXw0sv+R7AXr2CjqZ4++0H3bv7dQ3EyT5KJkVCvvjC3ytZpYq/xC0ikgqeecY/Yeass6BBg6CjKVl4hPmYMbBpU7CxSHIpmRQJeeAB/3r11XDQQcHGIiICsHNn4X2I4fsSU9WJJ0LLlrBune9JleyhZFIEWLIEXnzRP+XmppuCjkZExHvnHfjuOzjsMDjzzKCj2bNw72T4Hk/JDkomRfCPS9yxA84/Hw49NOhoRES88HRAvXqlxyNdL7wQ9t/fD2b84ougo5FkUTIpWe+33wpvGO/XL9hYRETCfvgBXnvNP4nryiuDjmbvVK0KPXr49cceCzYWSR4lk5L1Ro/2CeWpp0KLFkFHIyLiDR3qJyu/+GKoVSvoaPbeX//qR56PGwcrVwYdjSRDXJJJM+tgZt+a2UIz+79i3u9pZqvM7IvQclU8yhWJlXMwbJhfv+GGYGOR7KO2U0qyfj089ZRfT7e2qWFD6NgRtm7VJObZIuZk0szKA/8CzgSOBi42s6OLOfRF59wJoeWpWMsViYepU+Hbb6FuXejcOehoJJuo7ZTShK+Y/PGP0KxZ0NGUXTgBHjbMT2skmS0ePZMtgYXOue+dc9uAF4Bz4nBekYQL90pefTVUqBBsLJJ11HZKsXbuLLzfMN16JcPat4emTWHFCk0TlA3i8eezLrAkYnspcFIxx3U1s1OB+cBNzrklRQ8ws15AL4BatWqRl5cXh/D2zoYNG5JaXrJlev2AMtdv5cp9mDTpD5Qv72ja9BPy8lL363Omf36ZXr8SqO1MA0HUb/r0HBYuPJZatbZQvfoM8vJcQstLVP06dKjN3LmN+ec/f6Nu3c8wS0gxpcrk38+UqptzLqYF6AY8FbF9GTC0yDE5wD6h9auB9/Z03ubNm7tkmjZtWlLLS7ZMr5//VS6bO+90Dpy74IIEBBRnmf75JbN+wCwXY7sXj0VtZ3oIon7t2/u26YEHEl9WNG3n3tq0ybmcHF+XDz9MWDGlyuTfz2TXrbS2Mx6XuZcB9SO264X2RSasa5xzW0ObTwHN41CuSNR27IBRo/x6qj7rVjKe2k7Zzddf+3u5q1ZNn+mASlKlSuEk5o8+GmwskljxSCY/BRqa2eFmVgm4CHgt8gAzqx2x2Rn4Jg7likRt6lRYvBgOPxzatg06GslSajtlN4884l979IAaNYKNJR6uu87fjz5xIvz4Y9DRSKLEnEw65/KB64G38Q3dS865uWY2yMzC42P7mNlcM5sD9AF6xlquSCzCU25ccQWU02yrEgC1nVLUsmXw73/7ORr79Ak6mvioWxcuuMAPKnrooaCjkUSJy/hV59wbwBtF9v09Yv024LZ4lCUSq9Wr4dVXfRLZs2fQ0Ug2U9spkYYM8dPodOsGjRsHHU383Hqrn8B85EgYOBAOPjjoiCTe1CcjWef552H7djjjDKhXL+hoRERgzZrCCb5vy7CvD8cdB506wZYtuncyUymZlKzz3HP+9bLLgo1DRCRs6FDYuNF/yU3HScr3pH9///qvf8G6dcHGIvGnZFKyysKFMGMGVKsG52h6aBFJAWvXwsMP+/VM65UMa9UKcnN9IhkeZCSZQ8mkZJVx4/zreefBvvsGG4uICPhEct06P7NEmzZBR5M4d97pX4cM8Qm0ZA4lk5I1nIOxY/169+7BxiIiAv5eyXBP3aBBwcaSaG3a+Mcs/vabRnZnGiWTkjVmzYIFC6BWLd+giYgE7YEHYP16+NOf4JRTgo4m8cIJ86OPwqpVwcYi8aNkUrJGeODNRRf5SXRFRIK0ZEnh6OZ//CPYWJKldWs480zYsCHze2KziZJJyQr5+fDCC35dl7hFJBUMHOiny7ngAmjZMuhokue++/w8v8OHw/z5QUcj8aBkUrLC1Knw88/QqBG0aBF0NCKS7b74wj/tpmJFuOeeoKNJrmOP9Q+MyM/P3NHr2UbJpGSF8CXu7t39o8pERILiHNx4o3/t3RuOOCLoiJJv0CCoUgVefhny8oKORmKlZFIy3qZN8Morfv2SS4KNRUTk+efh/ffhwAPh9tuDjiYYdesW9kr27u2fSibpS8mkZLw33/Q3e7dsCUcdFXQ0IpLN1q2Dfv38+n33Qc2awcYTpL/9DY48EubN02MW052SScl44V7Jbt2CjUNEZOBAWLHCPxGmZ8+gowlW5cr+MZLgJzT/8cdAw5EYKJmUjLZtG7z+ul8/99xgYxGR7PbBBz55qlABhg3zI5qzXYcO/ov+xo3wl7/4+0gl/ehXWTJaXp6/rNS0KTRsGHQ0IpKtNm2CK67w67fdBiecEGw8qWToUMjJgXfegaefDjoaiYaSSclo4Uvc550XbBwikt1uvRW++85PizNwYNDRpJZateDxx/16377www/BxiNlp2RSMtbOnTBpkl9XMikiQZk0yfe+VawIo0dDpUpBR5R6LroIunb1j5a8+GKN7k43SiYlY82cCcuXw2GHwYknBh2NiGSjJUsKL2/fdx80axZsPKnKDEaOhEMPhRkz1HubbpRMSsYKX+I+91xNVC4iybdli+9t++UXOOssP1G5lKxGDRg3DsqXh/vv9xOaS3pQMikZybldk0kRkWRyDnr1gk8/hQYNYMwYfandGyef7HtwAS6/HL76Kth4ZO8omZSMNG8eLFjgRwieckrQ0YhItrnvPv/s7X339fdMHnhg0BGlj7594dJL/XRBnTv725UktSmZlIz06qv+tXNnP6ebiEiyPPOMn/7HDJ59Fo47LuiI0osZjBjhn1q2aBGceaaf4k1Sl5JJyUiaEkhEgvDyy37ybYDHHvP3TErZVaniHzjRqBHMmQNnn+0fiyupScmkZJzFi2H2bKhaFU47LehoRCRbjB8PF1wAO3b40cjXXx90ROntoIPgP/+BunXhv//1g5jWrw86KimOkknJOOFL3B06+G+3IiKJNnq0nx9xxw5/iXvQoKAjygyHHeafZFavnk8oTz8dVq0KOiopSsmkZBxd4haRZHEO7rwT/vxnn0j+/e9w990auR1PRx0F77/vE8sZM6B1az/AUlKHkknJKGvWwAcf+EE3HTsGHY2IZLLffoPzz4e77oJy5WDYML+uRDL+jjgCpk/3D6BYuNAPzpkyJeioJEzJpGSUyZP9YxTbtYPq1YOORkQy1Wefwe9/DxMnwv77++l/rr026KgyW+3avrPg7LPh11+hUyfo3x+2bQs6MlEyKRlFE5WLSCJt2wb/+AecdBLMnw/HHguzZvnERhKvWjV/X/zgwb43+J57/Gfx5ZdBR5bdlExKxti40Y/8AzjnnGBjEZHM8957cMIJ/r7I/Hy44Qb45BNo2DDoyLJLuXJ+kNP77/vL31984Z953revv/VAkk/JpGSMt97yz8L9wx+gTp2goxGRTPHll77nsX17+OYbnzy+9x488oh/wo0E45RT/ByU11/vB0I9/DAceSQ8+ihs3Rp0dNlFyaRkjPCUQBrFLSKxcg7efhtuvfVYjj/eD/aoWtVf4v7qK2jbNugIBfxl78cf989Ab90aVq+GG2/0I78HDYK1aysGHWJWUDIpGaICr7/u15RMiki0Vq+GJ56AY47xc9XOnJlDlSrQpw98/72fjHyffYKOUopq1gw+/BBeew2OPx5+/hnuuAMuvLAVV1wBH3/sB2dKYiiZlAyRy6+/wtFH6/4lESmbX36BUaN88njIIXDddTBvnr9d5qqrvmfJEn/p9OCDg45USmPmR3p//rm/DaFzZ8jPN555Bk4+GRo0gH79YOZMJZbxViHoAETiw3dHqldSRPZk/Xr46COfcLz3np/mxzn/XvnycMYZcPnlfg7Jjz5aTE7OEcEGLGVi5m9DaNsWnntuJp9/fhIvvghLlsCQIX458ED/frt2/rVRI80PGgslk5L2/DdMP9opAkwAABPWSURBVBeQkkkRCdu50ycQ337r73P87DOYPdtP6RNOHgEqVoQ2bfxztc87zycakhnq1t1M9+5w//1+5P1LL8HLL/vfi/Hj/QJwwAH+UnmzZn5i9CZN/FWu/fYLNv50EZdk0sw6AI8C5YGnnHP3Fnl/H+BZoDmwBrjQObcoHmWLzJwJUIf69X1DIJIu1HZGb/t2P3H16tWwbBksXepfw+uLFvlH7m3ZsvvPVqzop/hp186P0D75ZI3KznTlyvkBOq1b+1HfCxcW9ky//76/x3LaNL9EqlPHJ5X16kHduoWvdev692rW9L872d6rGXMyaWblgX8BpwNLgU/N7DXn3LyIw64E1jrnjjKzi4D7gAtjLVsECkdxn3uu/kNL+kh02xnZ85ao9Z07YcOG8vzyi593MXLZsaP07fx8n+ht2gSbN5e8bNzok8a1a/0SXt+wYW/+Ffw9kI0a+Z6m5s39F85jjtEgmmxm5hPEhg3h6qv97/Ly5YU913Pm+N7rBQvgp5/8UppKlXxSGV6qV/cj/0taqlTxP1Ox4q6vJa2XL++T4XLlfOzh9fXrK7BuXeF2cceYJefvYjx6JlsCC51z3wOY2QvAOUBkg3gOcGdofQIw1MzMucimSaTsnCt86o0ucUuaSUjbOXt2sr9U/TGZhRUoV87/0c7JKewpiuw5OvRQnywccEAg4UkaMfO9jHXq7Pokox07YPFi34sZ7vWO7AVfsQLWrPFzWq5Y4ZfkOmWvjiopuSzra2nikUzWBZZEbC8FTirpGOdcvpmtA3KA1ZEHmVkvoFfEdhzCk8zWBP+3dzXt2h0C7Ag4HpG9lqC2s3kJxUUOX3VxWt8RWvIjlrJsbwE272HZBKwNLb8WrO/cuZ5ffnH88ovvQZLo6W9trCoDNYCaoaU6ULXIUi1ivTJQEagUWioWeS26Xq6YxUrYv/v7zvnEeEcC/zym1AAc59wIYARAixYt3KxZs5JWdl5eHrm5uUkrL9kytX533+3nfYPJOJcfdDgJk6mfX1gy65eJfzh3bzuLOyoxM8HpdzO9mRmZfJEwkz+/vambc4W3pISX8P6yvlavXnI58UgmlwH1I7brhfYVd8xSM6sAHIC/mVwkJuFL3PAK8OcAIxEpM7WdIpJQ4cva5RI8q3g8Tv8p0NDMDjezSsBFwGtFjnkN6BFa7wa8p/slJVZLlvj7w/wozHeCDkekrNR2ikhGiDmZdP7a4vXA28A3wEvOublmNsjMOocOexrIMbOFQF/g/2ItVyQ8irtDB/D3XomkD7WdIpIp4nLPpHPuDeCNIvv+HrG+BTg/HmWJhEWO4n755WBjEYmG2k4RyQR6NrekpTVr4IMPoEIF6Ngx6GhERESyl5JJSUuTJ/tpDtq2hRo1go5GREQkeymZlLQU+dQbERERCY6SSUk7GzfC22/79XPOCTYWERGRbKdkUtLO22/7Z/qedJJ/bJqIiIgER8mkpB09i1tERCR1KJmUtLJ9O7z+ul/X/ZIiIiLBUzIpaeX99+HXX6FJE2jcOOhoRERERMmkpBVd4hYREUktSiYlbezcWTglkJJJERGR1KBkUtLGjBnw009Qvz40bx50NCIiIgJKJiWNTJjgX7t1A7NgYxERERFPyaSkBed2TSZFREQkNSiZlLQwaxYsXgx16sAf/hB0NCIiIhKmZFLSQrhXsmtXKKffWhERkZShP8uS8pyDiRP9eteuwcYiIiIiu1IyKSlvzhz47js4+GA45ZSgoxEREZFISiYl5YUvcXfpAuXLBxuLiIiI7ErJpKQ052D8eL+uUdwiIiKpR8mkpLS5c2H+fMjJgTZtgo5GREREilIyKSktPPDmvPOgQoVgYxEREZHdKZmUlKVL3CIiIqlPyaSkrC+/9Je5c3KgbdugoxEREZHiKJmUlPXcc/71ggugUqVgYxEREZHiKZmUlLRzJ4wb59e7dw82FhERESmZkklJSe+/D8uWQYMG0Lp10NGIiIhISZRMSkoKX+Lu3h3Mgo1FRERESqZkUlLOli2FT73RJW4REZHUpmRSUs6UKbBuHTRrBk2aBB2NiIiIlEbJpKScyEvcIiIiktqUTEpKWbvW90yawUUXBR2NiIiI7ImSSUkpEybAtm3Qrh3UqRN0NCIiIrInSiYlpYwd618vvTTYOERERGTvKJmUlLFgAXzwAey7L3TpEnQ0IiIisjeUTErKGDXKv15wAey/f7CxiIiIyN5RMikpIT8fRo/261ddFWgoIiIiUgYxJZNmVtPM3jGzBaHXGiUct8PMvggtr8VSpmSmN96AFSugcWM9PlEyn9pOEckksfZM/h8w1TnXEJga2i7OZufcCaGlc4xlSgYaOdK/XnWVHp8oWUFtp4hkjFiTyXOAMaH1McC5MZ5PstCiRX5uyUqV4PLLg45GJCnUdopIxqgQ48/Xcs4tD62vAGqVcFxlM5sF5AP3OudeLe4gM+sF9AKoVasWeXl5MYa39zZs2JDU8pItles3cuThOHcYp576M/PmfcO8edGdJ1XrFw+p/PnFQ6bXrxhqO9NEptcP1Hamq1SqmznnSj/A7F3gkGLeGgCMcc5Vjzh2rXNut3t/zKyuc26ZmR0BvAe0d859V1q5LVq0cLNmzdqbOsRFXl4eubm5SSsv2VK1flu3Qr16sHo1fPRR9PdLmhl7+l1OZ6n6+cVLMutnZrOdcy2SUI7azgyQ6fVT25m+kl230trOPfZMOudOK+XEP5tZbefccjOrDaws4RzLQq/fm1kecCJQaoMo2WH8eJ9InnACtGoVdDQi8aO2U0SyRaz3TL4G9Ait9wAmFT3AzGqY2T6h9QOBk4EoL2RKJnEOHn/cr/furYE3klXUdopIxog1mbwXON3MFgCnhbYxsxZm9lTomCbALDObA0zD3/ejBlH46COYORNycuCSS4KORiSp1HaKSMaIaQCOc24N0L6Y/bOAq0LrHwPHxlKOZKaHHvKv117rH6Eoki3UdopIJtETcCQQCxbApEl+OqDevYOORkRERKKlZFIC8cgj/p7J7t3hkOLGu4qIiEhaUDIpSbd8OTz9tF/v1y/YWERERCQ2SiYl6R54wM8v2aULNG0adDQiIiISCyWTklSrVsHw4X594MBgYxEREZHYKZmUpHroIdi8GTp1ghNPDDoaERERiZWSSUma5cvhscf8+u23BxuLiIiIxIeSSUmaQYN8r+R550HLlkFHIyIiIvGgZFKSYsECGDkSypWDu+8OOhoRERGJFyWTkhT9+8OOHdCzJzRpEnQ0IiIiEi9KJiXhpk2DCRP8IxPvuivoaERERCSelExKQuXnQ58+fr1/f6hXL9h4REREJL6UTEpCDRsGX38Nhx+up92IiIhkIiWTkjCLF8OAAX79kUegcuVg4xEREZH4UzIpCeEcXHMNbNgA3bpB585BRyQiIiKJoGRSEuLf/4Y334QaNeDxx4OORkRERBJFyaTE3fffw/XX+/UhQ+CQQ4KNR0RERBJHyaTEVX4+XHoprF8PXbtCjx5BRyQiIiKJpGRS4ur222H6dD8F0IgRYBZ0RCIiIpJISiYlbsaPh3vvhfLlYexYqFkz6IhEREQk0ZRMSlx89RX8+c9+/cEHoU2bYOMRERGR5FAyKTFbsgTOPBM2bvT3S95wQ9ARiYiISLIomZSY/PILdOgAy5bBqafCyJG6T1JERCSbKJmUqP3yC5x+OsybB02bwquv6ik3IiIi2UbJpERlzRqfSH72GRx1FLz1lp+gXERERLKLkkkpsx9+gNatCxPJadP8VEAiIiKSfZRMSpl88gm0agXz58Nxx8H77yuRFBERyWZKJmWvOAdPPOEH2fz8M7RrBx98AHXqBB2ZiIiIBEnJpOzRqlX+0YjXXQfbt/upf956Cw44IOjIREREJGgVgg5AUpdz8MILcNNNvjdyv/3gySfh4ouDjkxERERShZJJKdacOdCvH0yd6rfbtIExY+Cww4KNS0RERFKLLnPLLr780j/F5sQTfSJZsyY89RS8954SSREREdmdeiaFnTvhzTfh4YcLeyIrVYLevWHAAMjJCTY+ERERSV1KJrPY3Lnw4ovw/POwcKHfV7UqXHmlv0+yQYNAwxMREZE0oGQyi2zbBjNnwrvvwsSJ8PXXhe/Vrw99+sBVV0H16sHFKCIiIulFyWQGW7PGP6Vm9mx45ZVjmTsXNm4sfL9GDejSBS68ENq2hQr6bRAREZEyUvqQ5vLzYcUK/4jDb7/1T6b59ls/kGbRosgj/Y2PTZpA+/bQoYN/tnalSkFELSIiIpkipmTSzM4H7gSaAC2dc7NKOK4D8ChQHnjKOXdvLOVmKuf8pei1a0teVq2CZctg6VL/umKFH0BTnCpV4IQToHlz2H//b+jdu4meWCOSAtR2ikgmibVn8mugC/BkSQeYWXngX8DpwFLgUzN7zTk3r7QTr14No0b5BCssvF7cvljfnz+/Ll99VfrP7NzpewJ37PCv4aXodkn7tmyBTZtg8+Zdl8h9JSWGJTGDQw6BQw+FRo2gcWO/NGkCv/td4aXrvLyfqVOnSdlOLiKJkrC2U0Qk2WJKJp1z3wCYWWmHtQQWOue+Dx37AnAOUGqD+OOPflRx8jRMZmElqlDB38tY0pKTA/XqQd26/rV2bahYMeioRaQsEtl2iogkWzLumawLLInYXgqcVNyBZtYL6AVQocJR5OePCr3jaN36ZAA+/vjDgn0NGzaiUaOGvPvuu2zdugWA/fffn1NPPZUvv5zD4sU/Fhx7+ul/Yt26dcyc+UnBvuOPP4EGDRowadIrBTEcckgtWrVqxfTp01mxYnnBsV26dGXRou/47LOZQD6QT7t2bTj44Jq88MLYgn1HH/07/vSndowb9ywrVy4DdlCtWhX69u3Df//7DtOmTQE2A5u5445bqFgxn4ED+xbs6969Oz179qRbt27Mn78GgIYNGzJixAgefPBBpkyZUhDr+PHjmT9/PgMGDCjY17dvX84++2zatm1bsK9Vq1b079+f1q1bM3369IL906ZNY/LkyQwZMqRg3913302jRo04//zzC/Z17NiRm2++mV69erFgwQIAcnJymDBhAqNHj2bMmDEFxw4fPhyAa665pmBfjx49Cuq0Zk386jR48GD69+9fUCczy7g6hT+nCRMm7HJsJtQpqM8pjUTVdtaqVYu8vLyEBxe2YcOGpJaXbJlePyCj65fJn18q1c1c5DXd4g4wexc4pJi3BjjnJoWOyQNuLu6+HzPrBnRwzl0V2r4MOMk5d31p5bZo0cLNmlXsbUQJkZeXR25ubtLKS7ZMr5+Zsaff5XSW6Z9fMutnZrOdcy2SUI7azgyQ6fVT25m+kl230trOPfZMOudOi7H8ZUD9iO16oX0iIhlLbaeIZItkPJv7U6ChmR1uZpWAi4DXklCuiEg6U9spImkhpmTSzM4zs6VAK2CKmb0d2l/HzN4AcM7lA9cDbwPfAC855+bGFraISPpS2ykimSTW0dyvAK8Us/8n4KyI7TeAN2IpS0QkU6jtFJFMkozL3CIiIiKSoZRMioiIiEjUlEyKiIiISNSUTIqIiIhI1JRMioiIiEjUlEyKiIiISNSUTIqIiIhI1JRMioiIiEjUlEyKiIiISNSUTIqIiIhI1JRMioiIiEjUlEyKiIiISNTMORd0DMUys1XAj0ks8kBgdRLLSzbVL72pfvFzmHPuoCSVlXRqO+NO9UtvmVy/ZNetxLYzZZPJZDOzWc65FkHHkSiqX3pT/SRVZfpnp/qlt0yuXyrVTZe5RURERCRqSiZFREREJGpKJguNCDqABFP90pvqJ6kq0z871S+9ZXL9UqZuumdSRERERKKmnkkRERERiZqSSRERERGJmpLJYphZPzNzZnZg0LHEk5k9YGb/M7MvzewVM6sedEyxMrMOZvatmS00s/8LOp54MrP6ZjbNzOaZ2VwzuyHomBLBzMqb2edm9nrQsUhs1HamD7Wd6S+V2k4lk0WYWX3gT8DioGNJgHeAY5xzxwHzgdsCjicmZlYe+BdwJnA0cLGZHR1sVHGVD/Rzzh0N/AHonWH1C7sB+CboICQ2ajvTh9rOjJEybaeSyd09DNwCZNzIJOfcf5xz+aHNT4B6QcYTBy2Bhc65751z24AXgHMCjilunHPLnXOfhdbX4xuNusFGFV9mVg/oCDwVdCwSM7Wd6UNtZ5pLtbZTyWQEMzsHWOacmxN0LElwBfBm0EHEqC6wJGJ7KRnWYISZWQPgRGBGsJHE3SP4BGRn0IFI9NR2ph21nekvpdrOCkEHkGxm9i5wSDFvDQD64y/TpK3S6uecmxQ6ZgD+MsBzyYxNomNm1YCJwI3Oud+CjidezKwTsNI5N9vMcoOOR0qntlNtZ7pR25k8WZdMOudOK26/mR0LHA7MMTPwlzE+M7OWzrkVSQwxJiXVL8zMegKdgPYu/ScZXQbUj9iuF9qXMcysIr4xfM4593LQ8cTZyUBnMzsLqAzsb2ZjnXOXBhyXFENtp9rOdKK2M7k0aXkJzGwR0MI5tzroWOLFzDoAQ4A2zrlVQccTKzOrgL8Zvj2+IfwUuMQ5NzfQwOLE/F/mMcAvzrkbg44nkULfrm92znUKOhaJjdrO1Ke2M3OkStupeyazy1BgP+AdM/vCzIYHHVAsQjfEXw+8jb/B+qVMaQxDTgYuA9qFPq8vQt9ERSS51HamF7WdSaaeSRERERGJmnomRURERCRqSiZFREREJGpKJkVEREQkakomRURERCRqSiZFREREJGpKJkVEREQkakomRURERCRq/w+jInaOol0fOgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(11,4))\n",
    "\n",
    "plt.subplot(1,2,1)\n",
    "# coordinate \n",
    "plt.plot([-7, 7], [0, 0], 'k-', linewidth=1)\n",
    "plt.plot([-5, 5], [-1, -1], 'k--', linewidth=1)\n",
    "plt.plot([0, 0], [-2.2, 3.2], 'k-', linewidth=1)\n",
    "plt.plot([-7,7], [1,1], 'k--', linewidth=1)\n",
    "# Plot\n",
    "plt.plot(z, np.tanh(z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.grid(True)\n",
    "plt.title(\"Tanh activation function\", fontsize=14)\n",
    "plt.axis([-5, 5, -1.2, 1.2])\n",
    "\n",
    "plt.subplot(1,2,2)\n",
    "# coordinate \n",
    "plt.plot([-7, 7], [0, 0], 'k-', linewidth=1)\n",
    "plt.plot([0, 0], [-2.2, 3.2], 'k-', linewidth=1)\n",
    "# Plot\n",
    "plt.plot(z, derivative(np.tanh, z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.grid(True)\n",
    "plt.title(\"Derivative\", fontsize=14)\n",
    "plt.axis([-5, 5, -1.2, 1.2])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to use it with Keras and TensorFlow 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.core.Dense at 0x84a1721d0>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "Dense(10, activation='tanh')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1.       , -0.7615942,  0.       ,  0.8336547], dtype=float32)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.activations import tanh\n",
    "\n",
    "z = tf.constant([-20, -1, 0, 1.2], dtype=tf.float32)\n",
    "output = tanh(z)\n",
    "output.numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. ReLU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def relu(z):\n",
    "    return np.maximum(0, z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoEAAAEJCAYAAADrWP2QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZxcdZnv8c/DoqiACMGwJILKoCIqGi7XUUaiCLIJI6CCuLBoRnEBBVHEfSGEHUmAhEW9F9TxjsuMXJdBTQuIIlsIECAECBhcEGe80CxZf/ePc5oURS/VyalzTp36vF+veqW663Sd55fqfvpb56lTHSklJEmS1F/WqboASZIklc8QKEmS1IcMgZIkSX3IEChJktSHDIGSJEl9yBAoSZLUhwyBGlNEfDEibi1hP9tGRIqInUvY1wERcVdErIiIb3Z7f2PUMjVf94Qq65BUrYg4PCIGS9pXioiDy9iX6ssQOE4R8c38hyflAeL+iDg/Ip43zvtZHBHHj3DbsD+c+b4vX9PaO6hppBB2OrBbwfsaiIiZbZ/+A7AlMK/IfY3gYuD7wDbAMSXsDxjxcb+GbN1/K6sOSePX1v+XR8SDETE3Ij4cEesXsIt/BV5UwP08aZTfG1sCPy5yX+o9hsA18wuyH6BtgfcDbwXOq7KgbkopDaaUuh5QUkorU0p/Timt6OZ+ImITYDPg5ymlB1JK/6+b+xtLSmlZvm7fuV2qv9b+vydZkPoScFVEPGdN7zQi1k8pPZ5SerCQKseQ95ylZexL9WUIXDNL8x+gJSml/yR79rZn6wYRcURELIiIJyJiYUR8PCK6+v8dEZtFxHciYklEPB4Rt0XEEW3bREQcl49Cl+bbTs9vvjf/97r8me5A/jVPjoMjYs+IWBYRm7Xd78kRMb+TOvLx627Ah1ueVW873JHIiHhDRFyb/z/+JSLOiohntNw+EBHn5ft/KH9mfvpI/9cRMRX47/zDX+X7mzrcGKZ9TDu0TUTsHhG3RsSj+VGAF7Z93T55zY9HxN8i4scRsUH+/7kNcNrQuofbT/65AyPilvwx+kNEnBQR0XL74oj4bETMjoiH8//rTw63ZkmFGur/D6SU5qWUzgSmAq8BTgCIiGdExIz85/KxiLguIt4ydActP/P7RMTvI2IZ8JbWPhQR2+fbvKJ15xExLe9160fEuhFxcUTcm/ebuyLihKH+FxFfBN4H7NvSa6fmtz05cYqIayLijLb9bJzf54GdrEm9yRC4liLiRcBewPKWz30AOBn4PPAy4DjgU8DRXS5nA+BGYD/g5cA5wOyI2L1lm5OBzwHT823eTjaGBdgl/3cvsme6Bw6zj18CD+VfB2TBEngXcGmHdRwD/Bb4Rr6fLVtqeFJEbA38FLgJeDVwFHBoXnurw4AVwOuAjwDHAu8cpnbIRq8vz68flO/7mhG2Hc4zgROBI4F/BDYBLmipeS/gP4ArgCnAG4Ffk/2sHQgsAb7M6nU/TURMAf4P8APgFcCn831+pG3TjwO3kP3ymQGcGhH/OI61SCpASulW4GdkPQWy3rYbWV/cEfgW8OOIeFXbl84APgu8FLi27T4XAteR9bdWhwHfSyktJ+srDwDvIPtdcxLwGWDoSffpwPdYffRypH53KXBI25Png4AngP87zjWpl6SUvIzjAnyTLHAMAo8DKb98vGWb+4H3tH3dscCClo8XA8ePsI8EHDzCvi8fZ73fBS7Kr29I9kP9wRG23Tbf985tn/8icGvLx2cCV7V8vCuwEpjUSR35xwPAzNH2D3wNuAtYp2Wbw4GlwLNb7ue3bfdzReu+hqllQr6fqW33O9i23dR8uwkt2yTgJS3bHJbXE/nHvwG+O8q+n/a4D7Ofy4BfDfMYLGm7n++0bXMX8Nmqf0a8eGnqZbQeDJwCPAa8GFgFvKDt9h8B5+XXh37mD2rb5il9CPgYcF9Lf3lBft+vG6XGU4BfjFUzLb9nyF4eswzYveX2XwBz8utjrslLb148ErhmrgR2Ijtydi7wE+DrABGxOTCZ7MjX4NCF7Afzxd0sKh8NnBQR8/Mx5CDZ0acX5JvsQHYk65druatLgddHxDb5x4cBv04pLemwjk69DPhdSmlVy+euBp4BbNfyufltX/dH4Pnj3FenlqaU7mzb1zOAoRODXs3a//++jCxMtroa2DoiNm75XJnrljS6IAtWr8mvL2j7HbAvT/8dcP0Y9/ldYCvgn/KPDwXuTSk9eTQvIj4YEddHxF/z/XyccfbalL3m+2fkRx0jYiuyKcbQdGc8a1IPWa/qAnrUYymlRfn1j0XEXLIR6xdZPWL/IOMbM7Z6BHjuMJ/fBBjtJIbjyUbPx5CNCQfJxr+FBoOU0o0RcQfwrog4nWw0fELJdbSeRLF8mNvG+wRnFVmTazXc2X7tJ60M1VHWE6qi1y2pGDsA95D9DCbgf/D0n9HH2z5+dLQ7TCk9GBFXkIWzK/N/Lxu6PSLeCZxN1nOvAR4GPgy8bQ3qvxS4MCKOBg4he4nOVflt41mTeoi/MIrxJeBTEbFVSukvZEdkXpxSWtR+6fD+7iR7PdmTImJd4FX5bSPZFfhxSul/p5TmAXcD27fcfjvZ6HL34b6YbBwAsG4HNV5K1pD2Ap4D/Ns46hja11j7uR14bdvrVHbNv/buDmocj78Cz2470rbTGtzPTYz8/wudr/v1bZ/blWwc/Mga1CSpiyJiR7Je+G9kPSCALYb5HfDAGtz9pcDb89cKv4LVR+cg6wvXppRmppRuzH/HtB+Z66TnQPZaZshey30Y8O2U0tCTzqLXpJowBBYgpTQALCB7gS/AF4ATIjsj+CURsWNEvDciTmz70q0iYqe2ywSy19wdGdl7T20fETsBc4BN839HshDYPSJ2jYiXAjOBJ89czQPEOcD0yM5efnFE7BIRH8o3eZDsWd1bImJiRAx3NHLIZWTPfL9CFvge7rSO3GJgl8jOCJ4Qw5/Nex7ZKOS8iHhZROxLNlafmVJ6bJTa1sS1ZM/Kp0fEdhFxEGt2Is/XyBr2VyNih4h4ef598Oz89sXAP0XE1jHym0OfAewW2VnZ20fEYWRHVk9dg3okFeuZEbFFRGwVEa+KiE+QvTb5BuD0lJ3QcRnwzYg4OCJeFBE7R8TxQ2fajtOPyKYSFwPX5fc/ZCHwmojYOyL+ISI+x9Pf03UxsGP+u2hCjPB+himlJ8jeO/WzZOPfS1tuK3pNqglDYHHOAI6KiG1SSheRnT36HuBmskPq01j9FixDPk72DKv1ckhK6TtkZ3cdQfaakZ8BWwD/lFL68yg1fBX4PdkZtVeShZrL2rY5keyMtM+RHXH6PjAJIGXvz/cxsvc+/CPw7yPtKKV0H9nr1F7FU5+ZdlrH6WTPUBeQHYV72mtY8meYe5O9zm4ecAnwHbKz3wqVUvovsme/e5CNsKeR/R+N935+QjaK2Zvs8fw12Wtrhl7X+Hmy14zeTbbu4e7jRrIR+0HArWTB9xSyMC2pWm8G/kR2AuAvgf3JXgr0hpTS0Hj3CLKzaU8F7gAuB95AdpLHuORPeH/I8L12NtnZv98mO5N4W7LfRa0uJOv115P1nPYpQ6tL8/3clFJa0HZbYWtSfcTqo72SJEnqFx4JlCRJ6kMdnR0cEYvJzlhdCaxIKbX/bVlJUht7p6Q6G89bxLwxpfRQ1yqRpGayd0qqJcfBkiRJfaijE0Mi4l7gv8neLHJ2Sulpb1MSEdPIzqjkWc961pTJkycXXOrwVq1axTrrNDfLNn19CxcuZPvt299CsDma/viVtb4lS57FY4+tB9zwUEpp867vsCBj9c6q+ib4vdnr7J29q+y1LVy4cOS+2cnflgO2zv99PtlbnrxhtO2nTJmSyjJ37tzS9lWFpq8v+xZsrqY/fmWs78ILU4KUNt00JeD6VIO/t9npZTy9s8y+mZLfm73O3tm7yl7baH2zoyia8ncETyk9SPZ+RbuscSSVpA7dfz984hPZ9Zk9+C6J9k5JdTZmCIyI50TERkPXgT3J3sBWkromJXj/++GRR+Btb4NDDqm6ovGxd0qqu07ODp4I/DAihrb/dkrpZ12tSlLfu+giuOIK2HRTOP98yFpQT7F3Sqq1MUNgSukesj8jI0mluP9+OO647PrMmTBxYrX1rAl7p6S6a+apN5J6Vq+PgSWpVxgCJdXK0Bh4s816dgwsST3BECipNpowBpakXmEIlFQLrWPgAw+Ed76z6ookqdkMgZJqoXUMfN55joElqdsMgZIq5xhYkspnCJRUKcfAklQNQ6CkSl14oWNgSaqCIVBSZe67b/UYeNYsx8CSVCZDoKRKDI2BBwezMfA73lF1RZLUXwyBkipx4YXwi184BpakqhgCJZXOMbAkVc8QKKlUrWPggw5yDCxJVTEESirVnDmOgSWpDgyBkkpz331w/PHZ9Vmz4PnPr7YeSepnhkBJpXAMLEn1YgiUVIqhMfCECY6BJakODIGSum7xYsfAklQ3hkBJXdU6Bj74YMfAklQXhkBJXTVnDvzyl9kYeNasqquRJA0xBErqGsfAklRfhkBJXeEYWJLqzRAoqStmz3YMLEl1ZgiUVLjFi+GTn8yun3eeY2BJqiNDoKRCrVoFRx2VjYHf/vbsIkmqH0OgpELNmQO/+lU2Bp45s+pqJEkjMQRKKoxjYEnqHYZASYVwDCxJvcUQKKkQs2dnY+DNN/dsYEnqBYZASWvt3nufOgbefPNq65Ekjc0QKGmtDI2BH300GwEffHDVFUmSOmEIlLRWZs+GuXMdA0tSrzEESlpjf/rTBo6BJalHdRwCI2LdiLgpIi7vZkGSesOqVXDaaS/h0UezvwvsGHh49k5JdTWeI4HHALd3qxBJveWCC+Cmm57H5pv7ptBjsHdKqqWOQmBETAL2BS7qbjmSesG998IJJ2TXHQOPzN4pqc7W63C7s4ETgI1G2iAipgHTACZOnMjAwMBaF9eJwcHB0vZVhaavD2j0+pr4+K1aBccf/yoeffR57LrrH5kwYSENW2KRRu2dVfVNaOb3Zqumrw/snb2qVmtLKY16AfYDzsuvTwUuH+trpkyZksoyd+7c0vZVhaavL/sWbK4mPn6zZqUEKW2+eUo//OHVpe4buD6N0X/qchlv7yyzb6bUzO/NVk1fn72zd5W9ttH6Zifj4NcD+0fEYuC7wJsi4tICc6ikHtE6Bj7/fNhkk+XVFlRv9k5JtTZmCEwpnZhSmpRS2hY4BPhVSundXa9MUq2sWgVHHpm9KfQ73wkHHVR1RfVm75RUd75PoKSOXHABDAzg2cCS1BCdnhgCQEppABjoSiWSaqt9DDxhQrX19Bp7p6Q68kigpFE5BpakZjIEShrV+ednY+DnP98xsCQ1iSFQ0ojuuccxsCQ1lSFQ0rBWrYKjjoLHHoNDDoEDD6y6IklSkQyBkobVOgY+99yqq5EkFc0QKOlpHANLUvMZAiU9xdDZwI6BJanZDIGSnuK88+DXv3YMLElNZwiU9KR77oFPfSq7fsEFjoElqckMgZKAp4+B3/a2qiuSJHWTIVAS4BhYkvqNIVASd9/tGFiS+o0hUOpzrWPgQw91DCxJ/cIQKPW5WbPgyith4kTHwJLUTwyBUh+7+2749Kez6xdcAJttVm09kqTyGAKlPtU+Bv7nf666IklSmQyBUp9yDCxJ/c0QKPUhx8CSJEOg1Gdax8DvepdjYEnqV4ZAqc+0joG//vWqq5EkVcUQKPURx8CSpCGGQKlPOAaWJLUyBEp9YuZMx8CSpNUMgVIfWLRo9Rh49mzHwJIkQ6DUeENj4Mcfh8MOgwMOqLoiSVIdGAKlhps5E666KhsDn3NO1dVIkurCECg1mGNgSdJIDIFSQzkGliSNxhAoNdS552Zj4C228GxgSdLTGQKlBlq0CE48Mbs+ezZsumm19UiS6scQKDVM+xh4//2rrkiSVEeGQKlhHANLkjphCJQa5K67HANLkjozZgiMiA0i4vcRcXNE3BYRXyqjMEnj0zoGfve7HQNXzd4pqe7W62CbpcCbUkqDEbE+cHVE/DSl9Lsu1yZpHM49F66+OhsD+6bQtWDvlFRrY4bAlFICBvMP188vqZtFSRqf1jHwnDmOgevA3imp7jo5EkhErAvcAGwHzEopXTvMNtOAaQATJ05kYGCgwDJHNjg4WNq+qtD09QGNXl8Zj9+qVXDssTvx+OObsMcef2ajje6grP/Sfvj+XBtj9c6q+iY0/7Fr+vrA3tmrarW2lFLHF2ATYC6w42jbTZkyJZVl7ty5pe2rCk1fX/Yt2FxlPH5nnZUSpLTFFin97W9d391TlP39CVyfxtGz6nLppHeW2TdTan5vafr67J29q059c1xnB6eU/p43sr2Ki6GS1tRdd8FnPpNddwxcX/ZOSXXUydnBm0fEJvn1ZwF7AHd0uzBJo1u5Eo44Ijsb+D3vgbe+teqK1MreKanuOnlN4JbAt/LXtqwDfC+ldHl3y5I0lnPPhd/8Brbc0rOBa8reKanWOjk7eD7w6hJqkdSh1jHw7NnwvOdVW4+ezt4pqe78iyFSj3EMLEkqgiFQ6jFf/7pjYEnS2jMESj1k4cKnng3sGFiStKYMgVKPWLky+9vATzwB730v7Ldf1RVJknqZIVDqEa1j4LPPrroaSVKvMwRKPcAxsCSpaIZAqeaGzgZ2DCxJKpIhUKq5c86Ba66BrbZyDCxJKo4hUKqxhQvhpJOy646BJUlFMgRKNdU+Bt5336orkiQ1iSFQqinHwJKkbjIESjV0552OgSVJ3WUIlGqmdQz8vvc5BpYkdYchUKqZc86B3/7WMbAkqbsMgVKNtI+BN9mk2nokSc1lCJRqwjGwJKlMhkCpJs4+2zGwJKk8hkCpBu68Ez772ez6hRc6BpYkdZ8hUKpY6xj48MNhn32qrkiS1A8MgVLFWsfAZ51VdTWSpH5hCJQq5BhYklQVQ6BUEcfAkqQqGQKlipx1VjYG3nprx8CSpPIZAqUK3HGHY2BJUrUMgVLJVq6EI4+EpUuzcfDee1ddkSSpHxkCpZK1joHPPLPqaiRJ/coQKJXIMbAkqS4MgVJJhs4GdgwsSaoDQ6BUkrPOgt/9zjGwJKkeDIFSCRwDS5LqxhAodZljYElSHRkCpS4780zHwJKk+jEESl10333P5nOfy647BpYk1cmYITAiJkfE3IhYEBG3RcQxZRQm9bqVK2HGjJeydGn25tCOgfuLvVNS3a3XwTYrgONSSjdGxEbADRFxRUppQZdrk3ramWfC7bdvzKRJjoH7lL1TUq2NeSQwpfSnlNKN+fVHgNuBrbtdmNTLbr+dp4yBn/vcautR+eydkuqukyOBT4qIbYFXA9cOc9s0YBrAxIkTGRgYWPvqOjA4OFjavqrQ9PUBjVvfypXw0Y++hqVLN2aPPe5ngw3uoWFLfFI/fH8WYaTeWVXfhOY/dk1fHzSvd7Zq8uNXq7WllDq6ABsCNwAHjrXtlClTUlnmzp1b2r6q0PT1Zd+CzTJjRkqQ0qRJKf34x1dWXU5Xlf39CVyfOuxZdbl02jvL7JspNb+3NH19TeydrZr8+NWpb3Z0JDAi1ge+D1yWUvpB8VFUaobbb4fPfz67ftFF8Mxnrqy2IFXK3impzjo5OziAi4HbU0q+vF0awYoVcPjh2ZtCH3UUvOUtVVekKtk7JdVdJ+8T+HrgPcCbImJeftmny3VJPefMM+H3v4dJk+CMM6quRjVg75RUa2OOg1NKVwNRQi1Sz2ofA3s2sOydkurOvxgirSXHwJKkXmQIlNbSGWc4BpYk9R5DoLQWFixwDCxJ6k2GQGkNrVgBRxwBy5bB+9/vGFiS1FsMgdIaah0Dn3561dVIkjQ+hkBpDTgGliT1OkOgNE5DZwM7BpYk9TJDoDROZ5wB110Hkyd7NrAkqXcZAqVxaB8Db7xxtfVIkrSmDIFSh9rHwHvuWXVFkiStOUOg1KHTT3cMLElqDkOg1IHbboMvfCG77hhYktQEhkBpDK1vCv2BDzgGliQ1gyFQGsPQGPgFL/BNoSVJzWEIlEbhGFiS1FSGQGkErWcDf+ADsMceVVckSVJxDIHSCE47Da6/3jGwJKmZDIHSMG67Db74xey6Y2BJUhMZAqU2rWPgadMcA0uSmskQKLVpHQOfdlrV1UiS1B2GQKnFrbc6BpYk9QdDoJRrfVNox8CSpKYzBEq5U091DCxJ6h+GQImnjoEvvtgxsCSp+QyB6ntDY+Dly+Ff/gXe/OaqK5IkqfsMgep7rWPgU0+tuhpJksphCFRfcwwsSepXhkD1reXLszeFdgwsSepHhkD1rdNOgxtugG228WxgSVL/MQSqL7WPgTfaqNJyJEkqnSFQfad9DLz77lVXJElS+QyB6junnuoYWJKkMUNgRFwSEQ9GxK1lFCR10y23wJe+lF13DKxusndKqrtOjgR+E9iry3VIXdc6Bv7gBx0Dq+u+ib1TUo2tN9YGKaUrI2Lb7pciddepp8KNN2ZjYN8UWt1m71R3vZZvfKPqGrrnjju24N57q66iO+q0tjFDoNQErWPgSy5xDCypd/3lLwBXceSRVVfSTS+tuoAuqs/aCguBETENmAYwceJEBgYGirrrUQ0ODpa2ryo0fX1A19e3YkVw9NGvYfnyjdh//wdYZ527KOu/tOmPX9PX121V9U1o/mPX5PUtWrQhsDMbb7yc173uoarL6Yrly1ew/vrNPE5V9tp+9rNRbkwpjXkBtgVu7WTblBJTpkxJZZk7d25p+6pC09eXfQt211e+khKktM02KT38cNd39xRNf/zKXh9wfeqwD9XhMp7eWWbfTMnvzV521VVZT3vd66qupHua/PjVqW/6FjFqtFtugS9/ObvuGFhSEwwOZv9uuGG1daj3dfIWMd8Bfgu8JCKWRMRR3S9LWnutZwN/6EPwpjdVXZH6ib1T3WIIVFE6OTv40DIKkYo2Y0Z2NvC223o2sMpn71S3GAJVFMfBaqT581ePgS++2GYpqTkMgSqKIVCN4xhYUpMZAlUUQ6AaZ8YMuOkmx8CSmskQqKIYAtUorWPgSy6xSUpqHkOgimIIVGO0joGPPhre+MaqK5Kk4hkCVRRDoBrjlFNWj4FnzKi6GknqDkOgimIIVCPMnw9f+Up23TGwpCYzBKoohkD1PMfAkvrJUAj0LyBpbRkC1fOGxsAvfKFjYEnN55FAFcUQqJ7mGFhSvzEEqiiGQPWs1jHwhz8MU6dWXZEkdZ8hUEUxBKpnTZ++egx8yilVVyNJ5TAEqiiGQPWkm292DCyp/6S0OgQ+5znV1qLeZwhUzxkaA69Y4RhYUn9ZuhRWrgR4gvXXr7oa9TpDoHrO9Okwb55jYEn9Z+goIAyOtpnUEUOgeopjYEn9zBCoIhkC1TMcA0vqd488MnTNEKi1ZwhUzzj5ZMfAkvqbRwJVJEOgesLNN8NXv5pddwwsqV8ZAlUkQ6Bqr3UM/JGPOAaW1L8MgSqSIVC1NzQGftGLHANL6m+GQBXJEKhamzdv9Rj44ot9c1RJ/c0QqCIZAlVby5Y5BpakVoZAFckQqNqaPj07IcQxsCRlDIEqkiFQtdQ6Br7kEsfAkgSGQBXLEKjaaR0Df/SjsNtuVVckSfVgCFSRDIGqnZNPXj0Gnj696mokqT4MgSqSIVC1Mm8efO1r2XXHwJL0VIZAFckQqNpwDCxJozMEqkiGQNXG0Bj4xS92DCxJwzEEqkiGQNWCY2BJGpshUEUyBKpyy5bB+96XjYE/9jF4wxuqrkiS6skQqCIZAlW5r30N5s/PxsAnn1x1NZJUX6tD4CNVlqGG6CgERsReEXFnRCyKiE93uyj1k52eDH6OgdU09k4VzSOBKtKYITAi1gVmAXsDOwCHRsQO3S5MzbdgAcC/OgZWI9k7VbRly7LLeusBLKu6HDXAeh1sswuwKKV0D0BEfBc4AFgw0hf84Q9w7LHFFDiWJUu240c/KmdfVWjq+pYtg299C2B7XvlKx8BqpHH1zjL7JjS3twxp4vqW5blvww3h73+vthY1QychcGvgDy0fLwH+Z/tGETENmJZ9NIVzzimguo5MKmtHFWn6+v4X8+cfzYYbPlp1IVLRxuyd1fVNaH5vae76/v737HlERFRciXpdJyGwIymlOcAcgMmTd07HHVfUPY9u0aJFbLfdduXsrAJNXt9LXwp77/0+UkpVl9I1AwMDTJ06teoyuqbs9TXtl15VfROa3Vug2evbY48d2HFH7J09qk59s5MQ+AAwueXjSfnnRjRxYnljjYGBJUyd2swfdGj++qQGG1fvLLNvQvN7S9PXJxWhk7ODrwP+ISJeGBHPAA4B/qO7ZUlSz7N3Sqq1MY8EppRWRMRHgJ8D6wKXpJRu63plktTD7J2S6q6j1wSmlH4C/KTLtUhSo9g7JdWZfzFEkiSpDxkCJUmS+pAhUJIkqQ9FN95nKCL+CtxX+B0PbwLwUEn7qoLr622ur1jbpJQ2L3F/pSm5b4Lfm73O9fWu2vTNroTAMkXE9Smlnauuo1tcX29zfaqrpj92rq+3NXl9dVqb42BJkqQ+ZAiUJEnqQ00IgXOqLqDLXF9vc32qq6Y/dq6vtzV5fbVZW8+/JlCSJEnj14QjgZIkSRonQ6AkSVIfalQIjIjjIiJFxISqaylSRJwWEXdExPyI+GFEbFJ1TUWIiL0i4s6IWBQRn666nqJExOSImBsRCyLitog4puqauiEi1o2ImyLi8qpr0dqxd/aOpvZNsHdWoTEhMCImA3sC91ddSxdcAeyYUnolsBA4seJ61lpErAvMAvYGdgAOjYgdqq2qMCuA41JKOwCvBT7coLW1Oga4veoitHbsnb2j4X0T7J2la0wIBM4CTgAad6ZLSuk/U0or8g9/B0yqsp6C7AIsSindk1JaBnwXOKDimgqRUvpTSunG/PojZD/sW1dbVbEiYhKwL3BR1bVordk7e0dj+ybYO6vQiBAYEQcAD6SUbq66lhIcCfy06iIKsDXwh5aPl9CwH3aAiNgWeDVwbbWVFO5ssuCwqupCtObsnT2nL/om2DvLsl7VBXQqIn4BbDHMTScBnyEbZ/Ss0daXUvr3fJuTyA6XX1ZmbVozEbEh8H3g2JTSw1XXU5SI2B0dqEwAAAFKSURBVA94MKV0Q0RMrboejc7eae/sNfbO8vRMCEwpvXm4z0fEK4AXAjdHBGSH+2+MiF1SSn8uscS1MtL6hkTE4cB+wO6pGW/u+AAwueXjSfnnGiEi1idrYpellH5QdT0Fez2wf0TsA2wAbBwRl6aU3l1xXRqGvbNRvbPRfRPsnWVr3JtFR8RiYOeU0kNV11KUiNgLOBPYLaX016rrKUJErEf2Qu3dyZrYdcC7Ukq3VVpYASL7jfot4L9SSsdWXU835c9mj08p7Vd1LVo79s76a3LfBHtnFRrxmsA+MBPYCLgiIuZFxAVVF7S28hdrfwT4OdmLf7/XlEZG9mzvPcCb8sdrXv7MT1K5GtU7G943wd5ZusYdCZQkSdLYPBIoSZLUhwyBkiRJfcgQKEmS1IcMgZIkSX3IEChJktSHDIGSJEl9yBAoSZLUh/4/k4Zhjw12ET4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(11,4))\n",
    "\n",
    "plt.subplot(1, 2, 1)\n",
    "# coordinate \n",
    "plt.plot([-7, 7], [0, 0], 'k-', linewidth=1)\n",
    "plt.plot([0, 0], [-2.2, 5], 'k-', linewidth=1)\n",
    "# Plot\n",
    "plt.plot(z, relu(z), \"b-\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "plt.title(\"ReLU activation function\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.2, 5])\n",
    "\n",
    "plt.subplot(1, 2,2)\n",
    "# coordinate \n",
    "plt.plot([-7, 7], [0, 0], 'k-', linewidth=1)\n",
    "plt.plot([0, 0], [-2.2, 5], 'k-', linewidth=1)\n",
    "# Plot\n",
    "plt.plot(z, derivative(relu, z), \"b-\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "plt.title(\"Derivative\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.2, 5])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to use it with Keras and TensorFlow 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.core.Dense at 0x84a17e350>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "Dense(10, activation='relu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0. , 0. , 0. , 1.2], dtype=float32)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.activations import relu\n",
    "\n",
    "z = tf.constant([-20, -1, 0, 1.2], dtype=tf.float32)\n",
    "output = relu(z)\n",
    "output.numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Leaky ReLU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def leaky_relu(z, alpha=0.05):\n",
    "    return np.maximum(alpha * z, z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApMAAAEJCAYAAADIA6xFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de7xcdX3v/9c7IdzkEiAkhGRzUxJARCABRAQ2lwoBCURoC9UqLW1Oe+TUXvz1h/XUcjynPdraetqDt2gp2FLQioSAQRDMFiqIYEBuISFAIAkhXANskxB28jl/fNeYYTL7vvZaM2u/n4/HPGbNrDWzPt89e3/3e9Z3XRQRmJmZmZkNxZiyCzAzMzOz9uUwaWZmZmZD5jBpZmZmZkPmMGlmZmZmQ+YwaWZmZmZD5jBpZmZmZkPmMFkySSHpgrLraGeSLpbUXdC6Cvm8JJ0g6SFJmyR1jfT6+qnlgKzdM8usw8wGpop9orU2h8k+SLpK0s1l1zEYki7P/rhD0hZJz0m6RlLHIN+nS9IVvcxbIelTvaz7kaHWPsC6mnVc3wYOynk9vX32k4Gb8lxXL/4R+AXwTuDDBawP6PVzX0lq94NF1WFWZVn/Uuun35L0gqRFkj4haVwOq6hin2gtzGGympaS/sCnAr8JvAf4TqkVjaCI2BARLxS0rucj4s0CVvUu4EcRsTIiXilgfb2KiM1Zu3vKrMOsYm4n9dMHAB8kBbL/Adwl6R1DfVNJ4yraJ1oLc5gcBkmHSfq+pDeyb5bXStqnbv4xkm6T9JKk1yX9p6Tj+3nP/z9b/oTsNRc0zP+17JvspD7epif7A38uIu4CvgG8T9Jude9zjqSfS9oo6WlJfy1p+yH+KAZE0lhJ/5ytb4OkJyT9uaQxDct9XNLDkt6UtFbS1dnzK7JF/iP7Rr8ie/5XQzqSpmXz3tPwnnOzn+u4/uqQdDnwceDsuq0Hndm8t20ZlfQeSbdn7/NK9u1997r5V0m6WdInJa2W9Kqkf5G0cy8/owMkBbA7cGW2vosldWbTExqXrQ0/1y1zmqR7Ja2XdL+koxvW8T5JP5L0S0mvZdP7SroKOBn4RF27D2hcT/YeJ2Xr2Jh9Rl+q//3JtnB+RdLfZD/3FyR9sfGzNhvF3sz66dUR8WBE/APQCRwN/DmApO0lfUHSquzv+T5JZ9TeoO5v/ixJP5O0CTijrD5R0t2S/r5hPbtl7/nhgbTJ2pM79iGSNBm4E3gEOBY4HdgFuLHuH+auwL8CJ2bLPAgslLRXk/eTpC8C/w04OSJ+AlwL/G7Dor8L3BwRawdY5z6kYdLN2Y3sD/ca4Arg3dl7XgD8zYAaP3RjgNXAbwCHAp8B/gL4nbp6/wvwdeBfgCOAs0g/Y4BjsvvfJ32jrz3+lYhYBtwHfKRh1keA70TEWwOo44ukLbm1LQeTgbsb16W09eBWoJv0+c4B3g9c2bDoicDhpN+R38yW+2Tj+2VqQ8rrgT/Opr/dy7K9+d/AZaR/Si8D10hSVvN7gUXAcuAE4H3Z+2+X1XQP6Wdfa/fKJu2eAtwCPAAcBVwCXJStt95HgB7Sz+TSrD2/Oci2mI0aEfEI8APg/OypfyF9wfstUh9yNXBT9ndc7wvAfwcOAe5teM/C+kTg34ALG740ng9sBL4/yDZZO4kI33q5AVeRgluzeZ8D7mh4bg8ggGN7eY2ANcBH654L0j/YfwGWAfvXzZtJ+mc8pe79NwAf6qPmy0mhsZsUSCK7/WPdMncCf9nwuvOy1yh73AVc0cs6VgCf6mXdjwzyZ/x54Pa6x6uAz/exfAAXNDx3MdBd9/iPgGfq2rIfsAV4/yDqaPrZ16+fFGpfA3atm9+ZLfOuuvdZCYytW+Yb9evqpZ5u4OIm7zuh7rkDsudmNixzRt0yJ2TPTc0eXwPc08d6t/ncm6znr4EngDENn8GbwM5173NPw/v8EPjmUP8effOtKrfe+pds3uezvvudWb+1X8P8+cBXsuna3/z5DcuU1SfuBWwCTqubfzswL5vut02+tefNWyaHbgZwkqTu2o2tW3HeCSBpoqSvS1om6TXgDWAi6Q+53hdJncIHIuKZ2pMRcT/wMGl4AdI3uVdIW4X68iRwJGnL3WeAxaRvmfW1f6ah9n8H3gHs0/hmeZL0B9nQ64vZev+E7OchaSIwBbhjmKu5DtiXtEUQ0lazpyPiV9+k+6pjEA4FHoqIN+qeu5vUWR5W99xjEbG57vFzpN+DkfJQw7qoW99RwI+G+f6HAj+NiC11z/0nsD1pX89mddRqGcl2m1WBSAHt6Gz6sYa++myy/zF17u/nPQvpEyPiZdKW1Y9k77kvcAppiyWDbJO1ke3KLqCNjSFttt/mqGagNgR9NTCJ9Ee5grTl5g7SP916PyT9cZ9F+vZX75uk4ce/IQ1HX90QTJrZFBHLs+lHJR0MfJn0bbVW+/8A/qPJa1/s570BXift09doPGlLXVOSfhP4P6Sf2d3Z+3yCNOybm4h4QdIPSR3andn9NQXXEXXTbzWZN9gvcrXgprrnejvqs359tTqK+uKYd7vNRpvDgKdIfytB2ijQ+Le0oeHxL/t6w4L7xH8DviHpvwIXkjay3JXNG0ybrI04TA7dYtL+Jc9E2uekmQ8AfxQR3wdQOmhmcpPlFgLfIzuwJCKurpt3DfB3ki4lfau7cAi1/i9gqaT/GxE/z2o/pC5wDtZS0tbNRkdn83rzAeDeiPjVqWck/erbaNbhrQZOIwXsZt4Cxg6gxn8DrpA0j3Q0e/2BTH3Wkdk0gPUsAX5X0q51WyffT+owlwygxsGohfzJddNHDuF9HgBO7WP+QNv9G5LG1G2d/ED22ieHUJOZAZIOB84k9dkPkL487hMRi3J4+yL6RIAFpF15PkQKrf8eEbUvmXm3yVqEtxL0bzdJRzbcDiBt6dsd+Lak4yQdJOl0SfMk7Zq9dhnwUaWjvo8hDTVsaraSiLgZ+HXga5I+Vvf8OtIWxL8H7oyIJwbbgIh4ErgR+J/ZU58DfkvS5yQdLukQSRdI+tuGl05o0vZ9gS+Rjhj8y6xt75b018Dx2bzeLAOOljRL0sGS/pK0I3a9vwb+WNKfKB2FeKSkP6ubvwI4TdI+kvboY13zSVvu/hm4L9JO6IOpYwVwuKTpkiao+bnfriHt2/QtpaO6TyIdPPS9YQT13iwnfcO/PPu5fJC0w/1g/R1wVPZ7+t6sfb8nqTactQI4VukI7glqfvT1V0hDZl+RdKiks0n7V10REeuHUJPZaLRD1o/tm/0t/ilpX+OfA1/M+qxrgKuy/vkgSTMlfUrZkdGDVESfSERsBK4n9U9Hs3WImxFok7WKsnfabOUbacg5mty+m80/GPgu8CppE/1S4P8C22fz30s6sm4DaYvNb5OOTL68bh1vO6AEOCdb/mN1z52ULfexAdR8OU0OgiFtMQuyHa5J5zW7ixSGXiftc3Np3fJdvbT9iw2vf4V0xHAXcFI/tW1P6sheBdZl058FVjQsdwnwGCl4Pw9c2fDzeYK0hXJF9tzF1O1sXrfst7Ka/2iwdQB7A7eR9nMNoLOXz+s9pF0XNmTvdxWwe8Pv0M0N62/6GTUs87YDcOo+wwezdd1D2s+o2QE4vR6kkz33AdJQ14as/bcDk7N507L3rh28dUAv73ES6Xf7TdJuHV8Cdmj4/Wk8kGebn4Vvvo3GG2//39IDvJT9zVxK9v8jW25c1l88VdcfLgBmZPO3+ZvPni+tT8yeOzV7fnGTGvpsk2/teasd2WUtLNuf5evAvuEtP2ZmZtZCvM9kC1M6sfU+pCOxv+EgaWZmZq1m2PtMSupQuqboY5IelbTNyZiV/JOk5ZIeUsMVOaxXf04aOn+Frfs7mlkFSLpS6cpATa9nL+kjWX/5sNKVRXxSZzNrScMe5la6EszkiFicHXjyc+C8iHisbpmzSFd2OQs4jnQC7eOGtWIzszaWHbDVDXwrIg5vMv/9wJKIeFXSLNK+1u43zazlDHvLZESsiYjF2fQbpNOGTGlY7FxShxkR8VNgfBZCzcxGpYi4kzTq0Nv8uyPi1ezhT4GphRRmZjZIue4zmZ0y5ygarg1KCpf11/hdlT23puH1c4G5ADvttNOMjo6OPMvr05YtWxgzprpnSqpy+1auXMmGDRuYNm1a2aXk7o03xrFmzY7suGMP++1X3XP6Fvn7uWzZspciYu9CVpafS+jjylfuO0dO1du3bNmySvadNVX+/IpuW599Z16HhQO7kIa4P9xk3s2kSwXWHt9B3WlGmt1mzJgRRVq0aFGh6ytaldt38sknR/pVrp4LLoiAiEsvXVZ2KSOqyN9P4P5ogVNppFI4gP5PE3UKacRnr4G8p/vOfFW9fVXtO2uq/PkV3ba++s5cIm128tLrgWsi4ntNFlkN1H9Vnpo9Z2a92LABFi5M0yee+FK5xVgpJB1BuqTquZGue2xm1nLyOJpbpBOcLomIf+hlsQXAx7Kjut8HvBYRa3pZ1syA226D9evhmGNg4sQ3yy7HCpZdleh7wG/H269WYmbWUvLYZ/IE0pVdHpb0YPbcXwD7AUTE10jXnj6LdEm49cDv5LBes0r7XraN/8O+yFglSbqWdAWTCZJWAX9FujpIrd/8LLAX6bKVAD0RMbOcas3MejfsMBkR/0m6cHtfywTwieGuy2y0eOstWLAgTX/4w/Dcc+XWY/mLiIv6mf97wO8VVI6Z2ZBV8xAnszbX1QXr1sG73w0VPtDSzMwqwGHSrAV5iNvMzNqFw6RZi9m8GW64IU07TJqZWatzmDRrMffcA2vXwkEHwXt9NWYzM2txDpNmLaZ+iFt9HtpmZmZWPodJsxYS4f0lzcysvThMmrWQBx6AZ56ByZPhuOPKrsbMzKx/DpNmLaS2VXLOHBjjv04zM2sD/ndl1kI8xG1mZu3GYdKsRSxZkm577gknnVR2NWZmZgPjMGnWImpbJWfPhnHjyq3FzMxsoBwmzVpELUyef365dZiZmQ2Gw6RZC1ixAhYvhl12gdNPL7saMzOzgXOYNGsBtcsnnn027LhjubWYmZkNhsOkWQvwUdxmZtauHCbNSvb88/CTn8AOO8CsWWVXY2ZmNjgOk2Ylu/HGdBnFD34Qdt217GrMzMwGx2HSrGQe4jYzs3bmMGlWoldfhR/9CMaOhXPOKbsaMzOzwXOYNCvRTTdBTw90dsJee5VdjZmZ2eA5TJqVyCcqNzOzducwaVaS7m649VaQ4Lzzyq7GzMxsaBwmzUrygx/Axo1w/PEweXLZ1ZiZmQ2Nw6RZSXwUt5mZVYHDpFkJ3nwTbr45Tc+ZU24tZmZmw+EwaVaCO+6AN96AI4+Egw4quxozM7Ohc5g0K8H116d7D3GPXpKulPSCpEd6mS9J/yRpuaSHJB1ddI1mZgPhMGlWsJ6edAlFcJgc5a4Czuxj/izg4Ow2F/hqATWZmQ3admUXYDba3HUXvPwyTJ8Ohx1WdjVWloi4U9IBfSxyLvCtiAjgp5LGS5ocEWsKKdAq7/XXAe5g+vSyKxk5GzYcy047lV3FyGiltjlMmhWs/ihuqdxarKVNAVbWPV6VPbdNmJQ0l7T1kkmTJtHV1VVEfQB0d3cXur6iVbl99923B3Aqy5aVXclI2rnsAkZQ67TNYdKsQFu2wA03pGkPcVteImIeMA9g5syZ0dnZWdi6u7q6KHJ9Raty+9avT/cnngjf+Ea5tYyUe++9l+OOO67sMkZE0W075JDe5zlMmhXovvtg9Wro6IAZM8quxlrcaqCj7vHU7DmzXGzenO53243KDnWvWbPBbSuAD8AxK5CHuG0QFgAfy47qfh/wmveXtDzVwuTYseXWYe3PWybNChLhq97YVpKuBTqBCZJWAX8FjAOIiK8BC4GzgOXAeuB3yqnUqsph0vLiMGlWkEcegeXLYe+94YQTyq7GyhYRF/UzP4BPFFSOjUIOk5aXXIa5B3Dy3U5Jr0l6MLt9No/1mrWT2onKzzvPnbeZlc9h0vKS15bJq4ArgG/1scxdEfGhnNZn1nY8xG1mrcRh0vKSy5bJiLgTeCWP9zKroieegIcfht13h1NPLbsaMzOHSctPkftMHi/pF8BzwKci4tHGBXzi3ZFT5fatW7cOoKXbd+21HcA7OfbY57n77scH/foqf35Q/faZtSKHSctLUWFyMbB/RHRLOguYT7re7Nv4xLsjp8rtGz9+PEBLt++yy9L9H/7hPnR27jPo11f584Pqt8+sFTlMWl4KOc9kRLweEd3Z9EJgnKQJRazbrGyrVsG998JOO8EZZ5RdjZlZ4jBpeSkkTEraR0qnaJZ0bLbel4tYt1nZ5s9P97Nmwc6tcylVMxvlenrSvcOkDVcuw9wDOPnuBcAfSuoBNgAXZudQM6s8H8VtZq2otmVyO59x2oYpl1+hAZx89wrSqYPMRpUXX4Qf/xjGjYOzzy67GjOzrTzMbXnxtbnNRtCCBbBlC5x2GmTHCZmZtQSHScuLw6TZCKoNcZ9/frl1mJk1cpi0vDhMmo2Q116D22+HMWNg9uyyqzEzezuHScuLw6TZCFm4EDZtghNPhIkTy67GzOztHCYtLw6TZiPER3GbWStzmLS8OEyajYANG9KWSYA5c8qtxcysGYdJy4vDpNkIuO02WL8ejjkGOjrKrsbMbFsOk5YXh0mzEXD99eneQ9xm1qocJi0vDpNmOdu0CW66KU07TJpZq3KYtLw4TJrlrKsL1q2Dd78bpk0ruxozs+YcJi0vDpNmOfOJys2sHThMWl4cJs1ytHkzzJ+fpj3EbWatzGHS8uIwaZaje+6BtWvhoIPgiCPKrsbMrHcOk5YXh0mzHNWfqFwqtxYzs744TFpeHCbNchLhq96YWftwmLS8OEya5WTxYnjmGZg8GY47ruxqzMz65jBpeXGYNMtJbavknDkwxn9ZZtbiHCYtL/6XZ5YTD3HbYEk6U9JSScslXdZk/n6SFkl6QNJDks4qo06rJodJy4vDpFkOliyBxx+HPfeEk08uuxprB5LGAl8GZgGHARdJOqxhsf8OfCcijgIuBL5SbJVWZQ6TlheHSbMc1LZKnnsubLddubVY2zgWWB4RT0XEJuA64NyGZQLYLZveHXiuwPqs4hwmLS/+t2eWAw9x2xBMAVbWPV4FNB66dTlwm6T/BrwDOL3ZG0maC8wFmDRpEl1dXXnX2qvu7u5C11e0Krdv7drDgQksWfIIXV0vlV3OiKjy59dKbXOYNBumFSvSkdy77AKnN/1XbzZkFwFXRcTfSzoe+FdJh0fElvqFImIeMA9g5syZ0dnZWViBXV1dFLm+olW5fXvske7f+97DqWgTK/35tVLbPMxtNkw33JDuzz4bdtyx3FqsrawGOuoeT82eq3cJ8B2AiLgH2BGYUEh1Vnke5ra8OEyaDZOHuG2I7gMOlnSgpO1JB9gsaFjmWeA0AEmHksLki4VWaZXV05PuHSZtuBwmzYbh+efhJz+BHXaAWbPKrsbaSUT0AJcCtwJLSEdtPyrpc5JmZ4v9GfD7kn4BXAtcHBFRTsVWNbUtkz5o0IbLv0JmwzB/frqM4gc/CLvuWnY11m4iYiGwsOG5z9ZNPwacUHRdNjp4mNvy4i2TZsPgIW4za1cOk5YXh0mzIXrlFVi0KHXEs2f3v7yZWStxmLS8OEyaDdHNN6cd2E85JV35xsysnThMWl4cJs2GyEPcZtbOHCYtLw6TZkPQ3Q233goSnHde2dWYmQ2ew6TlxWHSbAh+8APYuBGOPx4mTy67GjOzwXOYtLw4TJoNwfXXp3sPcZtZu3KYtLw4TJoN0saN6eAbgDlzyq3FzGyoHCYtL7mESUlXSnpB0iO9zJekf5K0XNJDko7OY71mZbjjjrTP5JFHwkEHlV2NmdnQOExaXvLaMnkVcGYf82cBB2e3ucBXc1qvWeFqR3Gff365dZiZDYfDpOUllzAZEXcCr/SxyLnAtyL5KTBekg9bsLbT0wM33pimvb+kmbUzh0nLS1HX5p4CrKx7vCp7bk39QpLmkrZcMmnSJLq6ugoqD7q7uwtdX9Gq3L5169YBFNK+Bx4Yz8svH0lHx3rWrv0ZL7ww4qsEqv35QfXbZ9aKHCYtL0WFyQGJiHnAPICZM2dGZ2dnYevu6uqiyPUVrcrtGz9+PEAh7asdxf3Rj+7MKaeM/Ppqqvz5QfXbZ9aKHCYtL0Udzb0a6Kh7PDV7zqxtbNkCN9yQpj3EbWbtzmHS8lJUmFwAfCw7qvt9wGsRsaa/F5m1kp/9DFavho4OmDGj7GrMzIbHYdLyksswt6RrgU5ggqRVwF8B4wAi4mvAQuAsYDmwHvidPNZrVqT6a3FL5dZiZjZcDpOWl1zCZERc1M/8AD6Rx7rMyhDx9jBpZtbuHCYtL74CjtkAPPwwPPkkTJwIJ5xQdjVmZsPnMGl5cZg0G4DaVsnzznPHa2bV4DBpeXGYNBsAD3GbWdU4TFpeHCbN+vHEE2mYe/fd4ZRTyq7GzGz4ItLpzsBh0obPYdKsH7VzS55zDmy/fbm1mJnloRYkYYvPTmHD5jBp1o/aVW88xG1mVVEb4obNfS1mNiAOk2Z9WLkynax8p53gjDPKrsbMLB8Ok5Ynh0mzPsyfn+5nzYKddy63FjOzvPT01KYcJm34HCbN+uCjuM2sirZumezpazGzAXGYNOvFiy/CnXfCuHHwoQ+VXY1VkaQzJS2VtFzSZb0s8xuSHpP0qKR/L7pGqyYPc1uecrmcolkVLViQjng844x0WiCzPEkaC3wZ+DVgFXCfpAUR8VjdMgcDnwZOiIhXJU0sp1qrGodJy5O3TJr1wkPcNsKOBZZHxFMRsQm4Dji3YZnfB74cEa8CRMQLBddoFeUwaXnylkmzJl57DW6/HcaMgdmzy67GKmoKsLLu8SrguIZlpgFI+gkwFrg8In7Q+EaS5gJzASZNmkRXV9dI1NtUd3d3oesrWlXb9+KLOwDHA5sr2b6aqn5+0Fptc5g0a+L734dNm+Dkk2GiBxatPNsBBwOdwFTgTknviYh19QtFxDxgHsDMmTOjs7OzsAK7uroocn1Fq2r7nnmmNrW5ku2rqernB63VNg9zmzXhIW4rwGqgo+7x1Oy5equABRHxVkQ8DSwjhUuzYfEwt+XJYdKswfr1cMstaXrOnHJrsUq7DzhY0oGStgcuBBY0LDOftFUSSRNIw95PFVmkVZPDpOXJYdKswW23pUB5zDHQ0dH/8mZDERE9wKXArcAS4DsR8aikz0mq7al7K/CypMeARcD/FxEvl1OxVYnDpOXJ+0yaNfAQtxUlIhYCCxue+2zddAB/mt3McuMwaXnylkmzOps2wU03pWmHSTOrKodJy5PDpFmdri5Ytw4OPxymTSu7GjOzkeEwaXlymDSr4yFuMxsNHCYtTw6TZpnNm2H+/DTtMGlmVeYwaXlymDTL3H03rF0LBx0ERxxRdjVmZiPHYdLy5DBplqkf4pbKrcXMbCQ5TFqeHCbNgAjvL2lmo4fDpOXJYdIMWLwYnn0W9t0Xjjuu7GrMzEaWw6TlyWHSjK1bJefMgTH+qzCzinOYtDz536YZHuI2s9HFYdLy5DBpo96SJfD447DnnnDSSWVXY2Y28hwmLU8Okzbq1bZKnnsubOer1ZvZKOAwaXlymLRR7/rr072HuM1stHCYtDw5TNqo9vTT8MADsMsucPrpZVdjZlYMh0nLk8OkjWo33JDuzz4bdtyx3FrMzIriMGl5cpi0Uc1HcZvZaNTTU5tymLThc5i0UWvNmnQ97h12gFmzyq7GzKw4W7dM9vS1mNmA5BImJZ0paamk5ZIuazL/YkkvSnowu/1eHus1G44bb0yXUTzjDNh117KrMTMrjoe5LU/DPhGKpLHAl4FfA1YB90laEBGPNSz67Yi4dLjrM8uLh7jNbLRymLQ85bFl8lhgeUQ8FRGbgOuAc3N4X7MR88orsGgRjB0L55xTdjVmZsVymLQ85XGK5inAyrrHq4Djmix3vqSTgGXAn0TEysYFJM0F5gJMmjSJrq6uHMobmO7u7kLXV7Qqt2/dunUAg2rfD34wiZ6eQ5kx4xUeeuihEaosP1X+/KD67TNrNQ6TlqeirvdxE3BtRLwp6b8AVwOnNi4UEfOAeQAzZ86Mzs7OgspLQaTI9RWtyu0bP348wKDa96UvpftLLtmzLX4uVf78oPrtM2s1DpOWpzyGuVcDHXWPp2bP/UpEvBwRb2YPvwnMyGG9ZkPS3Q233goSnHde2dWYmRXPYdLylEeYvA84WNKBkrYHLgQW1C8gaXLdw9nAkhzWazYkt9wCb74Jxx8Pkyf3v7yZWdU4TFqehj3MHRE9ki4FbgXGAldGxKOSPgfcHxELgD+SNJt0QqtXgIuHu16zofJR3GY22jlMWp5y2WcyIhYCCxue+2zd9KeBT+exLrPh2LgRbr45TTtMWtkknQn8I+mL+Dcj4vO9LHc+8F3gmIi4v8ASraIcJi1PvgKOjSp33JH2mTzqKDjwwLKrsdGs7hy9s4DDgIskHdZkuV2BTwL3FluhVZnDpOXJYdJGFQ9xWwsZ6Dl6/yfwBWBjkcVZtTlMWp4cJm3U6OlJl1AEh0lrCc3O0TulfgFJRwMdEfH9Iguz6nOYtDwVdZ5Js9LdeSe8/DJMnw6HHlp2NWZ9kzQG+AcGcMCiL/gwcqravqeeOhDYH9hcyfbVVPXzg9Zqm8OkjRr1Q9xSubWY0f85encFDge6lH5h9wEWSJrdeBCOL/gwcqravltuqU1trmT7aqr6+UFrtc3D3DYqbNkCN9yQpj3EbS2iz3P0RsRrETEhIg6IiAOAnwLbBEmzofAwt+XJYdJGhZ/9DJ57Djo6YIavv2QtICJ6gNo5epcA36mdozc7L6/ZiHGYtDx5mNtGBQ9xWyvq7xy9Dc93FlGTjQ4Ok5Ynb5m0yovYGibPP7/cWszMWoHDpOXJYdIq7+GH4cknYeJEeP/7y67GzKx8DpOWJ4dJq7zaVsnzzoOxY8utxcysFThMWp4cJq3yrr8+3fsobjOzxGHS8uQwaZW2bBk88gjsvjucckrZ1ZiZtQaHScuTw6RVWu3ckuecA9tvX24tZmatwmHS8uQwaZVWf0ogMzNLHCYtT5ICUKYAAA1kSURBVA6TVlkrV6aTle+8M5xxRtnVmJm1DodJy5PDpFXW/PnpftasFCjNzCzp6alNOUza8DlMWmV5iNvMrLmtWyZ7+lrMbEAcJq2SXnwR7rwTxo2Ds88uuxozs9biYW7Lk8OkVdKNN8KWLXD66em0QGZmtpXDpOXJYdIqyUPcZma9c5i0PDlMWuW89hrcfjuMGQOzZ5ddjZlZ63GYtDw5TFrlfP/78NZbcOKJMHFi2dWYmbUeh0nLk8OkVY6HuM3M+uYwaXlymLRKWb8ebrklTc+ZU24tZmatymHS8uQwaZVy220pUB57LHR0lF2NmVlrcpi0PDlMWqV4iNvMrH8Ok5Ynh0mrjE2bYMGCNO0hbjOz3jlMWp4cJq0yFi1KpwU6/HCYNq3saszMWpfDpOXJYdIqw0PcZmYD4zBpeXKYtMqYPz/dO0yamfXNYdLy5DBpFbEbL7wABx0ERxxRdi1mZq3NYdLy5DBpFbE3kLZKSiWXYmbW4hwmLU8Ok1YREwA4//ySyzAzawMOk5anXMKkpDMlLZW0XNJlTebvIOnb2fx7JR2Qx3rNANasAdiR/fZLJys3axcD6Dv/VNJjkh6SdIek/cuo06rHYdLyNOwwKWks8GVgFnAYcJGkwxoWuwR4NSLeBXwJ+MJw12sG8OKL8NRTafpv/xbGeFu7tYkB9p0PADMj4gjgu8DfFlulVZXDpOVpuxze41hgeUQ8BSDpOuBc4LG6Zc4FLs+mvwtcIUkREb296dKlS+ns7MyhvIFZt24d48ePL2x9RatS+7ZsgQ0b0m3NGujpeRDo4atf7eSrXy27upFRpc+vmaq3rxf99p0Rsahu+Z8CHy20Qqssh0nLUx5hcgqwsu7xKuC43paJiB5JrwF7AS/VLyRpLjC39vjHP/5xDuVZ+9oO2LnJbUeg8SibDf59sXYzkL6z3iXALc1m1PedkyZNoqurK6cS+9fd3V3o+opW1fa9+eYJwDhgcyXbV1PVzw9aq215hMncRMQ8YB7AzJkz4/777y9s3V1dXYVuCS1aq7avpwdWrIDHH996W7o03b/0UvPXjBkDBx4IhxySbosWdbJ48Y/pY0N322vVzy8vRbZPbXi4v6SPAjOBk5vNb+w7i/xd8e9me9r6Z7C5ku2rqernB63VtjzC5Gqgo+7x1Oy5ZsuskrQdsDvwcg7rtjbxxhtbQ2L97Ykn0jW1m9lllxQWp0/fGhwPOQTe9S7Yccety7XI35LZYA2k70TS6cBngJMj4s2CarOK8zC35SmPMHkfcLCkA0kd4YXAbzUsswD4OHAPcAHwo772l7T2FAGrVm0bGB9/HJ57rvfXTZ369rBYu+27r88ZaZXWb98p6Sjg68CZEfFC8SVaVTlMWp6GHSazfSAvBW4FxgJXRsSjkj4H3B8RC4B/Bv5V0nLgFVKnaW1qw4a0RbF+SLo2/ctfNn/NDjvAtGlvD4vTp6fndt212PrNWsEA+86/A3YB/iMbnn82ImaXVrRVhsOk5SmXfSYjYiGwsOG5z9ZNbwR+PY91WTEi0ml3mm1lXLEizW9m4sRth6UPOQT23x/Gji20CWYtbwB95+mFF2WjQk9Pbcph0oavpQ7AseK99VY6T2NjYFy6FF59tflrxo6Fd75z28A4fTrsuWex9ZuZ2eBEpFOsJQ6TNnwOk6NEd/d23HvvtqFx+fL6b6hvt9tucOihbw+LhxySguT22xdbv5mZ5aMWJCUqfRYMK47DZIVs2QLPPtt8aHrt2g/0+rr9929+AMykST4Axsysamr7S44d2/vGBLPBcJhsQ7/8JSxbtu15GZcuhY0bm79mhx02c+ihY7cJjAcfDDvvXGz9ZmZWHodJy5vDZIuKgOefb76V8dlne3/d5MnbDksfcgg8+eRdnHpqZ0HVm5lZq6oPk2Z5cJgs2aZNab/FZgfAvP5689eMG5dO3N3sAJjdd2/+mqefHrk2mJlZ+3CYtLw5TBbk5Ze3PS/j44+nI6k393Iw3R57vP0AmNrtwANhO39yZmY2BA6TljdHkhxt3rztdaZrt76uM107zU7j+RknTPABMGZmli+HScubw+QQNF5nuja9bFnv15l+xzuaHzHdeJ1pMzOzkeQwaXlzmOxFBKxe3Xwr4+rVvb+u2XWmp0+HKVO8ldHMzMrnMGl5G/VhcuPGdJ3prq69ueuuwV1nunFY2teZNjOzVucwaXkbFWGy/jrTjQfAPP107TrT797mdXvv3Xxo2teZNjOzduUwaXmrVJh8660UDpsNTfd3nekJE17iAx+Y8Kth6enTYa+9iq3fzMxspDlMWt7aMkyuW7ftFsalS9P5Gt96q/lrdtut+VbG2nWmu7oeobOzs9B2mJmZFc1h0vLW0mHymWeab2V8/vneX+PrTJuZmfXOYdLy1rJhcvFiOOCA5vN22mnrwS/1B8FMm+brTJuZmfXFYdLy1rJhMgL22af5VsaOjnSybzMzMxsch0nLW8uGySOPhAceKLsKMzOzanGYtLy17PY9/5KbmZnlz2HS8tayWybNzGzwnnkG/uAPilvfc89N47rriltf0arYvrVr073DpOXFYdLMrEJeegm+/vUi17hvkSsrQXXbt/feZVdgVeEwaWZWIfvtB5/+dHHrW7ZsGdOmTStuhQWravvGjIGzz4apU8uuxKrAYdLMrEL23rvYYe6urufo7Kxe2KqpevvM8tCyB+CYmZmZWetzmDQzMzOzIXOYNDMzM7Mhc5g0MyuJpDMlLZW0XNJlTebvIOnb2fx7JR1QfJVmZn1zmDQzK4GkscCXgVnAYcBFkg5rWOwS4NWIeBfwJeALxVZpZtY/h0kzs3IcCyyPiKciYhNwHXBuwzLnAldn098FTpOkAms0M+uXTw1kZlaOKcDKusergON6WyYieiS9BuwFvFS/kKS5wNy6xyNRr1WUf19suLxl0syszUXEvIiYGREzZ8yYQUQUdlu0aFGh6yv6VvX2Zb8/lb1V+fMrum19cZg0MyvHaqCj7vHU7Lmmy0jaDtgdeLmQ6szMBshh0sysHPcBB0s6UNL2wIXAgoZlFgAfz6YvAH4U/W0iMDMrmPeZNDMrQaR9IC8FbgXGAldGxKOSPgfcHxELgH8G/lXScuAVUuA0M2spDpNmZiWJiIXAwobnPls3vRH49aLrMjMbjGENc0vaU9IPJT2R3e/Ry3KbJT2Y3RqHcczMzMysTQ13n8nLgDsi4mDgjuxxMxsi4sjsNnuY6zQzMzOzFjHcMFl/Qt2rgfOG+X5mZmZm1kY0nAMDJa2LiPHZtEiX/RrfZLke4EGgB/h8RMzv5f3qT7w7HVg65OIGbwINJwKuGLevvbl9+dk/IvYuaF2Fk/Qi8EyBq/TvZntz+9pX0W3rte/sN0xKuh3Yp8mszwBX14dHSa9GxDb7TUqaEhGrJR0E/Ag4LSKeHEwLRpqk+yNiZtl1jBS3r725fdaqqv7ZuX3trcrta6W29Xs0d0Sc3ts8SWslTY6INZImAy/08h6rs/unJHUBRwEtFSbNzMzMbPCGu89k/Ql1Pw7c2LiApD0k7ZBNTwBOAB4b5nrNzMzMrAUMN0x+Hvg1SU8Ap2ePkTRT0jezZQ4F7pf0C2ARaZ/JVgyT88ouYIS5fe3N7bNWVfXPzu1rb1VuX8u0bVgH4JiZmZnZ6OZrc5uZmZnZkDlMmpmZmdmQOUw2IenPJEV2wFBlSPo7SY9LekjSDZK2OSdou5F0pqSlkpZL6u0KTG1JUoekRZIek/SopE+WXdNIkDRW0gOSbi67Fhse953tw31n+2ulvtNhsoGkDuCDwLNl1zICfggcHhFHAMuAT5dcz7BIGgt8GZgFHAZcJOmwcqvKVQ/wZxFxGPA+4BMVa1/NJ4ElZRdhw+O+s32476yMluk7HSa39SXgz4HKHZkUEbdFRE/28KfA1DLrycGxwPKIeCoiNgHXkS7xWQkRsSYiFmfTb5A6jSnlVpUvSVOBs4Fv9restTz3ne3DfWeba7W+02GyjqRzgdUR8YuyaynA7wK3lF3EME0BVtY9XkXFOowaSQeQTvZ/b7mV5O7/kALIlrILsaFz39l23He2v5bqO/u9Ak7V9HN5yL8gDdO0rb7aFxE3Zst8hjQMcE2RtdnQSNoFuB7444h4vex68iLpQ8ALEfFzSZ1l12N9c9/pvrPduO8szqgLk71dHlLSe4ADgV9IgjSMsVjSsRHxfIElDktfl78EkHQx8CHS9dHbfThqNdBR93hq9lxlSBpH6gyviYjvlV1Pzk4AZks6C9gR2E3Sv0XER0uuy5pw3+m+s5247yyWT1reC0krgJkR8VLZteRF0pnAPwAnR8SLZdczXJK2I+0MfxqpI7wP+K2IeLTUwnKi9J/5auCViPjjsusZSdm3609FxIfKrsWGx31n63PfWR2t0nd6n8nR5QpgV+CHkh6U9LWyCxqObIf4S4FbSTtYf6cqnWHmBOC3gVOzz+vB7JuomRXLfWd7cd9ZMG+ZNDMzM7Mh85ZJMzMzMxsyh0kzMzMzGzKHSTMzMzMbModJMzMzMxsyh0kzMzMzGzKHSTMzMzMbModJMzMzMxuy/wc/q5LpOK+DWgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(11,4))\n",
    "\n",
    "plt.subplot(1, 2, 1)\n",
    "# coordinate \n",
    "plt.plot([-7, 7], [0, 0], 'k-', linewidth=1)\n",
    "plt.plot([0, 0], [-2.2, 5], 'k-', linewidth=1)\n",
    "# Plot\n",
    "plt.plot(z, leaky_relu(z, 0.05), 'b-', linewidth=2)\n",
    "plt.plot([-5,5], [0,0], 'k-')\n",
    "plt.plot([0,0], [-0.5, 5], 'k-')\n",
    "plt.grid(True)\n",
    "plt.title(\"Leaky ReLU activation function\", fontsize=14)\n",
    "plt.axis([-5,5,-0.5, 2])\n",
    "\n",
    "plt.subplot(1, 2, 2)\n",
    "# coordinate \n",
    "plt.plot([-7, 7], [0, 0], 'k-', linewidth=1)\n",
    "plt.plot([0, 0], [-2.2, 5], 'k-', linewidth=1)\n",
    "# Plot\n",
    "plt.plot(z, derivative(leaky_relu, z), \"b-\", linewidth=2, label=\"Leaky ReLU\")\n",
    "plt.grid(True)\n",
    "plt.title(\"Derivative\", fontsize=14)\n",
    "plt.axis([-5,5,-0.05, 1.2])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to use it with Keras and TensorFlow 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.core.Dense at 0x120ed2690>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.keras.layers import LeakyReLU, Dense\n",
    "\n",
    "leaky_relu = LeakyReLU(alpha=0.01)\n",
    "Dense(10, activation=leaky_relu)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Parametric leaky ReLU (PReLU)\n",
    "\n",
    "please check out the [article](https://bindichen.medium.com/7-popular-activation-functions-you-should-know-in-deep-learning-and-how-to-use-them-with-keras-and-27b4d838dfe6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to use it with Keras and TensorFlow 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.core.Dense at 0x84a186d90>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.keras.layers import PReLU, Dense\n",
    "\n",
    "para_relu = PReLU()\n",
    "Dense(10, activation=para_relu)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Exponential Linear Unit (ELU)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def elu(z, alpha=1):\n",
    "    return np.where(z < 0, alpha * (np.exp(z) - 1), z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAokAAAEJCAYAAAD4j71kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxU1fnH8c8DBBRFULCogGLFDbWCpLiXVNS6t3WtCoobKtqKQnlV0V+1RXDf6gK4QcG1VGvdakUJWpElkVVWAUUQ2SRgWBKSnN8fZ0JCJkCWO3Nn7nzfr9d95c7cO/c8JzN58sxdzjXnHCIiIiIilTUIOwARERERST0qEkVEREQkjopEEREREYmjIlFERERE4qhIFBEREZE4KhJFREREJI6KRKk1M7vbzGYloZ32ZubMLDsJbf3azBaYWYmZjUh0ezuJJSfW71ZhxiEiqcXMeplZYZLacmZ2YTLaktSlIjEJzGxE7A+u6jSx0jpfm1n/7by+2j/W2HbfSWDc2yvSHgK6BdxWrpk9WeXpb4F9gWlBtrUdzwP/BA4AbklCe8B23/cJ+H6vSVYcIlJ/VXL9FjNbaWbjzOwmM8sKoInXgJ8GsJ2tdvB/ZF/g7SDbkvTTKOwAMshYoGeV54rDCKS+nHOFQMK/zTrnSoHvE92OmbUAWgIfOOeWJbq9nXHOFZOEfotIQpTn+obA3sApwD1ATzPr7pzbUJeNmlmWc24TsCmwSHfAOaccJNqTmERFzrnvq0w/JLJBM2tpZq+Y2VIz22RmX5rZVVXWMTPrFzvUWhRbd0hs8eLYzymxb8a5sddsPdxsZqebWbGZtayy3cFmNqMmccQO73YDbqr0Lbx9dXsyzewXZjbJzDab2Qoze9TMGldanmtmT8faXx37Jv+QmVX7WTezHGBt7OHHsfZyqjusU/UwcPk6ZtbdzGaZ2YbYXoMDq7zurFjMm8xsjZm9bWa7xH6fBwAPlve7unZiz51vZjNj79G3ZjbQzKzS8q/N7E4zG2Zm62O/6z9W12cRSajyXL/MOTfNOfcIkAMcAwwAMLPGZnZ/7O90o5lNMbNflW+gUg44y8wmm1kx8KvKecnMDomtc1Tlxs2sdyz3ZZlZQzN73swWx/LPAjMbUJ4Pzexu4Erg7Eq5Nye2bOsRLDObYGYPV2lnj9g2z69JnyQ9qUiMtl2AL4BzgCOAx4FhZta90jqDgbuAIbF1LsIf5gXoGvt5Bv7Qw/nVtPERsDr2OsAXnsBlwOgaxnEL8DnwYqydfSvFsJWZtQHeB6YCnYFrgEtjsVd2OVACnADcDPQFLqkmdvCHdo+IzV8Qa3vCdtatThPgduBq4HigBTC0UsxnAP8GPgS6AL8ExuP/9s4HlgJ/oaLfccysC/AP4A3gKOBPsTZvrrLqrcBM/D+j+4EHzOz4WvRFRBLAOTcL+A8+x4DPdd3wefJIYCTwtpkdXeWl9wN3AocBk6pscz4wBZ/vKrsceN05twWfZ5YBFwOHAwOBO4DyL+kPAa/j936W56Dq8t9o4HdVvmxfAGwG3q1lnySdOOc0JXgCRuCLlsIq0/2V1vka6L+d1zvgwu1s951axvIq8Fxsfnf8H/kN21m3fazt7CrP3w3MqvT4EeDTSo9PAkqBtjWJI/Y4F3hyR+0D9wILgAaV1ukFFAFNK23n8yrb+bByW9XE0irWTk6V7RZWWS8ntl6rSus44NBK61wei8dijz8DXt1B23HvezXtvAR8XM17sLTKdl6pss4C4M6wP/+aNGXKtKOcDNwHbAQOAsqA/ass/xfwdGy+PAdcUGWdbfIS8Afgm0r5Zv/Ytk/YQYz3AWN3FjOV/u/gT8cpBrpXWj4WGB6b32mfNKXnpD2JyfMJ0KnK9GAiG4wdahhoZjNihzkL8Xuv9o+t0hG/J+yjejY1GjjRzA6IPb4cGO+cW1rDOGrqcGCic66s0nP/AxoDHSo9N6PK674DflLLtmqqyDk3r0pbjYE9Y487U//f7+H4YrOy/wFtzGyPSs8ls98iUjuGL7yOic3Pjp2uUhjLiWfji63K8nayzVeB/YCTY48vBRY757buDTSzG8wsz8xWxdq5lVrmXufcGvye0Mtj29wPf1Sk/GhRbfokaUQXriTPRufcV3V87Y9A82qebwGs28Hr+gP98IdzZ+L3Xg4m4MLBOfeFmc0FLjOzh/CHngckOQ5XaX5LNctq+4WoDJ/0Kqvu6sSS7cSRrC9gQfdbRBKjI7AI/zfpgJ8T/zdb9aKUHV7k4pxbaWYf4ou3T2I/XypfbmaXAI/hc/AEYD1wE/DbOsQ/GnjWzPoAv8OfEvRpbFlt+iRpRP9A0sM8/PlsW5lZQ+Do2LLtOQl42zk3yjk3DVgIHFJp+Rz8odHu1b2YiquvG9YgxtH4BHUGsBswphZxlLe1s3bmAMdVOS/mpNhrF9YgxtpYBTStsqeuUx22M5Xt/36h5v0+scpzJ+EPN/9Yh5hEJInM7Eh8bhyDzwkG7OOc+6rKVJfRFUYDF8XOXT6Kir174PPEJOfck865L2I7Kqru2atJDgJ/bjX4c8svB152zpV/SQ26T5IiVCQmTxMz26fKtHeVdfYzs05Vplb4c/6uNj/W1iFm1gkYDuwV+7k984HuZnaSmR0GPAlsvfI2VmA8Dgwxs6vM7CAz62pmN8ZWWYn/FvgrM2ttZtXtzSz3Ev6b8l/xBeH6msYR8zXQ1fwVza2s+quRn8YfWnnazA43s7Px59c86ZzbuIPY6mIS/lv8EDPrYGYXAH3qsJ178Ql8kJl1NLMjzOxWM2saW/41cLKZtbHtD579MNDN/FXlh5jZ5fg9sw/UIR4RSazyXL+fmR1tZrfhz5XOBx5y/oKTl4ARZnahmf3UzLLNrH/5lcK19C/8UY7ngSmx7ZebDxxjZmea2cFmdhfxY9x+DRxpZofGcm+14zk65zbjx5K9E394eXSlZUH3SVKEisTkORVYXmWaWmWdW2PPVZ5+55x7BX812lX4c1T+A+wDnOx2PJbVIGAy/orgT/BFz0tV1rkdfwXdXfg9Vv8E2gI450rwJ0Zfiz+/7a3tNeSc+wZ/ntzRbPtNtqZxPIT/Rjsbvxcv7pyZ2DfSM/Hn+U0DXgBewV+tFyjnhye6HDgNf4i8N/53VNvtvIc/tHMm/v0cjz+Xp/y8yv8D2uH3hK7azja+wB/CvwCYhS+M78MX2yKSWspz/RL8+cjn4S80+4WrGCPxKvzVwA8Ac4F3gF/gL0KpldgX5DepPvcOw1+9/DL+Suj2+C+dlT2Lz/15+BxU9ahFZaNj7Ux1zs2usiywPknqsIq9xSIiIiIinvYkioiIiEgcFYkiIiIiEkdFooiIiIjEUZEoIiIiInFCGUy7VatWrn379klrb8OGDey2225Jay/Zotq/efPmUVhYSJcuXXa+cpoK+71bsgRWrYJdd4XDDwerOnx4PSW7f/n5+audc1WHlooM5c7gRLlvAPn5+cqdaSxVcmcoVzdnZ2e7vLyd3W0oOLm5ueTk5CStvWSLav9ycnIYP348Ub4CP8z37v334ayzICsLpkyBo48Ovo1k98/M8p1z2UlrMMmUO4MT5b4BmJlyZxpLldypw80iGWjNGrj6aj8/aFBiCkQREUlvKhJFMoxzcMMN8P33cPLJ0K9f2BGJiEgqUpEokmFeegnGjIHdd4eRI6FhTe7aKiIiGafeRaKZ7WJmk81supl9aWb3BBGYiARvyRK4+WY///jjcGDVO2hL0ih3ikiqC+Lq5iLgFOdcYezG4P8zs/edcxMD2LaIBKSsDHr1gnXr4Lzz4Kqrwo4o4yl3ikhKq3eR6PzlU4Wxh1mxKbqXVImkqSeegHHjYO+94dlngx/uRmpHuVNEUl0g4ySaWUMgH+gAPOWcm1TNOr2B3gCtW7cmNzc3iKZrpLCwMKntJVtU+1dQUAAQyb6VS9Z7t3hxUwYMyAYa8Ic/zGT27DXMnp3wZiP72QyKcmd4oty3clHuX9Tfv5Tpn3MusAloAYwDjtzRel26dHHJNG7cuKS2l2xR7V+3bt2c/4hGVzLeu6Ii5zp3dg6cu/rqhDe3jWR/NoE8F2BOS9ak3Jl8Ue6bc065M82lSu4M9Opm51xBLNGdEeR2RaTu/vIXmDrVX6Ty2GNhRyPVUe4UkVQUxNXNe5tZi9j8rsBpwNz6bldE6m/CBBgyxJ9/OHIkNGsWdkRSTrlTRFJdEOck7guMjJ1b0wB43Tn3TgDbFZF6KCyEK67wVzUPGOAHzpaUotwpIiktiKubZwCdA4hFRALUvz8sXAg/+5k/5CypRblTRFKd7rgiEkHvvgvDhkHjxjB6NDRpEnZEIiKSblQkikTM6tVwzTV+ftAgOOqocOMREZH0pCJRJEKcg+uvhxUr4Be/gNtuCzsiERFJVyoSRSJk1Ch44w1/FfPIkdCwYdgRiYhIulKRKBIR33wDv/+9n3/iCWjfPtRwREQkzalIFImAsjLo1QvWr4ff/AauvDLsiEREJN2pSBSJgMceg9xc+MlPYPhwP3i2iIhIfahIFElzs2bBHXf4+eeeg733DjceERGJBhWJImmsuBh69ICiIrj2Wjj33LAjEhGRqFCRKJLG7r4bpk+HAw+ERx4JOxoREYkSFYkiaeqzz+D++6FBAz/0TbNmYUckIiJRoiJRJA39+CNccYW/qnnAADjxxLAjEhGRqFGRKJKG+vWDRYvg6KPhnnvCjkZERKJIRaJImnn7bXj2WWjcGEaP9j9FRESCpiJRJI2sWuWvYgYYPBiOPDLceEREJLpUJIqkCeegd29YuRJycuDWW8OOSEREokxFokiaGDkS/vUv2GMPGDHCX9UsIiKSKPo3I5IGvv4a/vAHP//EE3DAAaGGIyIiGUBFokiKKy2FK6/0w96cf74f+kZERCTRVCSKpLhHH4VPPoHWrWHYMDALOyIREckEKhJFUtjMmTBwoJ9//nlo1SrceEREJHOoSBRJUUVF0KMHFBf7q5rPPjvsiEREJJOoSBRJUX/+M8yYAQcdBA8/HHY0IiKSaVQkiqSgTz+FBx7ww9z8/e+w++5hRyQiIplGRaJIivnxR381s3Pwpz/BCSeEHZGIiGQiFYkiKebWW2HxYujc2R9yFhERCYOKRJEU8tZb/irmJk1g1Cho3DjsiEREJFOpSBRJEStXwnXX+fkhQ+CII8KNR0REMpuKRJEU4Jwf5mbVKvjlL+GWW8KOSEREMp2KRJEU8OKL/lDzHnvAiBH+qmYREZEw6V+RSMgWL67Yc/jkk7D//uHGIyIiAioSRUJVWgpXXAGFhXDhhf4OKyIiIqmg3kWimbUzs3FmNtvMvjQznU0lUkOvv96O//0P9tkHhg4Fs7AjkmRR7hSRVNcogG2UAP2cc1+YWTMg38w+dM7NDmDbIpE1fTq88MKBALzwArRsGXJAkmzKnSKS0uq9J9E5t9w590Vs/kdgDtCmvtsVibKiIujZE0pKGnDDDXDmmWFHJMmm3CkiqS6IPYlbmVl7oDMwqZplvYHeAK1btyY3NzfIpneosLAwqe0lW1T7V1BQABDJvg0d+lNmztyffffdwHnn5ZObWxZ2SAkR1c9m0JQ7ky/KfSsX5f5F/f1Lmf455wKZgN2BfOD8na3bpUsXl0zjxo1LanvJFtX+devWzfmPaLSMH++cmXMNGjj31FP5YYeTUMn+bAJ5LqCclqxJuTMcUe6bcy6SubOyqL9/qZI7A7m62cyygH8CLznn3ghimyJRtH69v5rZObjjDujYcX3YIUmIlDtFJJUFcXWzAc8Dc5xzj9Q/JJHo6tsXvvkGjjkG/u//wo5GwqTcKSKpLog9iScCPYFTzGxabDorgO2KRMq//uXvrLLLLjBqFGRlhR2RhEy5U0RSWr0vXHHO/Q/Q6G4iO7BiBVx3nZ+/7z7o2DHceCR8yp0ikup0xxWRBHPOF4irV0P37vD734cdkYiIyM6pSBRJsOefh7ffhubN/eHmBvqrExGRNKB/VyIJtHChv1gF4KmnoF27cOMRERGpKRWJIglSWgpXXgkbNsBFF8Fll4UdkYiISM2pSBRJkAcfhM8+g333hWeeAdMlCiIikkZUJIokwLRpFeMgvvACtGwZbjwiIiK1pSJRJGCbN0OPHrBlC/TpA2ecEXZEIiIitaciUSRgd94JX34JBx8MDzwQdjQiIiJ1oyJRJEC5ufDII9Cwob+rym67hR2RiIhI3ahIFAnIunX+ambnYOBAOPbYsCMSERGpOxWJIgG55RZYsgSys/0hZxERkXSmIlEkAG+8ASNHwi67+MPMWVlhRyQiIlI/KhJF6un776F3bz//wANw2GHhxiMiIhIEFYki9eAcXHstrFkDp54KN90UdkQiIiLBUJEoUg/PPgvvvgstWsCLL0ID/UWJiEhE6F+aSB0tXAi33ebnn34a2rYNNx4REZEgqUgUqYOSEujZEzZsgEsugUsvDTsiERGRYKlIFKmDBx6Azz+H/fbzexFFRESiRkWiSC1NnQp//rOff/FF2GuvcOMRERFJBBWJIrWweTP06OEPN990E5x+etgRiYiIJIaKRJFauOMOmD0bDjnEH3IWERGJKhWJIjX08cfw6KPQsCGMHg1Nm4YdkYiISOKoSBSpgYIC6NXLz995J/z856GGIyIiknAqEkVq4A9/gG+/9cXhwIFhRyMiIpJ4KhJFdmLMGBg1Cnbd1f/Mygo7IhERkcRTkSiyA8uXw/XX+/kHH4RDDw03HhERkWRRkSiyHc7BNdfADz/4oW769Ak7IhERkeRRkSiyHcOGwfvvw557wgsvgFnYEYmIiCSPikSRaixYAP36+flnnoE2bcKNR0REJNlUJIpUUVICV1wBGzfCpZfCJZeEHZGIiEjyqUgUqeK++2DiRL/38Kmnwo5GREQkHIEUiWb2gpmtNLNZQWxPJCz5+XDPPX7+xRf9+YgiiaC8KSKprlFA2xkBPAn8PaDtiSTdpk3Qo4c/3Pz738Npp4UdkUTcCJQ3JYEKC6GsLOwoEmPDhoasXx92FImTKv0LpEh0zn1iZu2D2JZIWG6/HebOhcMO84ecRRJJeVMSZckSgLE0axZ2JIl0ctgBJFhq9C+oPYk7ZWa9gd4ArVu3Jjc3N1lNU1hYmNT2ki2q/SsoKABISt/y81vw+OOdaNiwjL59pzJ58o8JbxOi+96Vi3r/kkG5MzGi2rfPP9+Le+/tCHSnUaMyGjeO5q5E5xwW4XHJkt2/jRt3EEgQE9AemFWTdbt06eKSady4cUltL9mi2r9u3bo5/xFNrLVrnWvb1jlw7p57Et7cNqL63pVLdv+APBdQTkvGVJu86ZQ7AxXFvs2d69xuu/lcBm+6lSvDjihxovj+VZYquVNXN0vGu/lmWLoUunaFO+4IOxoRkdrbtAkuvhg2bIDf/Q7gt+y9d9hRSbpTkSgZ7fXX4aWXYNddYdQoaJS0EzBERILTty/MmAEdOvi7RYkEIaghcF4BPgcONbOlZnZNENsVSaTvvoMbbvDzDz0EhxwSbjySWZQ3JSjPPgvDh0OTJvDaa7DHHmFHJFER1NXNlwaxHZFkcQ6uvhrWroVf/QpuvDHsiCTTKG9KECZO9KfMAAwdCsccE248Ei063CwZ6Zln4IMPYK+94IUXIMIXyYlIRC1cCOedB8XFvlDs1SvsiCRqVCRKxpk/H/r39/PPPAP77RduPCIitbVqFZxxhv95+unwyCNhRyRRpCJRMkpJCfTs6a8EvPxyfzWgiEg62bgRzj0XvvoKOnWCMWMgKyvsqCSKVCRKRhk8GCZPhrZt4cknw45GRKR2Skr8EDeTJsEBB8B77xHxO6tImFQkSsbIy4O//MXPjxgBLVqEGo6ISK2UlsI118Dbb8Oee8L778O++4YdlUSZRoWTjLBxoz/MXFoKt9wC3buHHZGISM2VlMCVV8LLL0PTpvDvf8Phh4cdlUSdikTJCH/6E8yd65PqkCFhRyMiUnNbtkCPHn7w/91283sQTzop7KgkE6hIlMj78EP429/83VRGjfJ3VxERSQfFxXDppfDGG/7cw//8B044IeyoJFOoSJRIW7sWrrrKz//5z9ClS7jxiIjUVEEBXHQRjB0LzZv7sV2PPTbsqCSTqEiUSLvpJli2DI47zh9yFhFJB4sWwTnnwJw58JOfwLvvQnZ22FFJptHVzRJZr74Kr7ziT/L++9/94WYRkVQ3YYL/YjtnDhxxhB/uRgWihEFFokTSsmUV92N++GE4+OBw4xER2Rnn4Lnn4JRTKu6k8tln0L592JFJplKRKJHjHFx9tT+f58wz4frrw45IRGTH1q+Hyy6D666DoiLo08cfYm7ePOzIJJPpAJxEztNPw3//Cy1bwvPPg1nYEYmIbF9+PlxyCSxc6Ie4GTrUD3kjEjbtSZRImTcP/vhHPz90qO5GICKpq7gYBg3yQ9osXAhHH+0LRhWIkiq0J1EiY8sWf1eVTZv8zwsvDDsiEZHqTZ4M114LM2f6xzfeCI88ArvsEm5cIpVpT6JExr33wpQp0K6dHzxbRCTVrF8Pt90Gxx/vC8Sf/tSPg/j00yoQJfWoSJRImDzZH7YBGDFCJ3uLSGopKYFnnoEOHeDRR/1z/fv7QlH3kpdUpcPNkvY2bvSHl0tL4dZb/fARIiKpwDl/lfKAAX7cQ4ATT4THH9cdoCT1aU+ipL0BA2D+fOjYEQYPDjsaEREoK4O33oKuXeHcc32BeNBBMGYMfPqpCkRJD9qTKGntgw/gqacgKwtGj9Y5PSISri1bfCE4ZEjFRSk/+Ym/LehNN0HjxuHGJ1IbKhIlbf3wA1x1lZ+/+27o3DnUcEQkg33/PQwf7ofeWr7cP9emjT/Sce21/vagIulGRaKkrT59fDI+/nifiEVEkqmkBD78EEaOhDfe8HsRAQ4/HG65BXr1giZNQg1RpF5UJEpaeuUVeO01f3eCv/8dGumTLCJJ4Bx88QW89BK8/DKsWOGfb9AAfvtbuPlm+OUvdacniQb9a5W0s3Sp34sIfvDZDh3CjUdEoq20FCZM8HsL33wTvvmmYtkhh8AVV/gRFvbfP7wYRRJBRaKklbIyfx5iQQGcfTZcd13YEYlI1DgHixb5Qa4//BA+/hjWrq1Yvs8+cMEFvjj8+c+111CiS0WipJWnnvKJu2VLeO45JWcRqT/n4KuvYNIk+OQTXxh+/fW26xx0kD+cfP75cOyx/vCySNSpSJS0MWdOxQUqw4f7b/MiIrW1ciVMnrwnn3wCEyf6OzatWbPtOnvu6e+EctppcOqp/vZ5IplGRaKkhS1b/Dk/mzfDlVf6b/MiIjuybh3Mm+fHK5w1y/+cOdMXiXD0Nuu2bu33EB5/vC8KO3eGhg1DCVskZahIlLTw179Cfr4/Mfzxx8OORkRSQXExfPcdfPstLFkCCxf6w8YLFvifq1dX/7pmzWD//ddx+unNOfZYOO44n1t0+orItlQkSsqbNMnfbs/Mj0fWvHnYEYlIIm3YAKtW+T1+q1ZVzC9d6qdvv/U/V6zw5xNuz667+tEPjjgCjjqqYjrgABg/fio5OTlJ65NIOgqkSDSzM4DHgYbAc865+4LYrgg0oGdPPwRFv36gnC5RErXc6RwUFfkib+PGip8//uhHJFi3zv+sPF/5udWrfUG4cWPN2mvQAPbbD9q2hXbt/MUlHTrAwQf7n/vuqwtMROqj3kWimTUEngJOA5YCU8zs38652fXdtggcxIIFcOSRMGhQ2LGIBKcuuXPLFli2zH9pqulUUrLt4y1b/GHaoiL/c3tTUREsWtSBV1/d9rnywq9yEVh5fkd79mqqSRPYe29/z+PKP9u0qSgI27b1RaAG0hdJnCD+vLoCXznnFgGY2avAr4HtJrp58+YldTd/QUEBLVq0SFp7yRbV/k2ZMg0AsxyaNIEzzgg5oASI6ntXLur9q6da584ZM3xxlDy1b6xxY38npKZNK6ZmzaBFC3+qSIsW1c83bw6tWvmCcPfddX6gSCoIokhsA3xb6fFS4NiqK5lZb6B3+ePx48cH0LREV2NgF8Bw7ivy85eFHZBI0OqQO38GfAeU1mIqqfK4uJqpaDvPV7fOBmBjbKo6v4ni4lKKi7cdfFrCYaq0pZ6StqPeOTccGA6QnZ3t8vLyktU0ubm5kT5BOWr9Ky31Y5ONG5cD/EBZ2YzI7lWI2ntXVbL7F8V/ivG5c7+ktR3lz2eU+wb+b8EFcew/RUX9/UuV3BnEKb3LgHaVHreNPSdSJw8+COPGQVYWwNzIFoiS8ZQ7RSSlBVEkTgEONrMDzawx8Dvg3wFsVzLQxIlw111+/rDDALaEGY5IIil3ikhKq3eR6JwrAW4GPgDmAK87576s73Yl8yxb5u+kUlICt90Ge+0VdkQiiaPcKSKpLpBzEp1z7wHvBbEtyUybNsFvfgPLl/uxEO+7z5+XKBJlyp0ikso0zKiEzjm4+mrIy4Of/hTGjCk/H1FERETCoiJRQjdkCLz6qh9L7d//hpYtw45IREREVCRKqF5/HQYO9APnvvyyv8eqiIiIhE9FooTmnXfg8sv9/P33wznnhBuPiIiIVFCRKKH46CO48EJ/JfOAAdC/f9gRiYiISGUqEiXpJkyAX/8aioqgTx9/JbMGzBYREUktKhIlqSZNgrPOgg0b4Ior4G9/U4EoIiKSilQkStJ88AGccgqsWwcXXADPPw8N9AkUERFJSfoXLUnxyiv+wpSNG+HKK/3jRoEM5S4iIiKJoCJREu6JJ+Cyy/xFKv37w4svarBsERGRVKciURKmuBhuugluucU/fuABePBBnYMoIiKSDnTATxJi+XK46CL47DNo3BiGD/eHmUVERCQ9qEiUwE2Y4MdAXL4c2rSBN96Arl3DjkpERERqQ4ebJTAlJXDvvdCtmy8Qf/ELyM9XgSgiIpKOtCdRArFggR/3cOJE//jWW/2t9nSBioiISHpSkSj1UlYGQ4fCH//oh7dp08ZfvXzaaWFHJiIiIvWhIlHqLC/PX708ebJ/fPnl/g4qe+4ZblwiIiJSfzonUWptzRq44QZ/ruHkybDffvCPf8Do0SoQRUREokJ7EqXGNmzwA2M/+CCsXevvmHLrrXDXXdCsWdjRiYiISL4RkOEAAAy4SURBVJBUJMpOFRXBsGEweDCsWOGf697dF4wdO4Ybm4iIiCSGikTZroICXxw+8QR8951/rmtXP8xN9+66c4qIiEiUqUiUOIsX+8LwueegsNA/d9RRMGgQnHuuikMREZFMoCJRAH+f5bfegmefhbFjwTn/fPfu0L8//OpXKg5FREQyiYrEDOYcTJoEr74KL78Mq1b555s0gYsvhttug06dwo1RREREwqEiMcOUlcGUKf5+yq+9Bt98U7HsqKPguuugRw8NZSMiIpLpVCRmgLVr4eOP4Z134L33YOXKimVt2vi9hpdeCtnZOqQsIiIinorECPrhB/j0U8jNhfHjYdq0inMMAdq39xegXHwxnHACNNCQ6iIiIlKFisQ0V1wMCxbszoIF/jZ5kybBjBnbFoWNG8Oxx8LZZ8M55/ixDbXHUERERHZERWKacM4PZD1nDsyeDbNmQX4+TJ8OxcXZ26zbuDEcdxzk5PjpuONg111DCVtERETSlIrEFOKcP19w8eJtp/LCcO3a6l/Xrt1GTjqpKdnZ0KWLH/BaRaGIiIjUh4rEJCkp8XsCv/8eli/3U/n8N9/4YvDrr2HTpu1vo3lzf6i4fOrcGY45BqZOnUxOTk6yuiIiIiIZoF5FopldBNwNHA50dc7lBRFUqnIONm/2dyEpLIR16/xFIpWnNWvin1u50o9BWPk8we3Zc0848MCKqX17OOwwXxTus4/OJRSJgkzLnSKSnuq7J3EWcD4wLIBYdsg5P8bfli1+r1zVqfLzW7ZAUZGfNm+GKVNasnq1ny9/rvJU/tymTRUF4I8/VsxXflxWVrf4zaB1a1/o7buvn8rn27WrKAibNw/01yYiqSlpuVNEpK7qVSQ65+YAWC13b02dOo/dd8/ZumfNOWja9GKaNu3Dli0bWbPmLMrK/PPlE/SKTauBC6vZ6o3AJcC3QM9qlvcDzgXmAddXs/xO4FRgGtC3muWDgRPIypqAc3fQsCE0auSnrCw4+eTHOPjgTqxZM5ZPPx1EVlbFsqwsePbZYRxxxKG8/fbbPPzwwyxaBIsWVWx91KhRNG/ejtdee41nnnkmrvUxY8bQqlUrRowYwYgRI7ZZVlBQwIQJE2jatClPP/00r7/+etzrc3NzAXjooYd45513tlm266678v777wPw17/+lY8++mib5S1btuSf//wnALfffjuff/75Nsvbtm3L6NGjAejbty/Tpk3bZvkhhxzC8OHDAejduzfz58/fZnmnTp147LHHAOjRowdLly4F2Lqd22+/nSFDhgBwwQUXsGbNmm1e3717d+666y4AzjzzTDZVOWZ/zjnn0L9/f4BqD8tffPHF9OnTh40bN3LWWWfFLe/Vqxe9evVi9erVXHhh/Gfvxhtv5JJLLuHbb7+lZ8/4z16/fv0499xzmTdvHtdfv+1nr6CggIceeohTTz2VadOm0bdv/Gdv8ODBnHDCCUyYMIE77rgjbvljjz1Gp06dGDt2LIMGDYpbPmzYMA49tOKzV9WoUaNo165unz2A9957b7ufvYKCgq3vYzI+e+mirrlTRCSZknZOopn1Bnr7R7uzYcO2yzdt8odqd6RBA0eDBmWUljrM2DqBY7fditltt03ARtasKaVBAzBzsTEAyzjggHXss88KiotXMnt20TbLzRxduy7loIPmsX79Qj76aEOsLUfDhr7dXr2m07nzFubOnclzzxXExXbWWXl06FBAfv508vIqlm/Z4qf8/EmsWrWcmTNnUlAQ//rPP/+chQsX8uWXX1a7/LPPPqN58+bMnTs3bnlpaSmffPIJu+yyC/Pnz6/29eVF4sKFC+OWb9q0aevyxYsXxy0vKyvbunzJkiVxy7OysrYuX7p0adzy7777buvy7777Lm750qVLty5fsWLF1uUlJSVb2yxfvmrVKtavX7/N6xcvXrx1+Q8//EBRUdE2yxcuXLh1eXW/m/nz55Obm8vmzZurXT537lxyc3NZt25dtcu//PJLcnNzWblyZbXLZ86cSbNmzar93ZWWljJ9+nQaNWrEV199Ve3rv/jiC4qLi5k1a1a1y/Py8igoKGD69OnVLp80aRLLlyfmswfs8LNXWlqa1M9eFFXOna1bt97a32QoLCxManvJFOW+lYty/6L+/qVK/8zt5EQ5MxsL7FPNooHOubdi6+QC/Wt6Xk3HjtnulVfyMPMDOZfvbSvfM1d5Kn/eF2u17F1Mbm5upC/siGr/cnJyGD9+PDv7jKazqL535ZLdPzPLd85l73zNxEtE7szOznZ5eck7fTHKn88o9w38XmrlzvSVKrlzp3sSnXOnBh1M06Zw9NFBb1VEJHUkIneKiCSTbsgmIiIiInHqVSSa2W/NbClwPPCumX0QTFgiItGl3Cki6aC+Vze/CbwZUCwiIhlBuVNE0oEON4uIiIhIHBWJIiIiIhJHRaKIiIiIxFGRKCIiIiJxVCSKiIiISBwViSIiIiISR0WiiIiIiMRRkSgiIiIicVQkioiIiEgcFYkiIiIiEkdFooiIiIjEUZEoIiIiInFUJIqIiIhIHBWJIiIiIhJHRaKIiIiIxFGRKCIiIiJxVCSKiIiISBwViSIiIiISR0WiiIiIiMRRkSgiIiIicVQkioiIiEgcFYkiIiIiEkdFooiIiIjEUZEoIiIiInFUJIqIiIhIHBWJIiIiIhJHRaKIiIiIxFGRKCIiIiJxVCSKiIiISBwViSIiIiISR0WiiIiIiMRRkSgiIiIicepVJJrZg2Y218xmmNmbZtYiqMBERKJKuVNE0kF99yR+CBzpnPsZMB+4vf4hiYhEnnKniKS8ehWJzrn/OudKYg8nAm3rH5KISLQpd4pIOmgU4LauBl7b3kIz6w30BmjdujW5ubkBNr1jhYWFSW0v2aLav4KCAoBI9q1cVN+7clHvX0CUO0MQ5b6Vi3L/ov7+pUr/zDm34xXMxgL7VLNooHPurdg6A4Fs4Hy3sw0C2dnZLi8vrw7h1k1ubi45OTlJay/Zotq/nJwcxo8fTw0+Umkrqu9duWT3z8zynXPZSWtwB5Q7U1uU+wZgZsqdaSxVcudO9yQ6507dyYZ7AecA3WuS5EREMoFyp4iku3odbjazM4ABQDfn3MZgQhIRiTblThFJB/W9uvlJoBnwoZlNM7OhAcQkIhJ1yp0ikvLqtSfROdchqEBERDKFcqeIpAPdcUVERERE4qhIFBEREZE4Ox0CJyGNmq0Cvklik62A1UlsL9mi3L8o9w3Uv6Ad4JzbO4ntJZVyZ6Ci3DdQ/9JdSuTOUIrEZDOzvFQZOy0Roty/KPcN1D9JbVF+/6LcN1D/0l2q9E+Hm0VEREQkjopEEREREYmTKUXi8LADSLAo9y/KfQP1T1JblN+/KPcN1L90lxL9y4hzEkVERESkdjJlT6KIiIiI1IKKRBERERGJk3FFopn1MzNnZq3CjiUoZvagmc01sxlm9qaZtQg7piCY2RlmNs/MvjKzP4UdT5DMrJ2ZjTOz2Wb2pZndEnZMQTOzhmY21czeCTsWqZ8o5k1Q7kxHyp3JlVFFopm1A04HloQdS8A+BI50zv0MmA/cHnI89WZmDYGngDOBjsClZtYx3KgCVQL0c851BI4DbopY/wBuAeaEHYTUT4TzJih3piPlziTKqCIReBQYAETqah3n3H+dcyWxhxOBtmHGE5CuwFfOuUXOuWLgVeDXIccUGOfccufcF7H5H/EJoU24UQXHzNoCZwPPhR2L1Fsk8yYod6Yj5c7kypgi0cx+DSxzzk0PO5YEuxp4P+wgAtAG+LbS46VEKBFUZmbtgc7ApHAjCdRj+MKiLOxApO4yKG+CcmfaUe5MvEZhBxAkMxsL7FPNooHAHfhDJmlpR31zzr0VW2cgflf8S8mMTerOzHYH/gn0dc6tDzueIJjZOcBK51y+meWEHY/sWJTzJih3RpVyZ3JEqkh0zp1a3fNmdhRwIDDdzMAfUvjCzLo6575PYoh1tr2+lTOzXsA5QHcXjcEvlwHtKj1uG3suMswsC5/kXnLOvRF2PAE6ETjPzM4CdgH2MLPRzrkeIccl1Yhy3gTlTpQ700nK5c6MHEzbzL4Gsp1zq8OOJQhmdgbwCNDNObcq7HiCYGaN8CeSd8cnuCnAZc65L0MNLCDm/+uOBH5wzvUNO55EiX0b7u+cOyfsWKR+opY3QbkzHSl3JlfGnJMYcU8CzYAPzWyamQ0NO6D6ip1MfjPwAf7E5NejkuRiTgR6AqfE3rNpsW+PIpI8yp3pR7kziTJyT6KIiIiI7Jj2JIqIiIhIHBWJIiIiIhJHRaKIiIiIxFGRKCIiIiJxVCSKiIiISBwViSIiIiISR0WiiIiIiMT5f3gCCtz09wqGAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(11,4))\n",
    "\n",
    "plt.subplot(1,2,1)\n",
    "# coordinate \n",
    "plt.plot([-7, 7], [0, 0], 'k-', linewidth=1)\n",
    "plt.plot([0, 0], [-2.2, 5], 'k-', linewidth=1)\n",
    "# Plot\n",
    "plt.plot(z, elu(z), 'b-', linewidth=2)\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([-5, 5], [-1, -1], 'k--')\n",
    "plt.plot([0, 0], [-2.2, 3.2], 'k-')\n",
    "plt.grid(True)\n",
    "plt.title(\"ELU activation function\", fontsize=14)\n",
    "plt.axis([-5, 5, -2.2, 3.2])\n",
    "\n",
    "plt.subplot(1,2,2)\n",
    "# coordinate \n",
    "plt.plot([-7, 7], [0, 0], 'k-', linewidth=1)\n",
    "plt.plot([0, 0], [-2.2, 5], 'k-', linewidth=1)\n",
    "# Plot\n",
    "plt.plot(z, derivative(elu, z), \"b-\", linewidth=2, label=\"ELU\")\n",
    "plt.grid(True)\n",
    "plt.title(\"Derivative\", fontsize=14)\n",
    "plt.axis([-5, 5, -2.2, 3.2])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to use it with Keras and TensorFlow 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.core.Dense at 0x84a172e90>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "\n",
    "Dense(10, activation='elu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1.        , -0.63212055,  0.        ,  1.2       ], dtype=float32)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.activations import elu\n",
    "\n",
    "z = tf.constant([-20, -1, 0, 1.2], dtype=tf.float32)\n",
    "output = elu(z, alpha=1)\n",
    "output.numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. Scaled Exponential Linear Unit (SELU)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.special import erfc\n",
    "\n",
    "# alpha and scale to self normalize with mean 0 and standard deviation 1\n",
    "# (see equation 14 in the paper):\n",
    "alpha_0_1 = -np.sqrt(2 / np.pi) / (erfc(1/np.sqrt(2)) * np.exp(1/2) - 1)\n",
    "scale_0_1 = (1 - erfc(1 / np.sqrt(2)) * np.sqrt(np.e)) * np.sqrt(2 * np.pi) * (2 * erfc(np.sqrt(2))*np.e**2 + np.pi*erfc(1/np.sqrt(2))**2*np.e - 2*(2+np.pi)*erfc(1/np.sqrt(2))*np.sqrt(np.e)+np.pi+2)**(-1/2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.6732632423543778"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alpha_0_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0507009873554805"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scale_0_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def selu(z, scale=scale_0_1, alpha=alpha_0_1):\n",
    "    return scale * elu(z, alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAokAAAEJCAYAAAD4j71kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZgU1dXH8e9hk1UQwVEERANuQaMOicQlAZdoDMQtRn1d4zIaAYmgglskuIvgAqIQQDQaUYMEUbOoYVwSMTC4gogEERBREFlm2Jn7/nG7nWZmGGapruqu/n2ep56uriqqzqVnzpyuW3XLnHOIiIiIiKSqF3UAIiIiIpJ5VCSKiIiISAUqEkVERESkAhWJIiIiIlKBikQRERERqUBFooiIiIhUoCJR6szMJprZiyEcp4eZOTNrE8KxCsxssZmVmtmQdB9vJ7FcbGbFUcYgItELMxckcu2vwjiWZC4ViQEzs7ZmNtrMFpnZJjP7ysxeM7MTU7YpTPwClp8mpWxT6S+omXVKrOtWybpCMxuVxrbtqEjrD5wf8LEWmdm15Rb/B9gL+CbIY1Vy7N2Ah4FhwN7Afek8XrljV/a5PwPsF1YMIlI7iS/MyXy+xcy+NrPpZtbHzBoGcIjAc0EVX/L3AqYFeSzJPg2iDiCGJgNNgUuBBcAewE+B3ctt9xhwY7llG9IeXRo459aEdJzNwPIQDrUP/nfjRefclyEcr0rOuQ1k6c+GSA56FbgAqA+0BY4D/gBcYGbHO+dKarNTM2sYZi5wzoWRayXD6UxigMysFXAsMNg595pz7nPn3Ezn3H3OuUnlNl/vnFtebkprsWVm3zOzqWa23MxKzGy2mfUqt00jM7vTzD5PnAldaGZXm1knYHpisxWJb8oTE//mu2+iiW7ar8ysfrn9/tnMXqhOHGZWiC/UhiW/lSeWVziTaWZnmNmHiViXmNlNZmYp6xeZ2c1mNsbM1prZUjO7ror/o4uBdxNvFyaO18nMhpjZR+W3Te36SW5jZueY2f/MbJ2Z/bX8mVczuygl5q/M7PFkrIlNnkscd1Flx0ksu8LMFpjZ5sTr5eXWu8Rn8Vzi/3ihmQV6tldEKrUpkc+/cM6955wbAfQAjgCuh+/y7D2JfLTezGaa2UnJHaTkulPM7L9mthk4KTUXmNn+iW0OST144vd+pZk1NLP6ZjbezD4zsw1m9qmZXW9m9RLbDgEuAn6Rcga0R2Ldd70aZvYfMxte7ji7JvZ5RnXaJNlJRWKwihPTL82scdTBVKI58DfgROAH+LOez5vZgSnbPA5cCAwADsKfEV0NLAHOTGzzfXxXRP9KjvEc0DJxDADMrDlwKvBkNeM4A1gKDE0cZ6/KGmNm+YnjPQ8cAgwGbgD6ltv0GuBDfJK+B7jXzH5c2T7x3TknJ+Z/lDj2kh1sW5lOwNnA6cDPgMOBO1JivgIYgz+TfChwCpAsPn+YeL08cdzk++2Y2enAKOABoCvwIDDazHqX2/T3wFT8//EzwAQz61iDtohIAJxzHwF/pyyHPobvYfo//O/w48A0M/tBuX96D3AzcCDwTrl9zgdmAueV+zfnAc8657bg/8Z/Afwan89vwvdg/Sax7X3As/izn8lc+59KmvAkcE6yuEw4E9gIvFTDNkk2cc5pCnDC/+Kswv/yvI3/JTyy3DaFwGbKisrkdFXKNg74VSX775RY162SdYXAqBrGOwO4OTHfJbHvk3ewbY/E+jbllk/Ed80m3z8P/Cnl/fnAGqBxdeJIvF8EXFvV8YGngH+V22YIsLTcfp4ut82nqceqJJZuieN0Krffj8ptdzFQXG6bjUDLlGU3AQtS3i8F7q7i2BU+90qO829gQiWfwVvl9nNXyvsGwHrg/Kh/RzRpiutUPheWW3d34nfwe0Ap0LHc+r8CoxPzyVx3ZrltyueCq4HPAUu875jY91FVxHg38OrOYk7NRfjLpTYDx6esfxUYm5jfaZs0ZeekM4kBc85NBtoBvfFny44CZphZ+esPnwEOKzc9lc7YzKyZmd1rZnPN7NtEt0U3fGIBf9arlLJu5dp6EjjNzJom3p8HTHbObaxmHNV1EL5gSvUWsLeZ7Zqy7INy2yzDXyuaDp+77S8b+O5YZrYH/kaY1+p4jB21++Byy75rt3NuK7CC9LVbRKpm+MLriMT8XDMrTk7AL/DFVqpZO9nnJPzfm2MT788FPnPOfXc20MyuNLNZZrYicZxrqGGudc59gz8Tel5in+2AnpT1DtWkTZJFdONKGiSKoVcS01AzGwcMMbP7nL/5AmCNc25BLXa/NvHaspJ1rfBn7HbkPnxX6rX4s2nrgSeARrWIoyovAVuBU83sNeAEIPXalDDicCnzWypZV9MvSKX4JJiqsrsVgzhWbbly76OMRUS2dzCwEP876PCXk5T/HS1/U0qVN7k45742s1fwxdsbidfvTjaY2dn4y1KuxXcjrwX64C+HqakngT+a2VXAOfjLcN5MrKtJmySL6A9GOObiC/I6X6fonFsFrATyU5cnzpx1Bj6p4p8fAzzhnJvsnPsA3/WZ+i3vPfzPRM8d/PtkgVt/B+uTMW7CXyt4Hv76vOX4rvDqxpE8VpXHAT4Gji637Bh8d/O6nfzbmloB5JlZaqF4WE124Jz7Gn990PFVbLaF2rd7bk3iEZFwmFlX/Bfjv+BvjDNgT+fcgnLTF7XY/ZPAWYlrtA+h7Owe+LzwjnNulHNuduLERG1yLcALidde+Nz+Z+dc8otp0G2SDKEziQEys93xxdEEfFffOnw36vXAa865tSmbNzWzPcvtYnOiCEzqZGblC5GFwAhgsJktw1/3uDtwC76Qea6KEOcDp5vZVHwxcisphatzbr6ZPQuMM7P+wGygPf7avD/hr31x+DvhpgEbnHM7Gtj1SXy36r74awJLqxtHwiLgWDN7En+34MpKjjEcmJm4Q+/P+G+xA6k4tFAQCoHWwI3mx7PsAdRmoNk7gPvN7Cv8Gdem+Ot8kncOLgKON7PX8e3+tpJ9DMPfAV0E/BP/x+c8/A0/IhKtXRK5vR5+CJzj8TmpCLjPOVdiZk8BE81sID7PtsbnlIXOuedreLy/4m+GGw/MdP6GlqT5wMVm9nP8kGzn4G8uSc0ri4Cfm9kB+DFo1zh/08t2nHMbzWwy/kaaH+CH+Umumx9wmyRTRH1RZJwmYBfgTvwdZ9/iu1E/xRd1rVO2K8QXW+Wn8jceVDb1wn/r64cvRIvxZ+ImkXKjxQ7i2wd/sXFJ4t9cC7wITCzXhnvxZ7w2Af8D+qasvwX4Et/9OjGxbCLlLnzGf6tclIj50FrE0R14H38jiEss60G5G2fwhdGH+G/DS/A3iljK+kVUvAGmkCpu8KGSG1cSy6/AF8olif/v/lS8caXKm1sSyy7Fn/VLjvs4IWVd78TPzBZgURX7uBKf9LckXi8vt76yG2Aq/F9o0qQpuCmRC5O5eiu+16cQP+JCo5TtGibyxcKUPPACkJ9YXyHXJZZXyAWJ5U8ktr+63PJG+OLxW/woFePxox4sStmmLf7L5rrEPnoklleWQ45LLJ9dSQxVtklTdk7JO6JERERERL6jaxJFREREpAIViSIiIiJSgYpEEREREalARaKIiIiIVBDJEDht2rRxnTp1Cu14JSUlNGvWLLTjhS3O7SsqKiI/P3/nG2apMD8752DuXNi4EfbcE/beO/3HDPtns6ioaKVzrm1oBwyZcmdw4tw2UO7MdpmSOyMpEjt16sSsWTt72lBwCgsL6dGjR2jHC1uc22dmof6shC3Mz27wYJg9Gw46yL82rvPQ7jsX9s+mmX0e2sEioNwZnDi3DZQ7s12m5E51N4vkgJkzYdgwqFcPJkwIp0AUEZHspiJRJOY2bYLf/AZKS+Gaa6B796gjEhGRbKAiUSTmbr8d5syBLl3gttuijkZERLJFnYtEM2tsZv81s/fNbI6Z/SGIwESk7t59F+66C8x8N3OTJlFHJEnKnSKS6YK4cWUTcJxzrtjMGgJvmdnfnHMzAti3iNTSli1wySWwbRv06wfHHBN1RFKOcqeIZLQ6F4nOP/y5OPG2YWLSA6FFInb33fDee7Dvvv5somQW5U4RyXSBDIFjZvWBIqAz8LBz7p1KtikACgDy8vIoLCwM4tDVUlxcHOrxwhb39sW5ben67BYubMbQoflAPfr1e4+ZM1cHfozqiPvPZl0pd0Ynzm1LinP74v75ZUz7nHOBTUArYDrQtart8vPzXZimT58e6vHCFuf2+R/R+ErHZ7dli3P5+c6Bc1deGfjuayTsn01glgswp4U1KXeGL85tc065M9tlSu4M9O5m59zqRKI7Ocj9ikj1DR8ORUXQsSPce2/U0Uh1KHeKSCYK4u7mtmbWKjHfBDgRmFfX/YpIzc2bB7fe6ufHjoUWLaKNR3ZMuVNEMl0Q1yTuBTyeuLamHvCsc+7FAPYrIjWwbZsfNHvTJn9X80knRR2R7IRyp4hktCDubv4AODyAWESkDh58EGbMgHbtfJezZDblThHJdHriikgMfPop3HSTnx87Flq1ijYeERHJfioSRbJcaSlceils3AgXXAC/+EXUEYmISByoSBTJcqNHw5tvQl4ePPBA1NGIiEhcqEgUyWKffQaDB/v5Rx6B1q2jjUdEROJDRaJIlnIOLrsMSkrg7LPh9NOjjkhEROJERaJIlho7Fv71L2jbFkaOjDoaERGJGxWJIllo8WK47jo/P2qULxRFRESCpCJRJMs4BwUFsG4dnHEGnHVW1BGJiEgcqUgUyTITJ8I//uFvUnn4YTCLOiIREYkjFYkiWeSLL+Caa/z8gw/CnntGG4+IiMSXikSRLOEcXHklrFkDvXrBeedFHZGIiMSZikSRLPHUU/Dii9CyJTz6qLqZRUQkvVQkimSB5cvh6qv9/P33w957RxuPiIjEn4pEkQznHPTpA99+CyedBBdfHHVEIiKSC1QkimS4556D55+HFi38ANrqZhYRkTCoSBTJYCtW+LOIAMOGQceO0cYjIiK5Q0WiSAbr1w9WroTjjvMDaIuIiIRFRaJIhpoyBZ55Bpo2hXHj1M0sIiLhUpEokoFWrYLf/tbP33037LtvtPGIiEjuUZEokoF+9zv46is49tiyaxJFRETCpCJRJMO89BL86U/QuDGMHw/19FsqIiIR0J8fkQyyenXZDSp33AFdukQbj4iI5C4ViSIZZOBAWLYMuneH/v2jjkZERHKZikSRDPHPf8KECbDLLv61fv2oIxIRkVymIlEkA6xbB5df7ueHDIGDDoo0HBERERWJIpng+uth8WLo1g2uvTbqaERERFQkikTuX/+CRx+Fhg3hscegQYOoIxIREVGRKBKpDRvqc9llfv6WW6Br12jjERERSVKRKBKhceP25bPP4LDDYPDgqKMREREpoyJRJCJvvQVTpuxNgwb+buaGDaOOSEREpIyKRJEIrF8Pl1wCzhmDB8Phh0cdkYiIyPZUJIpE4Pe/h08/hU6dSrj55qijERERqajORaKZdTCz6WY218zmmJmeEyFShRkz4P77/TOZBw2axy67RB2RREG5U0QyXRCDbWwFBjrnZptZC6DIzF5xzs0NYN8isbJxI/zmN1BaCoMGwYEHros6JImOcqeIZLQ6n0l0zn3pnJudmF8HfAzsXdf9isTR0KEwbx4ccIB/sorkLuVOEcl05pwLbmdmnYA3gK7OubXl1hUABQB5eXn5kyZNCuy4O1NcXEzz5s1DO17Y4ty+nj17Mn369KjDCMQnnzTnqqvycQ4eeuhdunZdG+vPDsL/2ezZs2eRc65baAcMiHJn+OLcNohX7qxM3D+/jMmdzrlAJqA5UAScsbNt8/PzXZimT58e6vHCFuf2+R/R7Ldpk3OHHOIcOHfNNWXL4/zZORd++4BZLqCcFtak3BmNOLfNufjkzh2J++eXKbkzkLubzawhMBl4yjn3fBD7FImTO+6ADz+Ezp3h9tujjkYyhXKniGSyIO5uNmA88LFzbkTdQxKJl/fegzvv9PPjx0PTptHGI5lBuVNEMl0QZxKPBi4AjjOz9xLTKQHsVyTrbdniB83euhX69IGf/CTqiCSDKHeKSEar8xA4zrm3AAsgFpHYufdeePdd6NQJ7r476mgkkyh3ikim0xNXRNJkzhw/5A3AuHEQ4xvxREQkhlQkiqTB1q1+0OzNm6GgAI4/PuqIREREakZFokgajBgBM2dChw4wbFjU0YiIiNScikSRgM2bB7//vZ8fOxZ23TXaeERERGojiGc3i0jCtm3+buZNm+Dii+Hkk6OOSERySWmpH1VBJAg6kygSoJEj4e23Ya+9fJeziEiY+vaFVq0AOkYdisSAziSKBGTBArjxRj8/Zgzstlu08YhIbvniC3+Jy7ZtAEdEHY7EgM4kigSgtBQuvRQ2bIDzzoPevaOOSERyzZgxyQIRoE2UoUhMqEgUCcAjj8Abb8Aee8CDD0YdjYjkmk2bfJFYZveoQpEYUZEoUkeLFsGgQX5+9GjYXblZREL2l7/A11+nLlEikrpTkShSB87B5ZdDSQmcdRaceWbUEYlIrnGurAfjyCOTS9XdLHWnIlGkDsaNg1dfhTZtYNSoqKMRkVz05pt+8P42baB//+RSnUmUulORKFJLS5bAwIF+fuRIfz2iiEjY7rvPv/bpA+3bJ5eqSJS60xA4IrXgnH8m87p1cNppcPbZUUckIrlo3jyYNg0aN4arroJvvkmuUXez1J2KRJFaeOIJ+Pvf/ViIo0eDWdQRiUguSp5FvOii8r0ZOpModafuZpEaWrYMfvc7P//AA/7pKiIiYVu82H9hrVev7NKX1q2Ta3dLGTNRpHZUJIrUgHPw29/C6tVwyilwwQVRRyQiueree/1zms85B7p08csaNICWLQHqs3p1lNFJHKhIFKmBp5+GF16AXXf1A9eqm1lEovDll350BSh7HGhSm8TliGXXJ4rUjopEkWr66ivo18/PjxiRehehiEi47r3XP2XlzDPh+9/ffl1yQH8ViVJXKhJFqqlPH1i1Ck48ES65JOpoRCRXLV3qHwUKcMstFdcni8SVK8OLSeJJRaJINfzlLzB5MjRvDn/8o7qZRSQ6t93mzyKefTb84AcV16u7WYKiIlFkJ1au9OOPge/i2WefaOMRkdy1YAGMHw/168PQoZVvo+5mCYqKRJGduPpqWLECevSAK66IOhoRyWU33wzbtvlxEfffv/JtVCRKUFQkilRh6lR/R3PTpv7bez39xohIRN5+G555xj9dZciQHW+X7G7WNYlSV/qTJ7IDq1bBlVf6+bvugv32izYeEcldzsGAAX5+4EDo0GHH2+pMogRFRaLIDgwYAMuXw9FHQ9++UUcjIrns2WdhxgzIy4NBg6reVkWiBEVFokgl/vY3ePxx360zYYK6mUUkOsXFZY/du+02aNGi6u01BI4ERX/6RMpZswYuv9zP33bbji8OFxEJw223wRdfQLdu1RujVUPgSFBUJIqUc+21PiEfeSRcc03U0YhILvv4Y/+EJzMYPdoPfbMzqd3NzqU3Pok3FYkiKV55xT8PtVEj381cnYQsIpIOpaVQUABbt8Jll8EPf1i9f9e4MUAJW7bAunXpjFDirkHUAYhkinXryrqZb70VDj442nhEJLeNGQNvveVvVrnnnpr+65VAMz75pOo7obPVqlWNWL486ijSJ1PaF0iRaGYTgF7A1865rkHsUyRsgwfD55/DEUfAdddFHY3EnfKmVGXJkrK7mEeNgt12q+kevgH24Uc/CjiwjHFU1AGkWWa0L6gziROBUcATAe1PJFSFhf56n4YN4bHH/KtImk1EeVMqUVoKl17qezdOOw3OPLM2e3mMvfY6gtLSoKPLDJs3b6ZRo0ZRh5E2Ybfvq68qXx5Ikeice8PMOgWxL5GwlZT4hAxw001w6KHRxiO5QXlTdmT0aH999O67wyOP+JtWam4Uy5aNDDq0jFFY+B969OgRdRhpE3b7dvQzFto1iWZWABQA5OXlUVhYGNahKS4uDvV4YYt7+9LdtlGjOrNwYXv226+Yo44qorAwvNsB4/7Zxb19YVDuTI9MbdvixU249tpuQH369fuIefNWMm9e7faVie0LSqZ+fkHJmPY55wKZgE7AR9XZNj8/34Vp+vTpoR4vbHFun/8RTZ+33nLOzLn69Z0rKkrroSoV58/OufDbB8xyAeW0MKaa5E2n3BmoTGzb+vXOHXqoc+DcBRfUbV/pzp1Ry8TPL0iZkjs1BI7krA0b/MC0zvkLxI84IuqIRCSXDRwIH3wAnTv7m1VEoqYiUXLWrbfC/Pl+qJvf/z7qaEQkl02a5K8/bNQInnkGdt016ohEAioSzexp4G3gADNbamaXBrFfkXR55x0YPtw/k/mxx2CXXaKOSHKN8qYkffBB2eP2RoxQr4ZkjqDubj43iP2IhGHTJp+QS0v9eIjxHUdMMpnypgCsWgWnn+4vf7nwQrjqqqgjEimj7mbJOUOHwty5sP/+8Ic/RB2NiOSqjRvh1FNh4UJ/9vDRR2s73I1IeqhIlJwye7Z/vJWZfzZzkyZRRyQiuai0FM4/3z92r317mDpV+Ugyj4pEyRmbN8NvfgPbtsHVV8PRR0cdkYjkqoEDYfJkaNkS/vY3XyiKZBoViZIz7rrLXyC+335wxx1RRyMiuWrECHjgAX8n85Qp0FVP7pYMpSJRcsIHH8Dtt/v58eOhWbNo4xGR3DR+vD+LCDBxIvTsGWk4IlVSkSixt2WL72beutXfORjjx32KSAYbMwYuu8zPDx8O5+r+dslwKhIl9u67z9+wss8+cPfdUUcjIrno4Yfhyiv9/PDhMGBAtPGIVIeKRIm1uXNhyBA//8c/QosWkYYjIjnogQegb18//+CDKhAle6hIlNjats0Pmr15s+/iOfHEqCMSkVxSWgqDB8M11/j3o0f7kRVEskUgT1wRyUT33+8fv9e+ve9yFhEJy/r1/gkqkydD/fowdmzZo/dEsoWKRIml+fPhllv8/JgxfiwyEZEwLF/un6Ty3//63POXv8AJJ0QdlUjNqUiU2El2M2/c6L/Jn3JK1BGJSK6YMQN+/WtYsgQ6dYKXXoKDD446KpHa0TWJEjsPPwz//jfsuafvchYRSbfSUrj3Xjj2WF8gdu/uC0YViJLNVCRKrPzvf3DDDX7+0Uehdeto4xGR+FuxAn7xCxg0yI/HOmAAvP465OVFHZlI3ai7WWKjtNTfxbx+vR+k9tRTo45IROLu+eehTx9/HWLr1vD449CrV9RRiQRDRaLExpgxUFgIbdvCQw9FHY2IxNmXX/qxD59/3r8/5hj485+hQ4do4xIJkrqbJRY+/xyuv97PP/wwtGkTbTwiEk/btvmB+Q8+2BeIzZvDyJG+e1kFosSNziRK1nMOLr8ciovhzDPhrLOijkhE4ujVV2HgQPjgA//+5z/31z537BhtXCLpojOJkvUmTIBXXoHdd/dnEUVEgjRnjr/O8MQTfYHYsSM8/bQf3kYFosSZziRKVlu6tOw5qA89pLsJRSQ4770Hd9zhn5rinH/2+403Qv/+0KRJ1NGJpJ+KRMlazsEVV8DatfDLX/o7mkVE6sI5+M9/4J57YNo0v6xRIz9ywq23wh57RBufSJhUJErW+tOf4OWXoVUreOQRMIs6IhHJVhs2+C7kUaPg3Xf9siZN4Mor4dproV27aOMTiYKKRMlKX37pu3zAP1VFCVxEaso536X8xBN+WrXKL2/TBgoKfI7RmUPJZSoSJes4B7/9Laxe7e8uvOiiqCMSkWyyeDFMmtSBfv3go4/KlnfrBv36+WcvN24cXXwimUJFomSdZ56BqVP9ReRjxqibWUSq5hzMmwdTpvixDYuKAL4H+FERzj0XLrwQfvjDSMMUyTgqEiWrfP21f8oBwPDhGrxWRCq3di3861/wj3/46bPPytY1awbdun3NgAF7cPLJ/sYUEalIRaJklb594Ztv4Pjj/d2GIiLgn53873+XTbNnw9atZetbt4beveGMM/x4h++8M5cePXTBoUhVVCRK1pg8GZ57zp8FGDdO3cwiuWrNGnj/fX/TSVGRH7JmwYLtt6lfH446Ck46yU/duvllIlJ9KhIlK3zzDVx1lZ+/5x7o1CnScEQkBOvXw6efwief+GsKk4XhwoUVt23WDLp3h2OOgaOP9vMtWoQfs0icqEiUrNC/v78e8ac/9Xc2i0j2Ky31w1ktXgyff172On++LwyXLKn83+2yCxxyCBx2mJ+6d4cf/AAa6C+aSKD0KyUZb9o0eOopP7DtuHFQT08cF8loW7fCihXw1VeVT0uX+mJw6VLYsmXH+2nYEL73PTjgAD8deqgvCg84QAWhSBgC+TUzs5OBB4H6wDjn3N1B7FcEWnHFFX7uzjuhc+dooxEJUibmzi1boKQEioth8eImzJ7t50tK/B3Dq1fDt99WPiXXrV7th52pjj32gI4dYZ99/NSxY1lhuO++KgZFolTnXz8zqw88DJwILAVmmtkLzrm5dd23CAzlyy/hxz/2g9yKxEVtcueaNfDXv/ozdclp82bYtKl208aNvvhLFoUlJX5/ZY6sZdugbVvIy6t82ntvXxB26OB7CEQkMwXxHe1HwALn3EIAM5sEnAqoSJQ6WboUoADwg2brzkSJmRrnzgUL4PTT0xtUvXrQvLmf6tVbT9u2TWnWrGzZbrv556Xvttv2U+qyVq10BlAkDoL4Nd4bSL28eCmVfP00swKSf/H9+wAOLfH2ENAPeJZDDz076mBEglaL3NkZ+CuwFdiWmDbVcSoBir97LS3dxNq1vmsZkl/WJBvp76zUVWjf9ZxzY4GxAN26dXOzZs0K69AUFhbSo0eP0I4Xtji274svYL/9fNfXhx/+mq5dfx11SGkRx88uVdjti+MfxYq587TQjh3nn884tw3874Kr7oWhWSjun1+m5M4g7hP9Akh9OFr7xDKRWhsxInlt1LN07Rp1NCJpodwpIhktiCJxJtDFzPY1s0bAOcALAexXclRJCUyYkHx3b5ShiKSTcqeIZLQ6F4nOua1AX+AfwMfAs865OXXdr+Sup57yQ2h07w5QFHU4Immh3CkimS6QaxKdcy8DLwexLwV5uawAAA35SURBVMltzsGoUX6+Xz+YMSPaeETSSblTRDKZnl0hGeXNN+HDD/1Yar/6VdTRiIiI5C4ViZJR7r/fvxYUQKNG0cYiIiKSy1QkSsaYPx+mTvXFYZ8+UUcjIiKS21QkSsa4/35/TeKFF/ruZhEREYmOikTJCCtWwMSJfn7AgEhDEREREVQkSoYYORI2boReveCgg6KORkRERFQkSuTWrIGHHvLzgwZFG4uIiIh4KhIlcqNG+UKxRw845piooxERERFQkSgRKy4uG/bmlluijUVERETKqEiUSD38MHzzDRx1FPTsGXU0IiIikqQiUSKzZg3cc4+fv/VWMIs2HhERESmjIlEiM3w4fPutvxbxxBOjjkZERERSqUiUSKxYUXYt4h136CyiiIhIplGRKJEYOtTftHLKKf56RBEREcksKhIldPPmwSOPQL16ZdckioiISGZRkSihu/562LYNLrsMunaNOhoRERGpjIpECdVrr8G0adC8ue9yFhERkcykIlFCs3kz9O3r52+4AfLyoo1HREREdkxFooTmgQf89YhdusDAgVFHIyIiIlVRkSihWLKkrHt55EjYZZdo4xEREZGqqUiUtHMOrroKSkrgzDPhpJOijkhERER2RkWipN0zz8CLL0LLlvDQQ1FHIyIiItWhIlHSauVKuPpqP3/ffdCuXbTxiIiISPWoSJS0cQ6uvNI/gq9nT7j00qgjEhERkepSkShp8+STMHkytGgBEybo+cwiIiLZREWipMWiRWVjIj74IHTqFGU0IiIiUlMqEiVwW7bAOefA2rVw2mlw8cVRRyQiIiI1pSJRAnfTTfDOO9ChA4wfr25mERGRbKQiUQI1dSoMGwb168PTT0Pr1lFHJCIiIrWhIlECM28eXHCBn7/zTjj66GjjERERkdpTkSiBWL3aX3+4bh2cdRZcd13UEYmIiEhd1KlINLOzzGyOmZWaWbeggpLssmmTLxA/+QS6dtVwNyI7o9wpItmgrmcSPwLOAN4IIBbJQqWlcNFF8Prr/mkqL70EzZtHHZVIxlPuFJGM16Au/9g59zGA6bRRzho0yD+buUULePll6Ngx6ohEMp9yp4hkgzoViTVhZgVAAUBeXh6FhYVhHZri4uJQjxe2qNo3efLejBrVhfr1S7n11g/59ttvSUcY+uyyV9zbFwblzvSIc9uS4ty+uH9+GdM+51yVE/Aqvmuk/HRqyjaFQLed7Ss55efnuzBNnz491OOFLYr2jRnjnH86s3NPPJG+4/gf0fjSz2awgFmumnko3ZNyZ2aLc9ucU+7MdpmSO3d6JtE5d0JQBanEw+jR0KePnx8+vGzYGxEpo9wpItlOQ+BIjTz0UFmB+MADMGBAtPGIiIhIetR1CJzTzWwp8GPgJTP7RzBhSSa67z7o39/PjxpVNi8iNaPcKSLZoK53N08BpgQUi2Sobdvgmmtg5Ej//tFH4Yoroo1JJJspd4pINgjt7mbJTuvWwbnn+vEPGzWC8ePh/POjjkpERETSTUWi7NDSpdCrF7z/Puy+O0yZAsceG3VUIiIiEgbduCKVevVVyM/3BeL++8OMGSoQRUREcomKRNnO1q1w883ws5/B11/DccfB229D585RRyYiIiJhUpEo31m6FHr2hDvuADP4wx/gn/+E1q2jjkxERETCpmsShdJSGDcOrrsO1q6FvfaCP/8ZevSIOjIRERGJiorEHPfpp3D55fD66/59797+Dua2baONS0RERKKl7uYctWED3H47HHqoLxDbtoVJk2DqVBWIIiIiojOJOae01Hcl33CDvwYR4MILYcQIP8yNiIiICKhIzBnOwWuv+eJw1iy/7LDDfHHYs2e0sYmIiEjmUXdzzDnnn5Zy1FFw4om+QGzXDh57zM+rQBQREZHK6ExiTG3eDJMnw7Bh8O67flmbNjBgAFx9NTRrFm18IiIiktlUJMbMsmUwZoyfvvrKL9tzTz+8zRVXqDgUERGR6lGRGAObNxuTJ8MTT8DLL/unpgB07Qr9+vkbUxo3jjZGERERyS4qErPU1q3wxhvw3HPw1FNHsW6dX16/PvzqV9C3L/zkJ/7JKSIiIiI1pSIxixQXw/TpMGUKvPACfPNNck1DDj/cnzE891zIy4syShEREYkDFYkZbOtWfwfyK6/Aq6/C22/Dli1l6w84AE4/Hbp0mckll/wwukBFREQkdlQkZpDiYl8UzpjhC8LXX4c1a8rW16sHRx7pH513xhlw0EF+eWFhSTQBi4iISGypSIxIcTHMmQMffgj//S+88w589JF/IkqqLl38+IYnnODHNGzVKpp4RUREJLeoSEyzNWvg00/9NHeuLwo//BAWLqy4bYMGcPjh/mzhkUfCT38K++wTfswiIiIiKhLrwDlYuxaWLPHPQU6+fvaZLwoXLICVKyv/tw0bwoEHwiGHwBFHQPfu/rVJk3DbICIiIlIZFYnlOAerV/vibsWKsin1/fLlZUVhcXHV+2vSBDp39lOyKDzkENh/f2jUKJw2iYiIiNRU1heJzvk7ftevhw0b/GtySr6fObMtCxf6rt81a/zZv9TX5HyyOEwORl0dTZtChw7Qvr1/7dABOnb01xJ26QJ77aWxCkVERCT7RFIkLlsGN97oi7vNm2v2umHD9sXghg2wbdvOjvj9GsXXogW0beufddy2bdmUfL/HHmUFYatWKgJFREQkfiIpEr/8Eu66K7j9NWzou3WbNi17TZ0vKVnBfvu1pWVLaNkSdt2VSudbtoTdd9cj7ERERETqRXHQ5s3XATcC1wH9GTRoMTffvAi4CDgX+BXnnPM0L78MrVv/GvgJ0J0DDzyPOXPg7LNvAPKAXYGGLFq0jCefnMby5cZnnxlz5hiXXTaW11+Hv//dePPNPXj8cWPhwt4MHQqvv96bggLj7LONn//c+PGP4a23xtK+vdGkiWFmTJs2jWXLlmFm300FBQUA5Ofnf7esXbt2AAwZMmS7bYuKiigqKtpu2ZAhQwBo167dd8vy8/MBKCgo2G7bZcuWMW3atO2WjR07FmC7Zb179wagd+/e2y0HGDt27HbLsrFNqevi0qY4fk47alPPnj1DbZOIiATHnHOhH7Rbt25u1qxZoR2vsLCQHj16hHa8sMW5fWZGFD+jYYnzZwfht8/Mipxz3UI7YMiUO4MT57aBcme2y5TcGcmZRBERERHJbCoSRURERKQCFYkiIiIiUoGKRBERERGpQEWiiIiIiFRQpyLRzIaZ2Twz+8DMpphZq6ACExGJK+VOEckGdT2T+ArQ1Tl3KDAfuKHuIYmIxJ5yp4hkvDoVic65fzrnkk86ngG0r3tIIiLxptwpItkgyMfyXQI8s6OVZlYAFADk5eVRWFgY4KGrVlxcHOrxwhb39sW5bXH/7OLevoAod0Ygzm1LinP74v75ZUr7dvrEFTN7FdizklU3OeemJra5CegGnOGqMcS7nhoQrDi3T08NyG6Z8tSAKCh3ZrY4tw2UO7NdpuTOnZ5JdM6dsJMdXwz0Ao6vTpITEckFyp0iku3q1N1sZicD1wM/dc6tDyYkEZF4U+4UkWxQ17ubRwEtgFfM7D0zezSAmERE4k65U0QyXp3OJDrnOgcViIhIrlDuFJFsoCeuiIiIiEgFKhJFREREpIKdDoGTloOarQA+D/GQbYCVIR4vbHFuX5zbBmpf0PZxzrUN8XihUu4MVJzbBmpftsuI3BlJkRg2M5uVKWOnpUOc2xfntoHaJ5ktzp9fnNsGal+2y5T2qbtZRERERCpQkSgiIiIiFeRKkTg26gDSLM7ti3PbQO2TzBbnzy/ObQO1L9tlRPty4ppEEREREamZXDmTKCIiIiI1oCJRRERERCrIuSLRzAaamTOzNlHHEhQzG2Zm88zsAzObYmatoo4pCGZ2spl9YmYLzGxw1PEEycw6mNl0M5trZnPMrH/UMQXNzOqb2btm9mLUsUjdxDFvgnJnNlLuDFdOFYlm1gH4GbA46lgC9grQ1Tl3KDAfuCHieOrMzOoDDwM/Bw4GzjWzg6ONKlBbgYHOuYOB7kCfmLUPoD/wcdRBSN3EOG+Ccmc2Uu4MUU4VicD9wPVArO7Wcc790zm3NfF2BtA+yngC8iNggXNuoXNuMzAJODXimALjnPvSOTc7Mb8OnxD2jjaq4JhZe+AXwLioY5E6i2XeBOXObKTcGa6cKRLN7FTgC+fc+1HHkmaXAH+LOogA7A0sSXm/lBglglRm1gk4HHgn2kgC9QC+sCiNOhCpvRzKm6DcmXWUO9OvQdQBBMnMXgX2rGTVTcCN+C6TrFRV25xzUxPb3IQ/Ff9UmLFJ7ZlZc2Ay8Dvn3Nqo4wmCmfUCvnbOFZlZj6jjkarFOW+CcmdcKXeGI1ZFonPuhMqWm9khwL7A+2YGvkthtpn9yDm3PMQQa21HbUsys4uBXsDxLh6DX34BdEh53z6xLDbMrCE+yT3lnHs+6ngCdDTwSzM7BWgM7GpmTzrnzo84LqlEnPMmKHei3JlNMi535uRg2ma2COjmnFsZdSxBMLOTgRHAT51zK6KOJwhm1gB/Ifnx+AQ3E/g/59ycSAMLiPm/uo8Dq5xzv4s6nnRJfBu+1jnXK+pYpG7iljdBuTMbKXeGK2euSYy5UUAL4BUze8/MHo06oLpKXEzeF/gH/sLkZ+OS5BKOBi4Ajkt8Zu8lvj2KSHiUO7OPcmeIcvJMooiIiIhUTWcSRURERKQCFYkiIiIiUoGKRBERERGpQEWiiIiIiFSgIlFEREREKlCRKCIiIiIVqEgUERERkQr+H1rI40vUhFV6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 792x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(11,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "# coordinate \n",
    "plt.plot([-7, 7], [0, 0], 'k-', linewidth=1)\n",
    "plt.plot([0, 0], [-2.2, 5], 'k-', linewidth=1)\n",
    "plt.plot([-5, 5], [-1.758, -1.758], 'k--', linewidth=1)\n",
    "# Plot\n",
    "plt.plot(z, selu(z), 'b-', linewidth=2)\n",
    "plt.grid(True)\n",
    "plt.title(\"SELU activation function\", fontsize=14)\n",
    "plt.axis([-5, 5, -2.2, 3.2])\n",
    "\n",
    "plt.subplot(122)\n",
    "# coordinate \n",
    "plt.plot([-7, 7], [0, 0], 'k-', linewidth=1)\n",
    "plt.plot([0, 0], [-2.2, 5], 'k-', linewidth=1)\n",
    "# Plot\n",
    "plt.plot(z, derivative(selu, z), \"b-\", linewidth=2, label=\"SELU\")\n",
    "plt.grid(True)\n",
    "plt.title(\"Derivative\", fontsize=14)\n",
    "plt.axis([-5, 5, -2.2, 3.2])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to use it with Keras and TensorFlow 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.core.Dense at 0x84a1727d0>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "Dense(10, activation='relu', kernel_initializer='lecun_normal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf-tutorial",
   "language": "python",
   "name": "tf-tutorial"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}