Tennis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Collaboration and Competition\n",
    "\n",
    "---\n",
    "\n",
    "### 1. Start the Environment\n",
    "\n",
    "Run the next code cell to install a few packages.  This line will take a few minutes to run!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mtensorflow 1.7.1 has requirement numpy>=1.13.3, but you'll have numpy 1.12.1 which is incompatible.\u001b[0m\r\n",
      "\u001b[31mipython 6.5.0 has requirement prompt-toolkit<2.0.0,>=1.0.15, but you'll have prompt-toolkit 3.0.7 which is incompatible.\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!pip -q install ./python"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The environment is already saved in the Workspace and can be accessed at the file path provided below. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from unityagents import UnityEnvironment\n",
    "from workspace_utils import active_session\n",
    "import random\n",
    "import torch\n",
    "import numpy as np\n",
    "import datetime\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from ddpg_agent import Agent, ReplayBuffer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:unityagents:\n",
      "'Academy' started successfully!\n",
      "Unity Academy name: Academy\n",
      "        Number of Brains: 1\n",
      "        Number of External Brains : 1\n",
      "        Lesson number : 0\n",
      "        Reset Parameters :\n",
      "\t\t\n",
      "Unity brain name: TennisBrain\n",
      "        Number of Visual Observations (per agent): 0\n",
      "        Vector Observation space type: continuous\n",
      "        Vector Observation space size (per agent): 8\n",
      "        Number of stacked Vector Observation: 3\n",
      "        Vector Action space type: continuous\n",
      "        Vector Action space size (per agent): 2\n",
      "        Vector Action descriptions: , \n"
     ]
    }
   ],
   "source": [
    "env = UnityEnvironment(file_name=\"/data/Tennis_Linux_NoVis/Tennis\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Environments contain **_brains_** which are responsible for deciding the actions of their associated agents. Here we check for the first brain available, and set it as the default brain we will be controlling from Python."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the default brain\n",
    "brain_name = env.brain_names[0]\n",
    "brain = env.brains[brain_name]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Examine the State and Action Spaces\n",
    "\n",
    "Run the code cell below to print some information about the environment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of agents: 2\n",
      "Size of each action: 2\n",
      "There are 2 agents. Each observes a state with length: 24\n",
      "The state for the first agent looks like: [ 0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.         -6.65278625 -1.5        -0.          0.\n",
      "  6.83172083  6.         -0.          0.        ]\n"
     ]
    }
   ],
   "source": [
    "# reset the environment\n",
    "env_info = env.reset(train_mode=True)[brain_name]\n",
    "\n",
    "# number of agents \n",
    "num_agents = len(env_info.agents)\n",
    "print('Number of agents:', num_agents)\n",
    "\n",
    "# size of each action\n",
    "action_size = brain.vector_action_space_size\n",
    "print('Size of each action:', action_size)\n",
    "\n",
    "# examine the state space \n",
    "states = env_info.vector_observations\n",
    "state_size = states.shape[1]\n",
    "print('There are {} agents. Each observes a state with length: {}'.format(states.shape[0], state_size))\n",
    "print('The state for the first agent looks like:', states[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MADDPG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "env_info = env.reset(train_mode=True)[brain_name]\n",
    "n_agents = len(env_info.agents)\n",
    "seed = 5\n",
    "# initialize agents\n",
    "agent = Agent(state_size=state_size, \n",
    "                action_size=action_size, \n",
    "                random_seed=seed,\n",
    "                num_agents=n_agents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start Time: 2020-09-10 18:15:09.565942\n",
      "Episode 100\tAverage Score: -0.00\tScore: -0.00\n",
      "Episode 200\tAverage Score: -0.00\tScore: -0.00\n",
      "Episode 300\tAverage Score: 0.02\tScore: 0.0500\n",
      "Episode 400\tAverage Score: 0.02\tScore: -0.00\n",
      "Episode 500\tAverage Score: 0.02\tScore: -0.00\n",
      "Episode 600\tAverage Score: 0.04\tScore: -0.00\n",
      "Episode 700\tAverage Score: 0.04\tScore: -0.00\n",
      "Episode 800\tAverage Score: 0.03\tScore: 0.050\n",
      "Episode 900\tAverage Score: 0.07\tScore: 0.05\n",
      "Episode 1000\tAverage Score: 0.05\tScore: -0.00\n",
      "Episode 1100\tAverage Score: 0.10\tScore: 0.050\n",
      "Episode 1200\tAverage Score: 0.47\tScore: 2.600\n",
      "Episode 1205\tAverage Score: 0.52\tScore: 2.600"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAH+hJREFUeJzt3XuUHHWZ//H3k5nJlVzJQAaSEJRwEWGBRARh/eEPkIscWBeUoLvibXO8IKLuzwW8gOx6RNdl1YUlgOICi4ggYoRgQOAEBIKZhBBCYsKQCxmSkMltJreZZGae3x9V3enp6enpzFR1V/d8XufMmeqq6urn2zVTT38v9W1zd0RERAAGlToAERFJDiUFERFJU1IQEZE0JQUREUlTUhARkTQlBRERSVNSEBGRNCUFERFJU1IQEZG06lIHcKDGjx/vU6ZMKXUYIiJlZeHChZvdvba3/couKUyZMoX6+vpShyEiUlbMbG0h+6n5SERE0pQUREQkTUlBRETSlBRERCRNSUFERNKUFEREJE1JQURE0pQUREQSYuHabSxb35J+/E5LK7c8uYLF67YXLQYlBRGRhLj09he58GfPpx9fcdd8fvZMA3932wtFi0FJQUQkoVY17Sr6ayopiIhImpKCiMgBuOu5VfxmwbrIj7u3vTO9fOntLzLl2se7bJ9y7ePc+9KayF83W9lNiCciUkrfn7McgI+/b1Kkx21t70gvL1y7Lec+DZt2RvqauaimICKSAB0dXuoQACUFEZFE2NfZ2ftORaCkICKSAB2dvdcUrAhxxJYUzGySmT1rZsvN7HUz+2qOfc4ys2YzWxz+fDeueEREkqw9Ic1HcXY0twPfcPdFZjYSWGhmT7n7sqz9nnf3i2KMQ0Qk8QqqKVj8dYXYagruvsHdF4XLO4DlwOFxvZ6ISDlrH0h9CmY2BTgZeDnH5tPN7FUze8LMju/h+TPNrN7M6puammKMVESkNNoLqCkUQ+xJwcwOAn4LXOPuLVmbFwFHuPvfAP8FPJrrGO5+p7tPd/fptbW18QYsIlICSelTiDUpmFkNQUK4390fyd7u7i3uvjNcngPUmNn4OGMSEUmiQvoUiiHO0UcG/AJY7u639LDPhHA/zOzUMJ4tccUkIpJUSelTiHP00RnAPwKvmdnicN31wGQAd58FXAZ80czagT3ADHdPRroUESmiQpqPijD4KL6k4O5/ppd7Ldz9VuDWuGIQESkXFd98JCIihRswo49ERKR3hfQpWBEmulBSEBFJgI5k9DMrKYiIJEFnQsbYKCmIiCRAITmhGKOPlBRERBJBNQUREQklpPVISUFEJAkSMiJVSUFEJAlczUciIpKi5iMREUnTkFQREUkcJQURkQRISEVBSUFEJAnUfCQiImkJyQlKCiIiSZCQnKCkICKSBGo+EhGR/ZKRE5QURESSQHc0i4hImuY+EhGRtIR0KSgpiIgkgZqPREQkTc1HIiKyX0Laj5QUREQSIBkpQUlBRCQROhPSfqSkICKSAMlICUoKIiKJkJAuhfiSgplNMrNnzWy5mb1uZl/NsY+Z2c/MrMHMlpjZKXHFIyKSZEmZ+6g6xmO3A99w90VmNhJYaGZPufuyjH0uAKaGP+8Hbg9/i4hICcRWU3D3De6+KFzeASwHDs/a7RLgXg/MB8aYWV1cMYmIJFVCKgrF6VMwsynAycDLWZsOB9ZlPG6ke+IQEal4SWk+ij0pmNlBwG+Ba9y9JXtzjqd0e2fMbKaZ1ZtZfVNTUxxhioiUVDJSQsxJwcxqCBLC/e7+SI5dGoFJGY8nAuuzd3L3O919urtPr62tjSdYEZESSkhFIdbRRwb8Alju7rf0sNts4FPhKKTTgGZ33xBXTCIiSZWU5qM4Rx+dAfwj8JqZLQ7XXQ9MBnD3WcAc4EKgAdgNfCbGeEREpBexJQV3/zO5+wwy93Hgy3HFICJSLjwhNQXd0SwikgAJyQlKCiIiSZCQ+fCUFEREkkDfvCYiImlqPhIRkTR1NIuISFoyUoKSgohIIiSkoqCkICKSBEm5o1lJQUQkAZKREpQUREQSISEVBSUFEZEk0OgjERFJS0ZKUFIQEUkE1RRERCRNcx+JiEjiKCmIiCRAQlqPlBRERGQ/JQURkQTQ1NkiIpI4SgoiIgmgPgUREUkcJQUREUlTUhARkTQlBRGRBNA0FyIikjhKCiIiCZCMeoKSgoiIZFBSEBFJgIR0KcSXFMzsbjPbZGZLe9h+lpk1m9ni8Oe7ccUiIiKFqY7x2P8D3Arcm2ef5939ohhjEBEpCxU/95G7Pwdsjev4IiISvVL3KZxuZq+a2RNmdnyJYxERKZmk9CnE2XzUm0XAEe6+08wuBB4Fpuba0cxmAjMBJk+eXLwIRUQGmIJrCmZ2ppl9JlyuNbMj+/PC7t7i7jvD5TlAjZmN72HfO919urtPr62t7c/LiogkUkIqCoUlBTO7AfgX4LpwVQ3wv/15YTObYGYWLp8axrKlP8cUEZH+KbT56KPAyQRNPrj7ejMbme8JZvYAcBYw3swagRsIkgnuPgu4DPiimbUDe4AZnpTJP0REiiwpV79Ck8Jed3czcwAzG9HbE9z9il6230owZFVERBLSgFRon8JvzOwOYIyZ/RPwJ+Cu+MISEZFSKKim4O4/NrNzgRbgGOC77v5UrJGJiAwgZdN8ZGZVwFx3PwdQIhARqWC9Nh+5ewew28xGFyEeEZEBqWxqCqFW4DUzewrYlVrp7lfHEpWIiJREoUnh8fBHRERikJQJ8QrtaL7HzAYDR4erVrj7vvjCEhGRUigoKZjZWcA9wBrAgElmdmU4E6qIiPRTufUp/AfwYXdfAWBmRwMPANPiCkxERIqv0JvXalIJAcDdVxJOWSEiIv2XkIpCwTWFejP7BXBf+PiTwMJ4QhIRkVIpNCl8EfgycDVBn8JzwH/HFZSIyEBTbn0K1cBP3f0WSN/lPCS2qEREpCQK7VN4GhiW8XgYwaR4IiISgaTcp1BoUhia+pY0gHB5eDwhiYhIqRSaFHaZ2SmpB2Y2neCLcUREJAoFVBQs/igK7lO4BnjIzNYThH4YcHlsUYmISDfFaGDKW1Mws/eZ2QR3XwAcCzwItAN/BFYXIT4RkQEhGT0KvTcf3QHsDZdPB64HbgO2AXfGGJeIiGRJQvNRlbtvDZcvB+50998CvzWzxfGGJiIycHgBNyqUvPkIqDKzVOI4G3gmY1uh/REiIlImeruwPwDMM7PNBKONngcws6OA5phjExEZMAqpBZS8+cjdv29mTwN1wJO+v34zCPhK3MGJiMh+xWg+6rUJyN3n51i3Mp5wREQGpqTMfVTozWsiIlJixWg+UlIQEUmAQioKSRh9JCIiORQyhLQcKSmIiPRB1DmhkCRT1s1HZna3mW0ys6U9bDcz+5mZNZjZkswJ90REpLtybz76H+D8PNsvAKaGPzOB22OMRUQkUlFfoJPSGBVbUnD354CteXa5BLjXA/OBMWZWF1c8IiJRKkWfQlk3HxXgcGBdxuPGcJ2IyMBTQI4p9+aj3uRKejnLbGYzzazezOqbmppiDktEpHdJae6JWimTQiMwKePxRGB9rh3d/U53n+7u02tra4sSnIhIPpGPPiogzVR689Fs4FPhKKTTgGZ331DCeEREEi0Rcx/1lZk9AJwFjDezRuAGoAbA3WcBc4ALgQZgN/CZuGIREYlaIZ/sD+h4CWmPii0puPsVvWx34Mtxvb6ISJyiv3mt930qvflIREQOQKWPPhIRkVDUzVF9paQgItIHpegDUPORiMgAUUiSUfORiEhCJaW5J2pKCiIifeAO7R2d0R2vgH3UfCQiklB/btjMUd96gkVvbSvaa6r5SEQkoeatDOZhW7A632TQhUvKzWtKCiIifaDRRyIiPdjV1k79mmg+MZeb6HJD70dS85GIlIWrH3iFy2a9xLZde0sdivSTkoKI9NvS9c0AtLVHNxon+Yo/IZ6aj0REJE3NRyIiCRX9l+wkg5KCiEgfaPSRiIjExgvIMmo+EhFJKM19JCIi3UTVjKS5j0REJHGUFERECpTZ7l+K72guBiUFEZE+SMg1PHJKCiIiBcr1aT6qDuekJBklBRGJTKWOyBlIlBREJDJJaRcvhuj7FJLx5ikpiEhkknFZi493Wa7M0iopiEhkOjsr80I5kCgpiIj0RYXmPyUFEYlMQprFY9PlPoXIjx3xAftISUFEIrNsQwtfe3AxHWpGKluxJgUzO9/MVphZg5ldm2P7p82sycwWhz+fjzMeEYnXl+5fyO9eeZs1W3aVOpSiiW7uo2Qk0uq4DmxmVcBtwLlAI7DAzGa7+7KsXR9096viikNEiqfSKwhdRh8lpb0nYnHWFE4FGtx9lbvvBX4NXBLj64mIlK2k5Jg4k8LhwLqMx43humyXmtkSM3vYzCblOpCZzTSzejOrb2pqiiNWEZFeZV64E3INj1ycSSHX1N/Z7+MfgCnufiLwJ+CeXAdy9zvdfbq7T6+trY04TBGJWjHm/a80A6Gm0AhkfvKfCKzP3MHdt7h7W/jwLmBajPGIiEgv4kwKC4CpZnakmQ0GZgCzM3cws7qMhxcDy2OMR0SkXzJHCEU+91FCGqRiG33k7u1mdhUwF6gC7nb3183sJqDe3WcDV5vZxUA7sBX4dFzxiIhEqVJvXostKQC4+xxgTta672YsXwdcF2cMIlJ8ZpXfq6AhqSIiA1yceSApKUZJQUQiV6mfojM1btsT2bF2trXzl9VbIztefygpiEjkKv3OZoDF67ZHdqwfz10R2bH6S0lBRGIwALJCKIpa0a629ggiiYaSgohEbiDUFKKUpH55JQURiVxnhfYpxFUsS9A94EoKIhK5Cs0JsVFNQUQqWqXWFHKJoqhKCiJS0SoxJ7S07otxKorkZAUlBRGJXKXVFH73SiMn3vgkf924o9u2JH3Kj4KSgohErsJyAs+t3AzAsvUtsRw/SYlFSUFEIldpNYWaquCqvbe9s9u2SPoU+n+IyCgpiMTg7e17eOWtbaUOo2Ry3afw9vY9LFxbnu9JTVVwqdzb0T0pRCFJNYVYZ0kVGajOuPkZANbc/JESR1Ia2Xf5btnZVtbvSSop7MtRU4hCvvsUzIrbHKeagohELvsa9tbW3fu3lWHT0uDqMCmUoKZQ7EqEkoKIRK4zzzwXu/Z2FDGSaFQPCvsUOrqXK4oUl6DWIyUFEYlevrmPtu/em17+Tf06nnhtQ6Sv7e78YM5yGjZ1Hz7aV+nmoxhqCu7OPS+t7XF7sb+wSElBRCKX7yav3Rk1hW8+vIQv3r8o0tfe2NLKHc+t4sq7F0R2zHRNIYY+hbVbdufdruYjESl7+boNWvfF23yU6rRt74zuAj4oTApxxJ60HhYlBRGJXL77FFr3xdNZ2z2G6I5VlUoKMY0+ShINSRWJUUenpy8oA0nmBfnMHz7T5asrP37HSwwfXMVPLj8p8te96L+eZ2h1FRDdKKddbe3c/MRfgdw1hXwv88iiRr7+m1dZdtN5DB/c9XK7s62d994wl2vOmZr39Yt9D4OSgkiM2js7qRpUVeowii7zgpzru4x37+3gtmcbIn/dpW/vn4YiqpGv77S0ppdzJYV8zVT/8eRKADbv2Mvkg7tebjdsD96Xn/zpjSjCjIyaj0Ri1DFAv4KsoAtyzB+B43jn23I0H+Vat39bRxhL92iKPaqoUEoKEokdrfvY2Nza+44lsH33XjbtKE1s+3KMax8ICpn7aN3W/KNuMu3r6GTN5l1d1m1sbmVH674en9PX5qOW1n08+fpG1m/fw7qtu3lj0870tr+s3tpt/1y1h4ZNO3D3dP9JZj9KZ6fzZtNOVuSYcTWXYn8rm5qPJBIX/PR5GrftSeQUBh+4+Rl27+0oSWwDtaaQKna+m9i27tp/v8KLDZv5wFHje9z3pj8s4775a1n0nXMZN2IwAKf94GmOOHg48/7fh/LGcKBOvPHJ9PLIodXsaG3Pu39bVsf5S29u4Yq75vOjS09MJ4xUjQHg1mcbuOWplX0LrghUU5BI5Go3TordJbyDNsphkeUk9Sk9X9NKpoamnXm3P7tiE9D1xjfoOsY/u2YQxUytvSUEgNb2rn9fqbK8sm477WFmyqwpzF+1pd9xxUlJQSRG7QO2+Sj4Xei4/kKv36mLdK6byLo11RXpre9WxhyFydwn6VM/qfkoIdyd++av5bi6UbTs2cfGllYufG8dY8Oqcj7tHZ3c+9JaPnnaZIZUdx3pkn3cXXs7uOH3S5k4djjTjhhL1SCjfu02Tp40ptehk0NrBjHzg+/mO48uZdyIwXS6475/sjCA3y9+m6eXb2L0sBqOmTCS+jVb+cC7x3Nk7Qjmv7mFDne+8H/ezdCa/XF2dDr//NCrXHhCHRNGDWXV5p3sbGvnE6dOpqW1nX+f+1f27O1k3IgaqgYN4qH6dRxbN5K97Z387dRaVm/exdotu2jctofB1YNwh+PqRrF5ZxvTjhibfp2b/rAstr7N5RtaOGzMMEYPq+my/id/WsnIoTU9PKurfR2dLGlsZtoRYzl8zDA+e+aRBxTDL19YXbIa2zstbV0eP1S/jgVrthZcS3ti6YYuk+ZlS5Vr1rw3OWzMsC7NMf/62DKg+xQUO9ra+fislxgzvIalbzdTN2YYqzfvYvK44dSOHELznn1MHjecle/sYNK44cxb0cSxE0YWFG+m19e3pGOA/V/E80LD5vS6++avZd7KJgAWJnxKdYtzxkIzOx/4KVAF/Nzdb87aPgS4F5gGbAEud/c1+Y45ffp0r6+vjyfgEnp51RYuv3N+l3XnHHcoP79yeq/P/fVf3uLaR17jG+cezVfO7jrm+cU3N/OJu17u8blDawalq7YHDen5M0JHp7NnXweXTZvIwwsbe40pn199/v1d2o8fW7Keq371Srf9nvzaB/nvZxt4dPH6Pr9W9SBLV+Hzla8/Ot3TF7/Ua+xsaz/g10w9JxXzK985t6APBQDNu/fxNzc9yeDqQQyuKn4DQCr2lMxyZ2/Lpbf3Kdf7mWuduxd9wr1cse9sa+/xf2vX3vYeawvTjhjb7TsnBlcNSn+Pw+fPPJJvX/SePsVpZgvdvdcLSmw1BTOrAm4DzgUagQVmNtvdl2Xs9jlgm7sfZWYzgB8Cl8cVU5LtyVHNbipwxExLOAJj+57uIzH25PkHOXbCSL7/0RO49PYXmTBqKPOvP7vHfddv38MHbn6Gxm3dP80dO2Fkzu+u7Ul2nDt7aLdt3dfBph1tObcV6lOnT+HuF1Zz4sTRzL7qzH4dqydbd+3llH99CoCl3zuvz8c54ca57GhtT8e8fc++gpPC9j1BW/sPPnoCl06b2OcYKsGMO19i/qruo4QOxL2fPZUPHl2bfjzl2seB4nwXRFt7B8d8+4+xv05P4vxIcSrQ4O6r3H0v8Gvgkqx9LgHuCZcfBs62pA7ejVlcbc/tvQzBSDV39Paujxke7Ld9d/fEk91k0ptcx8hlZ2vPn6gKdcTBw4F459sZNTTaz1ZTxgcxZ3eq5pN6T1PnaSCLokZ4UMTntJzEWfLDgXUZjxuB9/e0j7u3m1kzcDCwmYjNW9nEvz22rPcdSyTXKIdXG5s595Z5vT43NbTvvvlreS5st8x33JQRQ6oZMSRo2+/tH2lYTRU1VUbDpu6jRDL7FApxy1Mr+eULq9OPm3PUcAD++aFXWd/Pex8mjh0GBGWNS3VEzTWjhtawo7WdSWODpHDVr15h+ODC7oZONV8daIKuRAePGNLvY1SXcGqS7A9CwwZXsXdP0HxUc4D/a30RZ1LI9a5mf+4rZB/MbCYwE2Dy5Ml9CuagIdVMPfSgPj23WOa8tpETJ45mQ3MrTTvaOPc9h6a/MLyQ55597CE5P/GnjruxuZWaqkG8vX0PH5s2ka+eM5UJo4by9XOP5pKTDst7fDPjmnOO5vX1zTy9fBNnH3cIc17byE2XHM+7a4P39fk3NnPIyCFs2tHGpHHDqDJjzZbd1FQZJxw+mpXv7OSQkUM4tq57Z96c1zYy5eDhHDS0mk0tbWza0cZJk8dwQqcz9/V3mDBqKO89fBRHHzqSRxa9zYghVbzZtAszOGXyWE6aNIZf/Hk1o4fVMHxwFX9/yuEMH1zNGUeN54pTJ3NZzE0q//Z37+W4ulH9Osa9nzuVPy7dyOnvPpgZ75uUbhYs1JlHjef4w0b3K4ZKcP1HjmPM8BqOrRvJHfNWMXHscFY17eTDx09g1rw3+cgJdXzkxDo2NrcyYfRQ7nhuFXWjhlK/divHThhFpzvvzXofH/3yGSx9u7ko8Q+tqeKb5x/Dn5a9wzETRvG5M4/kY7Ne5ISJY7jqQ0fF/vqxdTSb2enAje5+Xvj4OgB3/0HGPnPDfV4ys2pgI1DreYKq1I5mEZE4FdrRHGddZAEw1cyONLPBwAxgdtY+s4Erw+XLgGfyJQQREYlXbM1HYR/BVcBcgiGpd7v762Z2E1Dv7rOBXwD3mVkDsJUgcYiISInE2sXu7nOAOVnrvpux3Ap8LM4YRESkcJrmQkRE0pQUREQkTUlBRETSlBRERCRNSUFERNJinSU1DmbWBKzt49PHE8MUGiVSKWWplHJA5ZSlUsoBKkumI9y9tredyi4p9IeZ1RdyR185qJSyVEo5oHLKUinlAJWlL9R8JCIiaUoKIiKSNtCSwp2lDiBClVKWSikHVE5ZKqUcoLIcsAHVpyAiIvkNtJqCiIjkMWCSgpmdb2YrzKzBzK4tdTz5mNkkM3vWzJab2etm9tVw/Tgze8rM3gh/jw3Xm5n9LCzbEjM7pbQl6MrMqszsFTN7LHx8pJm9HJbjwXBqdcxsSPi4Idw+pZRxZzOzMWb2sJn9NTw3p5fxOfla+Le11MweMLOh5XJezOxuM9tkZksz1h3weTCzK8P93zCzK3O9VgnK8e/h39cSM/udmY3J2HZdWI4VZnZexvpor23uXvE/BFN3vwm8CxgMvAq8p9Rx5Ym3DjglXB4JrATeA/wIuDZcfy3ww3D5QuAJgm+yOw14udRlyCrP14FfAY+Fj38DzAiXZwFfDJe/BMwKl2cAD5Y69qxy3AN8PlweDIwpx3NC8DW4q4FhGefj0+VyXoAPAqcASzPWHdB5AMYBq8LfY8PlsQkox4eB6nD5hxnleE943RoCHBlez6riuLaV/A+0SG/+6cDcjMfXAdeVOq4DiP/3wLnACqAuXFcHrAiX7wCuyNg/vV+pf4CJwNPA/wUeC/85N2f84afPDcF3b5weLleH+1mpyxDGMyq8kFrW+nI8J6nvRh8Xvs+PAeeV03kBpmRdTA/oPABXAHdkrO+yX6nKkbXto8D94XKXa1bqnMRxbRsozUepf4KUxnBd4oVV9ZOBl4FD3X0DQPj7kHC3JJfvJ8A3gc7w8cHAdndvDx9nxpouR7i9Odw/Cd4FNAG/DJvCfm5mIyjDc+LubwM/Bt4CNhC8zwspz/OScqDnIbHnJ8NnCWo5UMRyDJSkkOPr7En8sCszOwj4LXCNu7fk2zXHupKXz8wuAja5+8LM1Tl29QK2lVo1QVX/dnc/GdhF0EzRk8SWJWxvv4SgGeIwYARwQY5dy+G89Kan2BNdJjP7FtAO3J9alWO3WMoxUJJCIzAp4/FEYH2JYimImdUQJIT73f2RcPU7ZlYXbq8DNoXrk1q+M4CLzWwN8GuCJqSfAGPMLPWtf5mxpssRbh9N8DWtSdAINLr7y+HjhwmSRLmdE4BzgNXu3uTu+4BHgA9Qnucl5UDPQ2LPT9jpfRHwSQ/bhChiOQZKUlgATA1HVwwm6CybXeKYemRmRvD91cvd/ZaMTbOB1CiJKwn6GlLrPxWOtDgNaE5VpUvJ3a9z94nuPoXgPX/G3T8JPAtcFu6WXY5U+S4L90/Epzd33wisM7NjwlVnA8sos3MSegs4zcyGh39rqbKU3XnJcKDnYS7wYTMbG9acPhyuKykzOx/4F+Bid9+dsWk2MCMcCXYkMBX4C3Fc20rZWVTkDp0LCUbxvAl8q9Tx9BLrmQRVwCXA4vDnQoJ23KeBN8Lf48L9DbgtLNtrwPRSlyFHmc5i/+ijd4V/0A3AQ8CQcP3Q8HFDuP1dpY47qwwnAfXheXmUYNRKWZ4T4HvAX4GlwH0Eo1rK4rwADxD0hewj+KT8ub6cB4I2+4bw5zMJKUcDQR9B6v9+Vsb+3wrLsQK4IGN9pNc23dEsIiJpA6X5SERECqCkICIiaUoKIiKSpqQgIiJpSgoiIpKmpCADhpl1mNnijJ+8M0qa2RfM7FMRvO4aMxvfh+edZ2Y3hmPp5/Q3DpFCVPe+i0jF2OPuJxW6s7vPijOYAvwtwQ1lHwReKHEsMkAoKciAF07D8SDwoXDVJ9y9wcxuBHa6+4/N7GrgCwTz0Sxz9xlmNg64m+Cmr93ATHdfYmYHE9yYVEtws5dlvNY/AFcTTHP8MvAld+/Iiudygtku30UwR9GhQIuZvd/dL47jPRBJUfORDCTDspqPLs/Y1uLupwK3EszPlO1a4GR3P5EgOUBwV/Ar4brrgXvD9TcAf/Zg4rzZwGQAMzsOuBw4I6yxdACfzH4hd3+Q/fPsn0Bw1/HJSghSDKopyECSr/nogYzf/5lj+xLgfjN7lGCKCwimI7kUwN2fMbODzWw0QXPP34frHzezbeH+ZwPTgAXBlEMMY//EbdmmEkxbADDc3XcUUD6RflNSEAl4D8spHyG42F8MfMfMjif/tMW5jmHAPe5+Xb5AzKweGA9Um9kyoM7MFgNfcffn8xdDpH/UfCQSuDzj90uZG8xsEDDJ3Z8l+MKgMcBBwHOEzT9mdhaw2YPvvchcfwHBxHkQTNR2mZkdEm4bZ2ZHZAfi7tOBxwn6E35EMMnZSUoIUgyqKchAMiz8xJ3yR3dPDUsdYmYvE3xQuiLreVXA/4ZNQwb8p7tvDzuif2lmSwg6mlNTN38PeMDMFgHzCKaqxt2Xmdm3gSfDRLMP+DKwNkespxB0SH8JuCXHdpFYaJZUGfDC0UfT3X1zqWMRKTU1H4mISJpqCiIikqaagoiIpCkpiIhImpKCiIikKSmIiEiakoKIiKQpKYiISNr/B6LnsSZnvs9tAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7faf5178d9e8>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "End Time: 2020-09-10 18:23:07.803122\n"
     ]
    }
   ],
   "source": [
    "with active_session():\n",
    "    def ddpg(n_episodes=1500, max_t=1000):\n",
    "        scores_deque = deque(maxlen=100)\n",
    "        scores_mean = []\n",
    "        max_score = -np.Inf\n",
    "        UPDATE_FREQ = 2\n",
    "        for i_episode in range(1, n_episodes+1):\n",
    "            env_info = env.reset(train_mode=True)[brain_name]  \n",
    "            states = env_info.vector_observations                  # get the current state (for each agent)\n",
    "            scores = np.zeros(num_agents) \n",
    "            agent.reset()\n",
    "            update = False\n",
    "            for t in range(max_t):\n",
    "                actions = agent.act(states)\n",
    "                env_info = env.step(actions)[brain_name]           # send all actions to tne environment\n",
    "                next_states = env_info.vector_observations         # get next state (for each agent)\n",
    "                rewards = env_info.rewards                         # get reward (for each agent)\n",
    "                dones = env_info.local_done                        # see if episode finished\n",
    "                scores += env_info.rewards                         # update the score (for each agent) \n",
    "                if t % UPDATE_FREQ == 0:\n",
    "                    update = True\n",
    "                agent.step(states, actions, rewards, next_states, dones, update)\n",
    "                update = False\n",
    "                states = next_states \n",
    "                if np.any(dones):\n",
    "                    break \n",
    "            scores_deque.append(np.mean(scores))\n",
    "            scores_mean.append(np.mean(scores))\n",
    "            print('\\rEpisode {}\\tAverage Score: {:.2f}\\tScore: {:.2f}'.format(i_episode, \n",
    "                                                                              np.mean(scores_deque), \n",
    "                                                                              np.mean(scores)), end=\"\")\n",
    "            if i_episode % 100 == 0:\n",
    "                torch.save(agent.actor_local.state_dict(), 'checkpoint_actor.pth')\n",
    "                torch.save(agent.critic_local.state_dict(), 'checkpoint_critic.pth')\n",
    "                print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_deque)))   \n",
    "            if np.mean(scores_deque) > 0.5:\n",
    "                break\n",
    "        return scores_mean\n",
    "    print('Start Time: {}'.format(str(datetime.datetime.now())))\n",
    "    scores_mean = ddpg()\n",
    "    fig = plt.figure()\n",
    "    ax = fig.add_subplot(111)\n",
    "    plt.plot(np.arange(1, len(scores_mean)+1), scores_mean)\n",
    "    plt.ylabel('Score')\n",
    "    plt.xlabel('Episode #')\n",
    "    plt.show()\n",
    "    print('End Time: {}'.format(str(datetime.datetime.now())))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When finished, you can close the environment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}