play.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false",
    "toc-hr-collapsed": false
   },
   "source": [
    "# Reinforcement learning for compute graph topology and weight search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "%pylab inline\n",
    "import os \n",
    "os.environ['KERAS_BACKEND']='tensorflow'\n",
    "os.environ['THEANO_FLAGS'] = f'mode=FAST_RUN,device=cpu,floatX=float32'\n",
    "import keras\n",
    "import sys\n",
    "sys.path.append('/home/peter')\n",
    "sys.path.append('/home/ubuntu')\n",
    "#from universal import *\n",
    "sys.path.append('/home/peter/code/projects')\n",
    "sys.path.append('C:/Users/spook/Dropbox/code/projects')\n",
    "sys.path.append('/home/peter/code/projects/deepneat')\n",
    "sys.path.append('/home/ubuntu')\n",
    "sys.path.append('/home/ubuntu/new/automl')\n",
    "from massimport import *\n",
    "from project_common import *\n",
    "from paramspace import *\n",
    "import project_common\n",
    "from aidevutil import *\n",
    "from keras.utils.vis_utils import plot_model\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from deepneat import *\n",
    "from ipywidgets import interactive\n",
    "from scipy.optimize import minimize\n",
    "from sklearn.model_selection import *\n",
    "from sklearn.preprocessing import OneHotEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "from dask import compute, delayed, persist\n",
    "from dask.distributed import Client, wait\n",
    "from dask.distributed import as_completed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "import ipywidgets as widgets\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false",
    "code_folding": [
     0,
     4,
     74
    ]
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "from empyrical import sortino_ratio, calmar_ratio, omega_ratio\n",
    "from stable_baselines.common.policies import MlpPolicy, MlpLstmPolicy, ActorCriticPolicy, FeedForwardPolicy\n",
    "from stable_baselines.common.vec_env import SubprocVecEnv, VecEnv, VecEnvWrapper\n",
    "from stable_baselines import A2C, PPO2, DQN, ACKTR, ACER#, TRPO\n",
    "from stable_baselines.common.vec_env import DummyVecEnv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "import matplotlib as mpl\n",
    "import shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false",
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "gc.enable()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "#### sigmoids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def revsigm(data):\n",
    "    return 1.-(1./(1.+np.exp(-data)))\n",
    "\n",
    "def sigmoid(x):\n",
    "    return 1/(1+np.exp(-x))\n",
    "\n",
    "def retanher(x):\n",
    "    if x > 0:\n",
    "        return np.tanh(x)\n",
    "    else:\n",
    "        return 0\n",
    "    \n",
    "retanh = np.vectorize(retanher)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Get data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def Outputs(data):\n",
    "    return np.round(revsigm(data))\n",
    "    \n",
    "def MungeData(data):\n",
    "    # Sex\n",
    "    data.drop(['Ticket', 'Name'], inplace=True, axis=1)\n",
    "    data.Sex.fillna('0', inplace=True)\n",
    "    data.loc[data.Sex != 'male', 'Sex'] = 0\n",
    "    data.loc[data.Sex == 'male', 'Sex'] = 1\n",
    "    # Cabin\n",
    "    cabin_const = 1\n",
    "    data.Cabin.fillna(str(cabin_const), inplace=True)\n",
    "    data.loc[data.Cabin.str[0] == 'A', 'Cabin'] = cabin_const\n",
    "    data.loc[data.Cabin.str[0] == 'B', 'Cabin'] = cabin_const\n",
    "    data.loc[data.Cabin.str[0] == 'C', 'Cabin'] = cabin_const\n",
    "    data.loc[data.Cabin.str[0] == 'D', 'Cabin'] = cabin_const\n",
    "    data.loc[data.Cabin.str[0] == 'E', 'Cabin'] = cabin_const\n",
    "    data.loc[data.Cabin.str[0] == 'F', 'Cabin'] = cabin_const\n",
    "    data.loc[data.Cabin.str[0] == 'G', 'Cabin'] = cabin_const\n",
    "    data.loc[data.Cabin.str[0] == 'T', 'Cabin'] = cabin_const\n",
    "    # Embarked\n",
    "    data.loc[data.Embarked == 'C', 'Embarked'] = 1\n",
    "    data.loc[data.Embarked == 'Q', 'Embarked'] = 2\n",
    "    data.loc[data.Embarked == 'S', 'Embarked'] = 3\n",
    "    data.Embarked.fillna(0, inplace=True)\n",
    "    data.fillna(-1, inplace=True)\n",
    "    return data.astype(float)\n",
    "\n",
    "train = pd.read_csv(\"../automl/titanic/train.csv\", dtype={\"Age\": np.float64}, )\n",
    "test = pd.read_csv(\"../automl/titanic/test.csv\", dtype={\"Age\": np.float64}, )\n",
    "mdt = MungeData(train)\n",
    "\n",
    "bdy = mdt['Survived'].values\n",
    "bdx = mdt.drop(['Survived'], axis=1).values[:,1:]\n",
    "bdx_test_df = MungeData(test)\n",
    "x_val_real = bdx_test_df.values[:,1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "np.random.seed()\n",
    "rnd.seed()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false",
    "toc-hr-collapsed": true,
    "toc-nb-collapsed": true
   },
   "source": [
    "### nrows and split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false",
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "nrows = 50000000\n",
    "bdx = bdx[0:nrows]\n",
    "bdy = bdy[0:nrows]\n",
    "seed = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false",
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "# sc = StandardScaler()\n",
    "# sc.fit(bdx)\n",
    "# bdx = sc.transform(bdx)\n",
    "# x_val_real = sc.transform(x_val_real)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "#bdx = np.tanh(bdx)\n",
    "#x_val_real = np.tanh(x_val_real)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false",
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "np.min(bdx, axis=0), np.max(bdx, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "rtit = pd.read_csv('../automl/titanic/titanic_100p_score.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false",
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "# cp = int(0.9 * bdx_test.shape[0])\n",
    "# bdx_val = bdx_test[cp:]\n",
    "# bdy_val = bdy_test[cp:]\n",
    "# bdx_test = bdx_test[0:cp]\n",
    "# bdy_test = bdy_test[0:cp]\n",
    "\n",
    "bdx_val = x_val_real\n",
    "bdy_val = rtit.values[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false",
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false",
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Num nodes and min/max weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "num_inputs = 8+1 #+ the bias\n",
    "num_outputs = 1\n",
    "\n",
    "max_nodes = num_inputs+num_outputs + 6\n",
    "max_links = 16\n",
    "prefer_incoming_links = 3\n",
    "prefer_incoming_links_enabled = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "max_weight = 5\n",
    "max_init_weight = 5\n",
    "# if true, the new links will have random weights, otherwise init_structural_with\n",
    "random_init_structural = 0\n",
    "init_structural_with = 1.0\n",
    "# if true, some inputs can become disconnected\n",
    "enable_disconnected_inputs = 1\n",
    "\n",
    "# the last (bias) input\n",
    "bias_const = 2.0\n",
    "enforce_bias_link = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "m = zeros((max_nodes,max_nodes))\n",
    "m[0:num_inputs, num_inputs:num_inputs+num_outputs] = 1\n",
    "m[3, 8] = 0\n",
    "\n",
    "wm = randn(max_nodes, max_nodes)\n",
    "#m, wm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "matshow(m);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### funcs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "funcs = [#'add', 'mul', 'inv', 'neg', 'abs', \n",
    "         #'div', 'arctan2', 'hypot', 'pow', 'sub', 'mod', 'hside', # all of these have arity 2\n",
    "#      'sqrt', 'sqr', \n",
    "       #'sig',\n",
    "       'tanh',\n",
    "#       'cbrt', 'cube', \n",
    "      'min', 'max', #'mean', #'span',\n",
    "#      'std', 'var', \n",
    "#      'argmin', 'argmax', 'sign', \n",
    "#      'exp', 'log',  \n",
    "      'sin', 'cos', #'tan', 'arcsin', 'arccos', 'arctan', \n",
    "#      'sinh', 'cosh', 'tanh', 'arcsinh', 'arccosh', 'arctanh',\n",
    "        ]\n",
    "len(funcs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# the node matrix\n",
    "# row - node idx\n",
    "# col - func\n",
    "nm = zeros((max_nodes-num_inputs, len(funcs)))\n",
    "nm[:,0] = 1 # all are \"add\" initially"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### See graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def num_nodes(m):\n",
    "    sml = (np.sum(m[:,num_inputs:], axis=0) > 0).tolist()[::-1]\n",
    "    if 1.0 not in sml:\n",
    "        return num_inputs + num_outputs\n",
    "    else:\n",
    "        return (max_nodes - num_inputs - num_outputs - \n",
    "                sml.index(1)) + num_inputs + num_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def see_graph(m, pos=None, wm=None, nm=None):\n",
    "    \n",
    "    inpout_list = list(range(num_inputs))+list(range(num_inputs+num_outputs))\n",
    "    \n",
    "    g = nx.DiGraph(m)\n",
    "    #g.remove_nodes_from(nx.isolates(g))\n",
    "    g.remove_nodes_from([x for x in list(nx.isolates(g)) if x not in inpout_list])\n",
    "    if pos is None:\n",
    "        pos = nx.spring_layout(g,k=50)\n",
    "        \n",
    "    if wm is not None:\n",
    "        for i in range(max_nodes):\n",
    "            for j in range(max_nodes):\n",
    "                if m[i,j] == 1:\n",
    "                    g[i][j]['weight'] = wm[i,j]\n",
    "                    \n",
    "    nlabs = {}\n",
    "    for i in range(max_nodes):\n",
    "        if i in g.nodes():\n",
    "            nlabs[i] = str(i)\n",
    "    if nm is not None:\n",
    "        for i in range(num_inputs, num_nodes(m)):\n",
    "            if i in g.nodes():\n",
    "                #g.node[i]['mm'] = funcs[argmax(nm[i-num_inputs,:].reshape(-1))]\n",
    "                nlabs[i] = str(i) + '\\n' + funcs[argmax(nm[i-num_inputs,:].reshape(-1))]\n",
    "            \n",
    "     \n",
    "    cls = []\n",
    "    if wm is not None:\n",
    "        for i,o in g.edges():\n",
    "            cls.append( g.adj[i][o]['weight'] )\n",
    "\n",
    "    nx.draw_networkx_nodes(g, pos=pos, node_size=800)\n",
    "    nx.draw_networkx_labels(g, pos=pos, labels=nlabs, font_size=10);\n",
    "\n",
    "    if cls:\n",
    "        nx.drawing.nx_pylab.draw_networkx_edges(g, pos=pos, \n",
    "                               edge_vmin = -max_weight, \n",
    "                               edge_vmax = max_weight, \n",
    "                               edge_cmap = mpl.cm.get_cmap(name='inferno'#'RdBu'\n",
    "                                                          ), \n",
    "                               width=1, edge_color=cls, \n",
    "                               alpha=1.0, \n",
    "                               arrows=1,\n",
    "                               node_size=800,\n",
    "                               arrowstyle='fancy', \n",
    "                               arrowsize=12\n",
    "                              );\n",
    "    else:\n",
    "        nx.drawing.nx_pylab.draw_networkx_edges(g, pos=pos, \n",
    "                               width=1,\n",
    "                               alpha=0.75, \n",
    "                               arrows=1, \n",
    "                              );"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "#see_graph(m,wm=wm, nm=nm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### is_ok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def is_ok(m, num_inputs, num_outputs, tell=0):\n",
    "    \n",
    "    inpout_list = list(range(num_inputs))+list(range(num_inputs+num_outputs))\n",
    "\n",
    "    if np.sum(m) == 0:\n",
    "        if tell: print('no edges')\n",
    "        return False \n",
    "\n",
    "    g = nx.Graph(m)\n",
    "    #g.remove_nodes_from(nx.isolates(g))\n",
    "    g.remove_nodes_from([x for x in list(nx.isolates(g)) \n",
    "                         if x not in inpout_list])\n",
    "    \n",
    "    if enable_disconnected_inputs:\n",
    "        trl = list(nx.connected_components(g))\n",
    "        trl = [x for x in trl if not (len(x)==1 and all([y in inpout_list for y in x]))]\n",
    "        if len(trl) > 1:\n",
    "            if tell: print('is not connected')\n",
    "            return False\n",
    "    else:\n",
    "        if len(list(nx.connected_components(g))) > 1:\n",
    "            if tell: print('is not connected')\n",
    "            return False\n",
    "    \n",
    "    \n",
    "    g = nx.DiGraph(m)\n",
    "    g.remove_nodes_from(list(nx.isolates(g)))\n",
    "    if not nx.dag.is_directed_acyclic_graph(g):\n",
    "        if tell: print('is not DAG')\n",
    "        return False\n",
    "    \n",
    "    # make sure the remaining nodes will be connected \n",
    "    inps=list(range(num_inputs)) \n",
    "    outs=list(range(num_inputs, num_inputs+num_outputs))\n",
    "    for k in g.nodes():\n",
    "        if k in inps:\n",
    "            # input\n",
    "            if (len(list(g.out_edges(nbunch=k))) == 0):\n",
    "                if tell: print('input',k, 'is not connected')\n",
    "                return False\n",
    "        elif k in outs:\n",
    "            # output\n",
    "            if (len(list(g.in_edges(nbunch=k))) == 0):\n",
    "                if tell: print('output',k, 'is not connected')\n",
    "                return False\n",
    "        else:\n",
    "            # hidden\n",
    "            if (len(list(g.in_edges(nbunch=k))) == 0 ):\n",
    "                if tell: print('node',k, 'has no incoming edges')\n",
    "                return False\n",
    "                \n",
    "            if (len(list(g.out_edges(nbunch=k))) == 0):\n",
    "                if tell: print('node',k, 'has no outgoing edges')\n",
    "                return False\n",
    "            \n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Add link"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# add link\n",
    "def add_link(m, si, di, num_inputs, num_outputs, max_links, wm=None):\n",
    "    # outputs can't be sources\n",
    "    if si in list(range(num_inputs, num_inputs+num_outputs)):\n",
    "        return False\n",
    "    # inputs can't be destinations\n",
    "    if di in list(range(num_inputs)):\n",
    "        return False\n",
    "    \n",
    "    if np.sum(m) >= max_links:\n",
    "        return False\n",
    "    \n",
    "    tm = np.copy(m)\n",
    "    tm[si, di] = 1\n",
    "    \n",
    "    if not is_ok(tm, num_inputs, num_outputs):\n",
    "        #print(\"Not acyclic in add_link!\")\n",
    "        return False\n",
    "\n",
    "    m[si, di] = 1\n",
    "    \n",
    "    if wm is not None: # the weight matrix\n",
    "        if random_init_structural:\n",
    "            wm[si, di] = rnd.uniform(-max_init_weight, max_init_weight)\n",
    "        else:\n",
    "            wm[si, di] = init_structural_with\n",
    "        \n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove link"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# remove link\n",
    "def remove_link(m, si, di, num_inputs, num_outputs, wm=None):\n",
    "    \n",
    "    # don't allow removing links coming from bias\n",
    "    if enforce_bias_link:\n",
    "        if si == num_inputs-1:\n",
    "            return False\n",
    "    \n",
    "    tm = np.copy(m)\n",
    "    tm[si, di] = 0\n",
    "    \n",
    "    if not is_ok(tm, num_inputs, num_outputs):\n",
    "        #print(\"Not acyclic in remove_link!\")\n",
    "        return False\n",
    "            \n",
    "    m[si, di] = 0\n",
    "    if wm is not None: # the weight matrix\n",
    "        wm[si, di] = 0\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Add node"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# add node\n",
    "def add_node(m, si, di, num_inputs, num_outputs, max_links, wm=None, nm=None):\n",
    "    if m[si, di] == 0:\n",
    "        return False # can't split nonexistent connections\n",
    "    \n",
    "    # don't split bias links\n",
    "    if enforce_bias_link:\n",
    "        if si == num_inputs-1:\n",
    "            return False\n",
    "    \n",
    "    if np.sum(m) >= max_links-1:\n",
    "        return False\n",
    "    \n",
    "#     if (si, di) not in innovs.innovs:\n",
    "#         # new innovation\n",
    "#         inn = innovs.last_innov_num\n",
    "#         innovs.innovs[(si, di)] = inn\n",
    "#         innovs.last_innov_num += 1\n",
    "#     else:\n",
    "#         # known\n",
    "#         inn = innovs.innovs[(si, di)]\n",
    "    \n",
    "    if not any (np.sum(m[:,num_inputs+num_outputs:], axis=0) == 0):\n",
    "        return False\n",
    "    else:\n",
    "        # there is a free slot, use that\n",
    "        inn = argmin(np.sum(m[:,num_inputs+num_outputs:], axis=0)) + num_inputs + num_outputs\n",
    "    \n",
    "    m[si, di] = 0\n",
    "    m[si, inn] = 1\n",
    "    m[inn, di] = 1\n",
    "    \n",
    "    # hack\n",
    "    # always add bias connection\n",
    "    if enforce_bias_link:\n",
    "        if si != num_inputs-1:\n",
    "            m[num_inputs-1, inn] = 1\n",
    "    \n",
    "    if wm is not None:\n",
    "        wm[si, di] = 0\n",
    "        if random_init_structural:\n",
    "            wm[si, inn] = rnd.uniform(-max_init_weight,max_init_weight)\n",
    "            wm[inn, di] = rnd.uniform(-max_init_weight,max_init_weight)\n",
    "            if enforce_bias_link:\n",
    "                if si != num_inputs-1: \n",
    "                    wm[num_inputs-1, inn] = rnd.uniform(-max_init_weight,max_init_weight)\n",
    "        else:\n",
    "            wm[si, inn] = init_structural_with\n",
    "            wm[inn, di] = init_structural_with\n",
    "            if enforce_bias_link:\n",
    "                if si != num_inputs-1: \n",
    "                    wm[num_inputs-1, inn] = init_structural_with\n",
    "        \n",
    "    if nm is not None:\n",
    "        # random func\n",
    "        if random_init_structural:\n",
    "            ri = rnd.choice(arange(len(funcs)))\n",
    "        else:\n",
    "            ri = 0\n",
    "        nm[inn-num_inputs, :] = 0\n",
    "        nm[inn-num_inputs, ri] = 1\n",
    "        \n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Remove node"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# remove a simple node (connected by 2 edges only)\n",
    "def remove_node(m, ni, num_inputs, num_outputs, wm=None, nm=None):\n",
    "    # also ensure only one incoming & outgoing edge to ni\n",
    "    if (np.sum(m[:,ni]) != 1) or (np.sum(m[ni, :]) != 1):\n",
    "        return False\n",
    "    \n",
    "    si = argmax(m[:,ni])\n",
    "    di = argmax(m[ni, :])\n",
    "    \n",
    "    # can't remove inputs/outputs\n",
    "    if ni < num_inputs+num_outputs:\n",
    "        return False\n",
    "    \n",
    "    tm = np.copy(m)\n",
    "    tm[si, ni] = 0\n",
    "    tm[ni, di] = 0\n",
    "    tm[si, di] = 1\n",
    "    \n",
    "    if not is_ok(tm, num_inputs, num_outputs):\n",
    "        #print(\"Not acyclic in remove_node!\")\n",
    "        return False\n",
    "    \n",
    "    m[si, ni] = 0\n",
    "    m[ni, di] = 0\n",
    "    m[si, di] = 1\n",
    "    \n",
    "    if wm is not None:\n",
    "        wm[si, ni] = 0\n",
    "        wm[ni, di] = 0\n",
    "        if wm[si, di] == 0:\n",
    "            if random_init_structural:\n",
    "                wm[si, di] = rnd.uniform(-max_init_weight,max_init_weight)\n",
    "            else:\n",
    "                wm[si, di] = init_structural_with\n",
    "            \n",
    "    if nm is not None:\n",
    "        nm[ni-num_inputs, :] = 0\n",
    "        \n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Input/output description"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "Inputs to the agent:\n",
    "\n",
    "- The adjacency matrix's operating field\n",
    "- The weight matrix\n",
    "- The link caret x/y position\n",
    "- What's under the link caret position (link or not)\n",
    "- What weight is under the caret position (0 if no link)\n",
    "- The node matrix\n",
    "- The node caret position\n",
    "- What's under the node caret\n",
    "- Amount of links / limit\n",
    "- Amount of nodes / limit\n",
    "- Sum of link caret row / num_nodes\n",
    "- Sum of link caret col / num_nodes\n",
    "- Link caret is at diagonal or not\n",
    "- Node removal is possible at link caret position or not\n",
    "- Last action was successful or not\n",
    "- Last score/reward\n",
    "- Charge for adding a node \n",
    "- Charge for adding a link"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "Possible actions for the agent:\n",
    "\n",
    "- Move link caret up\n",
    "- Move link caret down\n",
    "- Move link caret left\n",
    "- Move link caret right\n",
    "- Randomize link caret position\n",
    "\n",
    "- Move node caret up\n",
    "- Move node caret down\n",
    "- Randomize node caret\n",
    "\n",
    "- Attempt to add link at caret position\n",
    "- Attempt to remove link at caret position\n",
    "- Attempt to add node at caret position\n",
    "- Attempt to remove node at caret position\n",
    "\n",
    "- Mutate weight at link caret position +\n",
    "- Mutate weight at link caret position -\n",
    "- Randomize weight at link caret position\n",
    "- Set weight to 1.0 at link caret position\n",
    "\n",
    "- Mutate node at node caret position +\n",
    "- Mutate node at node caret position -\n",
    "- Randomize node at node caret position\n",
    "- Set node to default at node caret position\n",
    "\n",
    "- Save state\n",
    "- Restore state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "action_codes = [\"Move link caret up\",\n",
    "\"Move link caret down\",\n",
    "\"Move link caret left\",\n",
    "\"Move link caret right\",\n",
    "#\"Randomize link caret position\",\n",
    "\n",
    "\"Move node caret up\",\n",
    "\"Move node caret down\",\n",
    "#\"Randomize node caret\",\n",
    "\n",
    "\"Attempt to add link at caret position\",\n",
    "\"Attempt to remove link at caret position\",\n",
    "\"Attempt to add node at caret position\",\n",
    "\"Attempt to remove node at caret position\",\n",
    "\n",
    "\"Mutate weight at link caret position +\",\n",
    "\"Mutate weight at link caret position -\",\n",
    "#\"Randomize weight at link caret position\",\n",
    "\"Set weight to 1.0 at link caret position\",\n",
    "\n",
    "\"Mutate node at node caret position +\",\n",
    "\"Mutate node at node caret position -\",\n",
    "#\"Randomize node at node caret position\",\n",
    "#\"Set node to default at node caret position\"]\n",
    "               ]\n",
    "\n",
    "len(action_codes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### make input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def make_input(m, wm, nm, caret_row, caret_col, node_caret,\n",
    "               max_nodes, max_links, \n",
    "               num_inputs, num_outputs,\n",
    "               la=0, ls=0, nnc=0, nlc=0):\n",
    "    iii = []\n",
    "    # The adjacency matrix's operating field\n",
    "    iii += [m[:,num_inputs:].reshape(-1) ]\n",
    "    # The weight matrix\n",
    "    iii += [wm[:,num_inputs:].reshape(-1) / max_weight] \n",
    "    # The link caret x/y position\n",
    "    i = np.zeros(max_nodes)\n",
    "    i[caret_row] = 1\n",
    "    iii += [i]\n",
    "    i = np.zeros(max_nodes)\n",
    "    i[caret_col] = 1\n",
    "    iii += [i]\n",
    "    # What's under the link caret position (link or not)\n",
    "    iii += [array([m[caret_row, caret_col]])]\n",
    "    # What weight is under the caret position (0 if no link)\n",
    "    iii += [array([wm[caret_row, caret_col]])]\n",
    "    # The node matrix\n",
    "    iii += [nm.reshape(-1)]\n",
    "    # The node caret position\n",
    "    i = np.zeros(max_nodes-num_inputs)\n",
    "    i[node_caret]=1\n",
    "    iii += [i]\n",
    "    # What's under the node caret\n",
    "    iii += [nm[node_caret,:].reshape(-1)]\n",
    "    # Amount of links / limit\n",
    "    iii += [array([np.sum(m) / max_links])]\n",
    "    # Amount of nodes / limit\n",
    "    iii += [array([num_nodes(m)/max_nodes])]\n",
    "    # Sum of link caret row / num_nodes\n",
    "    iii += [array([np.sum(m[caret_row, :]) / num_nodes(m)])]\n",
    "    # Sum of link caret col / num_nodes\n",
    "    iii += [array([np.sum(m[:, caret_col]) / num_nodes(m)])]\n",
    "    # Link caret is at diagonal or not\n",
    "    if caret_row == caret_col:\n",
    "        iii += [array([1])]\n",
    "    else:\n",
    "        iii += [array([0])]\n",
    "    # Node removal is possible at link caret position or not\n",
    "    if (caret_row == caret_col) and ((sum(m[:,caret_col]) == 1) and (sum(m[caret_row, :]) == 1)):\n",
    "        iii += [array([1])]\n",
    "    else:\n",
    "        iii += [array([0])]\n",
    "        \n",
    "    # Last action was successful or not\n",
    "    iii += [array([la])]\n",
    "    \n",
    "    # Last score\n",
    "    iii += [array([ls])]\n",
    "    \n",
    "    # charge for nodes\n",
    "    iii += [array([nnc])]\n",
    "\n",
    "    # charge for links\n",
    "    iii += [array([nlc])]\n",
    "    \n",
    "    inp = hstack(iii)\n",
    "    return inp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "exmpi = make_input(m, m, nm, 0, 2, 0, max_nodes, max_links, num_inputs, num_outputs)\n",
    "exmpi.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false",
    "toc-hr-collapsed": false
   },
   "source": [
    "## The algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### activate graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def activate_graph(gr, inputs, num_outputs=1): \n",
    "    num_inputs = inputs.shape[0]\n",
    "    allnodes = list(nx.dfs_postorder_nodes(gr))[::-1]\n",
    "    for a in allnodes: gr.node[a]['act'] = None\n",
    "    \n",
    "    # separate input from non-input nodes\n",
    "    allnodes = [x for x in allnodes if x > (num_inputs-1)]\n",
    "    \n",
    "    # input the data\n",
    "    for i,inp in zip(range(0, num_inputs), inputs): \n",
    "        gr.node[i]['act'] = inp\n",
    "        \n",
    "    # pass through the graph\n",
    "    for an in allnodes:\n",
    "        # collect the inputs to this node\n",
    "        mm = gr.node[an]['mm']\n",
    "        \n",
    "        # also sort the incoming edges by id for consistency\n",
    "        inedg = list(gr.in_edges(nbunch=an))\n",
    "        \n",
    "        #inedg = sorted(inedg, key = lambda x: x[0])\n",
    "        \n",
    "        inps = [gr.node[i]['act'] for i,o in inedg]\n",
    "        inedgw = list(gr.in_edges(nbunch=an, data=True))\n",
    "        \n",
    "        #print(inedgw)\n",
    "        ws = [ts['weight'] for i,o,ts in inedgw]\n",
    "        # weighted stack\n",
    "        inps = np.vstack([w*x for w,x in zip(ws, inps)])\n",
    "        \n",
    "        sact = np.sum(inps, axis=0) # this node's default activation\n",
    "        act = sact\n",
    "\n",
    "        try:\n",
    "            if mm=='add':\n",
    "                act = sact\n",
    "            if mm=='neg':\n",
    "                act = -sact\n",
    "            if mm=='mul':\n",
    "                act = np.prod(inps, axis=0)\n",
    "            if mm=='inv':\n",
    "                act = 1.0 / sact\n",
    "            if mm=='sqr':\n",
    "                act = sact**2\n",
    "            if mm=='cube':\n",
    "                act = sact**3\n",
    "            if mm=='sqrt':\n",
    "                act = np.sqrt(sact)\n",
    "            if mm=='sig' or mm=='sigmoid':\n",
    "                act = sigmoid(sact)\n",
    "            if mm=='cbrt':\n",
    "                act = np.cbrt(sact)\n",
    "            if mm=='sin':\n",
    "                act = np.sin(sact)\n",
    "            if mm=='cos':\n",
    "                act = np.cos(sact)\n",
    "            if mm=='tan':\n",
    "                act = np.tan(sact)\n",
    "            if mm=='arcsin':\n",
    "                act = np.arcsin(sact)\n",
    "            if mm=='arccos':\n",
    "                act = np.arccos(sact)\n",
    "            if mm=='arctan':\n",
    "                act = np.arctan(sact)\n",
    "            if mm=='log':\n",
    "                act = np.log(sact)\n",
    "            if mm=='exp':\n",
    "                act = np.exp(sact)\n",
    "            if mm=='abs':\n",
    "                act = np.abs(sact)\n",
    "            if mm=='sinh':\n",
    "                act = np.sinh(sact)\n",
    "            if mm=='cosh':\n",
    "                act = np.cosh(sact)\n",
    "            if mm=='tanh':\n",
    "                act = np.tanh(sact)\n",
    "            if mm=='arcsinh':\n",
    "                act = np.arcsinh(sact)\n",
    "            if mm=='arccosh':\n",
    "                act = np.arccosh(sact)\n",
    "            if mm=='arctanh':\n",
    "                act = np.arctanh(sact)\n",
    "            if mm=='min':\n",
    "                act = np.min(inps, axis=0)\n",
    "            if mm=='max':\n",
    "                act = np.max(inps, axis=0)\n",
    "            if mm=='mean':\n",
    "                act = np.mean(inps, axis=0)\n",
    "            if mm=='span':\n",
    "                act = np.max(inps, axis=0) - np.min(inps, axis=0)\n",
    "            if mm=='var':\n",
    "                act = np.var(inps, axis=0)\n",
    "            if mm=='std':\n",
    "                act = np.std(inps, axis=0)\n",
    "            if mm=='argmax':\n",
    "                act = np.argmax(inps, axis=0) / inps.shape[0] # normalized argmax\n",
    "            if mm=='argmin':\n",
    "                act = np.argmin(inps, axis=0) / inps.shape[0] # normalized argmax                \n",
    "            if mm=='sign':\n",
    "                act = np.sign(sact)\n",
    "            # arity 2\n",
    "            if mm=='div':\n",
    "                if inps.shape[0]>1:\n",
    "                    act = inps[0]/inps[1]\n",
    "                else:\n",
    "                    act = inps[0]\n",
    "            if mm=='arctan2':\n",
    "                if inps.shape[0]>1:\n",
    "                    act = np.arctan2(inps[0],inps[1])\n",
    "                else:\n",
    "                    act = inps[0]\n",
    "            if mm=='hypot':\n",
    "                if inps.shape[0]>1:\n",
    "                    act = np.hypot(inps[0],inps[1])\n",
    "                else:\n",
    "                    act = inps[0]\n",
    "            if mm=='pow':\n",
    "                if inps.shape[0]>1:\n",
    "                    act = np.power(inps[0],inps[1])\n",
    "                else:\n",
    "                    act = inps[0]\n",
    "            if mm=='sub':\n",
    "                if inps.shape[0]>1:\n",
    "                    act = np.subtract(inps[0],inps[1])\n",
    "                else:\n",
    "                    act = inps[0]\n",
    "            if mm=='mod':\n",
    "                if inps.shape[0]>1:\n",
    "                    act = np.mod(inps[0],inps[1])\n",
    "                else:\n",
    "                    act = inps[0]\n",
    "            if mm=='hside':\n",
    "                if inps.shape[0]>1:\n",
    "                    act = np.heaviside(inps[0],inps[1])\n",
    "                else:\n",
    "                    act = inps[0]\n",
    "        \n",
    "        except Exception as ex:\n",
    "            print(ex)\n",
    "            act = sact\n",
    "\n",
    "        # apply slope and bias\n",
    "        #if use_bias: act = act + bias\n",
    "        #act = act.reshape(1,-1)\n",
    "        \n",
    "        #act = sigmoid(sact)\n",
    "        \n",
    "        # store activation\n",
    "        gr.node[an]['act'] = act \n",
    "        \n",
    "    #print('act:', act)\n",
    "    outputs = [gr.node[o]['act'] for o in allnodes[-num_outputs:]]\n",
    "    return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "add_node(m, 0, 8, num_inputs, num_outputs, max_links, wm=wm)\n",
    "add_node(m, 9, 8, num_inputs, num_outputs, max_links, wm=wm)\n",
    "add_node(m, 0, 9, num_inputs, num_outputs, max_links, wm=wm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "inpout_list = list(range(num_inputs))+list(range(num_inputs+num_outputs))\n",
    "g = nx.DiGraph(m)\n",
    "g.remove_nodes_from([x for x in list(nx.isolates(g)) if x not in inpout_list])\n",
    "\n",
    "for i in range(max_nodes):\n",
    "    for j in range(max_nodes):\n",
    "        if m[i,j] != 0:\n",
    "            g.edges[i,j]['weight'] = wm[i,j]\n",
    "\n",
    "for i in range(num_inputs, num_nodes(m)):\n",
    "    if i in g.nodes():\n",
    "        g.node[i]['mm'] = funcs[argmax(nm[i-num_inputs,:].reshape(-1))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "t=activate_graph(g, vstack([bdx.T, ones(bdx.shape[0])*25]), \n",
    "                 num_outputs=num_outputs)[0]\n",
    "t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "np.sum(isnan(bdx_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### score metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "score_metric = accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "#max_error = mean_squared_error(~bdy_val+2, bdy_val)\n",
    "#max_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### evaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def test_xor(m, wm, nm, num_inputs, num_outputs, max_nodes):\n",
    "    inpout_list = list(range(num_inputs))+list(range(num_inputs+num_outputs))\n",
    "    g = nx.DiGraph(m)\n",
    "    g.remove_nodes_from([x for x in list(nx.isolates(g)) \n",
    "                         if x not in inpout_list])\n",
    "\n",
    "    for i in range(max_nodes):\n",
    "        for j in range(max_nodes):\n",
    "            if m[i,j] == 1:\n",
    "                g.edges[i,j]['weight'] = wm[i,j]\n",
    "\n",
    "    for i in range(num_inputs, num_nodes(m)):\n",
    "        if i in g.nodes():\n",
    "            g.node[i]['mm'] = funcs[argmax(nm[i-num_inputs,:].reshape(-1))]\n",
    "            \n",
    "    try:\n",
    "        p = activate_graph(g, vstack([bdx.T, ones(bdx.shape[0])*bias_const]), \n",
    "                           num_outputs=num_outputs)[0]\n",
    "                   \n",
    "        t = ((np.tanh(p)+1.0)/2)#(p-np.min(p))/(np.max(p) - np.min(p))\n",
    "        if any(isnan(t)): return [0]*4\n",
    "        if any(isinf(t)): return [0]*4\n",
    "        acc = score_metric(bdy, np.round(t))\n",
    "        acc_real = acc#accuracy_score(bdy, np.round(t))\n",
    "\n",
    "        p = activate_graph(g, vstack([bdx_val.T, ones(bdx_val.shape[0])*bias_const]), \n",
    "                           num_outputs=num_outputs)[0]\n",
    "        t1 = ((np.tanh(p)+1.0)/2)#(p-np.min(p))/(np.max(p) - np.min(p))\n",
    "        if any(isnan(t1)): return [0]*4\n",
    "        if any(isinf(t1)): return [0]*4\n",
    "        acc1 = score_metric(bdy_val, np.round(t1))\n",
    "        acc1_real = acc1#accuracy_score(bdy_val, np.round(t1))\n",
    "    \n",
    "        return float(acc), float(acc1), float(acc_real), float(acc1_real)\n",
    "    except Exception as ex:\n",
    "        print(ex)\n",
    "        return [0]*4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "m = zeros((max_nodes,max_nodes))\n",
    "m[0:num_inputs, num_inputs:num_inputs+num_outputs] = 1\n",
    "g = nx.DiGraph(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def make_pos(m, wm):\n",
    "    inpout_list = list(range(num_inputs))+list(range(num_inputs+num_outputs))\n",
    "    g = nx.DiGraph(m)\n",
    "\n",
    "    for i in range(max_nodes):\n",
    "        for j in range(max_nodes):\n",
    "            if m[i,j] != 0:\n",
    "                g.edges[i,j]['weight'] = wm[i,j]\n",
    "        \n",
    "    pos = {}#nx.spring_layout(g,k=0.9)\n",
    "    #for k,v in pos.items():\n",
    "    #    pos[k] *= 0.6\n",
    "    #    pos[k] -= 0.1\n",
    "    # will provide my own coords\n",
    "    for i,x in zip(range(num_inputs), linspace(0,1,num_inputs)):\n",
    "        pos[i] = array([x,1])\n",
    "    \n",
    "    if num_outputs == 1:\n",
    "        pos[num_inputs] = array([0.5,0])\n",
    "    elif num_outputs == 2:\n",
    "        pos[num_inputs] = array([0.25,0])\n",
    "        pos[num_inputs+1] = array([0.75,0])\n",
    "    else:\n",
    "        for i,x in zip(range(num_inputs,num_inputs+num_outputs), \n",
    "                       linspace(0,1,num_outputs)):\n",
    "            pos[i] = array([x,0])\n",
    "    \n",
    "    for i,x in zip(range(num_inputs+num_outputs,max_nodes), \n",
    "                   linspace(0,1,max_nodes-(num_inputs+num_outputs))):\n",
    "        if i%2==0: \n",
    "            ofs = 0.2\n",
    "        else:\n",
    "            ofs = -0.2\n",
    "        pos[i] = array([x,0.5+ofs])\n",
    "    return pos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "pos = make_pos(m, wm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### probs and params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "add_node_prob = 1.0\n",
    "add_link_prob = 1.0\n",
    "rem_node_prob = 0.0\n",
    "rem_link_prob = 1.0\n",
    "\n",
    "mut_weight_prob = 1.0\n",
    "replace_weight_prob = 1.0\n",
    "defaultize_weight_prob = 1.0\n",
    "\n",
    "min_mut_power = 0.05\n",
    "max_mut_power = 0.5\n",
    "\n",
    "min_extra_links = 2\n",
    "min_extra_links_enabled = 1\n",
    "\n",
    "max_steps = 100000\n",
    "max_stag = 6000\n",
    "\n",
    "steps_to_charge_newnode = 160/8\n",
    "steps_to_charge_newlink = 160/8\n",
    "\n",
    "display_every = 1000\n",
    "display_matrix = 0\n",
    "display_weight_matrix = 0\n",
    "display_sleep = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "penalize_unsuccessful_move = 1\n",
    "penalize_unsuccessful_move_mul = 0.9\n",
    "\n",
    "penalize_unsuccessful_modify = 1\n",
    "penalize_unsuccessful_modify_mul = 0.5\n",
    "\n",
    "penalize_unsuccessful_struct = 1\n",
    "penalize_unsuccessful_struct_mul = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# caret movement will wrap around, it can never be unsuccessful\n",
    "infinite_caret = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false",
    "toc-hr-collapsed": false
   },
   "source": [
    "### Env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "class GraphMakerEnv(gym.Env):\n",
    "\n",
    "    metadata = {'render.modes': ['human']}\n",
    "    \n",
    "    def _configure(self, display=None):\n",
    "        self.display = display\n",
    "\n",
    "    def _seed(self, seed=None):\n",
    "        self.np_random, seed = seeding.np_random(seed)\n",
    "        return [seed]\n",
    "\n",
    "    def _step(self, action):\n",
    "        return self.step(action)\n",
    "\n",
    "    def _reset(self):\n",
    "        return self.reset()\n",
    "    \n",
    "    def _render(self, mode='human', close=False):\n",
    "        # ... TODO\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "#### init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def __init__(self, max_nodes, max_links, num_inputs, num_outputs):\n",
    "\n",
    "    self.max_nodes = max_nodes\n",
    "    self.max_links = max_links\n",
    "    self.num_inputs = num_inputs\n",
    "    self.num_outputs = num_outputs\n",
    "\n",
    "    self.action_space = spaces.Discrete(15)\n",
    "\n",
    "    self.matrix = zeros((self.max_nodes, self.max_nodes))\n",
    "    self.weight_matrix = zeros((self.max_nodes, self.max_nodes))\n",
    "    self.node_matrix = zeros((self.max_nodes-self.num_inputs, len(funcs)))\n",
    "    self.matrix[0:self.num_inputs, \n",
    "                self.num_inputs:self.num_inputs+self.num_outputs] = 1\n",
    "    for i in range(self.max_nodes):\n",
    "        for j in range(self.max_nodes):\n",
    "            if self.matrix[i,j]==1:\n",
    "                self.weight_matrix[i,j] = 1.0\n",
    "    for i in range(self.num_inputs, num_nodes(self.matrix)):\n",
    "        self.node_matrix[i-self.num_inputs, rnd.choice(arange(len(funcs)))]=1\n",
    "\n",
    "    self.caret_row = 0\n",
    "    self.caret_col = self.num_inputs\n",
    "    self.node_caret = 0\n",
    "    exmpi = make_input(self.matrix, self.weight_matrix, self.node_matrix, \n",
    "                              self.caret_row, self.caret_col, self.node_caret,\n",
    "                              self.max_nodes, self.max_links, \n",
    "                              self.num_inputs, self.num_outputs)\n",
    "\n",
    "    self.observation_space = spaces.Box(np.zeros(exmpi.shape[0]),\n",
    "                                        np.ones(exmpi.shape[0]), )\n",
    "    self.best_ever = (0,0)\n",
    "    self.reset()\n",
    "\n",
    "GraphMakerEnv.__init__ = __init__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "#### step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def step(self, action):\n",
    "\n",
    "    info = {}\n",
    "    self.step_counter += 1\n",
    "\n",
    "    move_attempt = 0\n",
    "    move_success = 0\n",
    "    modify_attempt = 0\n",
    "    modify_success = 0\n",
    "    struct_attempt = 0\n",
    "    struct_success = 0\n",
    "\n",
    "\n",
    "    # for testing\n",
    "    #action = rnd.choice(arange(15))\n",
    "\n",
    "    # execute action\n",
    "    if action == 0: \n",
    "        # move caret up\n",
    "        move_attempt = 1\n",
    "        if self.caret_row > 0:\n",
    "            self.caret_row -= 1\n",
    "            move_success = 1\n",
    "        else:\n",
    "            move_success = 0\n",
    "\n",
    "    elif action == 1:\n",
    "        # move caret down\n",
    "        move_attempt = 1\n",
    "        if self.caret_row < num_nodes(self.matrix)-1:\n",
    "            self.caret_row += 1\n",
    "            move_success = 1\n",
    "        else:\n",
    "            move_success = 0\n",
    "\n",
    "    elif action == 2:\n",
    "        # move caret left\n",
    "        move_attempt = 1\n",
    "        if self.caret_col > self.num_inputs:\n",
    "            self.caret_col -= 1\n",
    "            move_success = 1\n",
    "        else:\n",
    "            move_success = 0\n",
    "\n",
    "    elif action == 3:\n",
    "        # move caret right\n",
    "        move_attempt = 1\n",
    "        if self.caret_col < num_nodes(self.matrix)-1:\n",
    "            self.caret_col += 1\n",
    "            move_success = 1\n",
    "        else:\n",
    "            move_success = 0\n",
    "\n",
    "#     elif action == 4:\n",
    "#         # randomize caret\n",
    "#         move_attempt = 1\n",
    "#         prv = (self.caret_row, self.caret_col)\n",
    "#         self.caret_row = rnd.randint(0, num_nodes(self.matrix)-1)\n",
    "#         self.caret_col = rnd.randint(self.num_inputs, num_nodes(self.matrix)-1)\n",
    "#         if (self.caret_row, self.caret_col) != prv:\n",
    "#             move_success = 1\n",
    "#         else:\n",
    "#             move_success = 0\n",
    "\n",
    "    elif action == 4:\n",
    "        # move node caret up\n",
    "        move_attempt = 1\n",
    "        if self.node_caret > 0:\n",
    "            self.node_caret -= 1\n",
    "            move_success = 1\n",
    "        else:\n",
    "            move_success = 0\n",
    "\n",
    "    elif action == 5:\n",
    "        # move node caret down\n",
    "        move_attempt = 1\n",
    "        if self.node_caret < num_nodes(self.matrix)-self.num_inputs-1:\n",
    "            self.node_caret += 1\n",
    "            move_success = 1\n",
    "        else:\n",
    "            move_success = 0\n",
    "\n",
    "#     elif action == 7:\n",
    "#         # randomize node caret\n",
    "#         move_attempt = 1\n",
    "#         prv = self.node_caret\n",
    "#         self.node_caret = rnd.randint(0, num_nodes(self.matrix)-self.num_inputs-1)\n",
    "#         if self.node_caret != prv:\n",
    "#             move_success = 1\n",
    "#         else:\n",
    "#             move_success = 0\n",
    "\n",
    "    elif action == 6:\n",
    "        # add link attempt\n",
    "        struct_attempt = 1\n",
    "        if (rnd.uniform(0,1)<add_link_prob) and (self.newlink_charge <= 0) and add_link(self.matrix, self.caret_row, self.caret_col, \n",
    "                                                                                        self.num_inputs, self.num_outputs, self.max_links, \n",
    "                                                                                        wm = self.weight_matrix):\n",
    "            struct_success = 1\n",
    "            self.newlink_charge = steps_to_charge_newlink\n",
    "        else:\n",
    "            struct_success = 0\n",
    "\n",
    "    elif action == 7:\n",
    "        # remove link attempt\n",
    "        struct_attempt = 1\n",
    "        if (rnd.uniform(0,1)<rem_link_prob) and remove_link(self.matrix, self.caret_row, self.caret_col, \n",
    "                                                            self.num_inputs, self.num_outputs, \n",
    "                                                            wm = self.weight_matrix):\n",
    "            struct_success = 1\n",
    "            # also boost charge by 50%\n",
    "            self.newlink_charge *= 0.5\n",
    "        else:\n",
    "            struct_success = 0\n",
    "\n",
    "    elif action == 8:\n",
    "        # add node attempt\n",
    "        struct_attempt = 1\n",
    "        if (rnd.uniform(0,1)<add_node_prob) and (self.newnode_charge <= 0) and add_node(self.matrix, self.caret_row, self.caret_col, \n",
    "                                                                                        self.num_inputs, self.num_outputs, self.max_links, \n",
    "                                                                                        wm = self.weight_matrix, \n",
    "                                                                                        nm = self.node_matrix):\n",
    "            struct_success = 1\n",
    "            self.newnode_charge = steps_to_charge_newnode\n",
    "        else:\n",
    "            struct_success = 0      \n",
    "\n",
    "    elif action == 9:\n",
    "        # remove node attempt\n",
    "        struct_attempt = 1\n",
    "        if (rnd.uniform(0,1)<rem_link_prob) and remove_node(self.matrix, self.caret_row, \n",
    "                                                            self.num_inputs, self.num_outputs, \n",
    "                                                            wm = self.weight_matrix, \n",
    "                                                            nm = self.node_matrix):\n",
    "            struct_success = 1\n",
    "            # also boost charge by 50%\n",
    "            self.newnode_charge *= 0.5\n",
    "        else:\n",
    "            struct_success = 0\n",
    "\n",
    "    elif action == 10:\n",
    "        # mutate parameter +\n",
    "        modify_attempt = 1\n",
    "        if (rnd.uniform(0,1)<mut_weight_prob) and (self.matrix[self.caret_row,\n",
    "                                                               self.caret_col] == 1):\n",
    "            self.weight_matrix[self.caret_row, self.caret_col] += \\\n",
    "                            rnd.uniform(min_mut_power, max_mut_power)\n",
    "            modify_success = 1\n",
    "        else:\n",
    "            modify_success = 0\n",
    "\n",
    "    elif action == 11:\n",
    "        # mutate parameter -\n",
    "        modify_attempt = 1\n",
    "        if (rnd.uniform(0,1)<mut_weight_prob) and (self.matrix[self.caret_row, \n",
    "                                                               self.caret_col] == 1):\n",
    "            self.weight_matrix[self.caret_row, self.caret_col] -= \\\n",
    "                            rnd.uniform(min_mut_power, max_mut_power)\n",
    "            modify_success = 1\n",
    "        else:\n",
    "            modify_success = 0\n",
    "\n",
    "#     elif action == 14:\n",
    "#         # randomize parameter \n",
    "#         modify_attempt = 1\n",
    "#         if (rnd.uniform(0,1)<replace_weight_prob) and (self.matrix[self.caret_row, \n",
    "#                                                                    self.caret_col] == 1):\n",
    "#             self.weight_matrix[self.caret_row, self.caret_col] = \\\n",
    "#                             rnd.uniform(-max_init_weight, max_init_weight)\n",
    "#             modify_success = 1\n",
    "#         else:\n",
    "#             modify_success = 0\n",
    "\n",
    "    elif action == 12:\n",
    "        # set parameter to 1\n",
    "        modify_attempt = 1\n",
    "        if (rnd.uniform(0,1)<defaultize_weight_prob) and (self.matrix[self.caret_row, \n",
    "                                                                      self.caret_col] == 1):\n",
    "            self.weight_matrix[self.caret_row, self.caret_col] = init_structural_with\n",
    "            modify_success = 1\n",
    "        else:\n",
    "            modify_success = 0\n",
    "\n",
    "    elif action == 13:\n",
    "        # mutate node parameter +\n",
    "        modify_attempt = 1\n",
    "        if np.sum(self.node_matrix[self.node_caret, :])==1:\n",
    "            ocp = argmax(self.node_matrix[self.node_caret, :])\n",
    "            ncp = ocp+1\n",
    "            cp = np.clip(ncp, 0, len(funcs)-1)\n",
    "            self.node_matrix[self.node_caret, :] = 0\n",
    "            self.node_matrix[self.node_caret, cp] = 1\n",
    "            if cp != ocp:\n",
    "                modify_success = 1\n",
    "            else:\n",
    "                modify_success = 0\n",
    "        else:\n",
    "            modify_success = 0\n",
    "\n",
    "    elif action == 14:\n",
    "        # mutate node parameter -\n",
    "        modify_attempt = 1\n",
    "        if np.sum(self.node_matrix[self.node_caret, :])==1:\n",
    "            ocp = argmax(self.node_matrix[self.node_caret, :])\n",
    "            ncp = ocp-1\n",
    "            cp = np.clip(ncp, 0, len(funcs)-1)\n",
    "            self.node_matrix[self.node_caret, :] = 0\n",
    "            self.node_matrix[self.node_caret, cp] = 1\n",
    "            if cp != ocp:\n",
    "                modify_success = 1\n",
    "            else:\n",
    "                modify_success = 0\n",
    "        else:\n",
    "            modify_success = 0\n",
    "\n",
    "#     elif action == 18:\n",
    "#         # randomize node parameter \n",
    "#         modify_attempt = 1\n",
    "#         if np.sum(self.node_matrix[self.node_caret, :])==1:\n",
    "#             ocp = argmax(self.node_matrix[self.node_caret, :])\n",
    "#             ncp = rnd.choice(arange(len(funcs)))\n",
    "#             self.node_matrix[self.node_caret, :] = 0\n",
    "#             self.node_matrix[self.node_caret, ncp] = 1\n",
    "#             if ocp != ncp:\n",
    "#                 modify_success = 1\n",
    "#             else:\n",
    "#                 modify_success = 0\n",
    "#         else:\n",
    "#             modify_success = 0\n",
    "\n",
    "#     elif action == 15:\n",
    "#         # set node parameter to default\n",
    "#         modify_attempt = 1\n",
    "#         if np.sum(self.node_matrix[self.node_caret, :])==1:\n",
    "#             ocp = argmax(self.node_matrix[self.node_caret, :])\n",
    "#             ncp = 0\n",
    "#             self.node_matrix[self.node_caret, :] = 0\n",
    "#             self.node_matrix[self.node_caret, ncp] = 1\n",
    "#             if ocp != ncp:\n",
    "#                 modify_success = 1\n",
    "#             else:\n",
    "#                 modify_success = 0\n",
    "#         else:\n",
    "#             modify_success = 0\n",
    "\n",
    "#         elif action == 14:\n",
    "#             # save state\n",
    "#             self.saved_matrix = np.copy(self.matrix)\n",
    "#             self.saved_weight_matrix = np.copy(self.weight_matrix)\n",
    "#             self.saved_caret = (self.caret_row, self.caret_col)\n",
    "#             success = 1\n",
    "#             modified = False\n",
    "\n",
    "#         elif action == 15:\n",
    "#             # restore state\n",
    "#             self.matrix = np.copy(self.saved_matrix)\n",
    "#             self.weight_matrix = np.copy(self.saved_weight_matrix)\n",
    "#             self.caret_row, self.caret_col = self.saved_caret\n",
    "#             success = 1\n",
    "#             modified = True\n",
    "\n",
    "    self.newnode_charge -= 1\n",
    "    self.newlink_charge -= 1\n",
    "\n",
    "    self.caret_row = clip(self.caret_row, 0, num_nodes(self.matrix)-1)\n",
    "    self.caret_col = clip(self.caret_col, self.num_inputs, num_nodes(self.matrix)-1)\n",
    "    self.node_caret = clip(self.node_caret, 0, num_nodes(self.matrix)-num_inputs-1)\n",
    "    self.weight_matrix = clip(self.weight_matrix, -max_weight, max_weight)\n",
    "\n",
    "    self.newnode_charge = clip(self.newnode_charge, 0, steps_to_charge_newnode)\n",
    "    self.newlink_charge = clip(self.newlink_charge, 0, steps_to_charge_newlink)\n",
    "\n",
    "\n",
    "    info = {}\n",
    "    done = False\n",
    "\n",
    "    # reevaluate if modified\n",
    "    if struct_success or modify_success:\n",
    "        f,fr,facc,faccr = test_xor(self.matrix, self.weight_matrix, self.node_matrix, \n",
    "                     self.num_inputs, self.num_outputs, self.max_nodes)\n",
    "    else:\n",
    "        f,fr,facc,faccr = self.prev_score\n",
    "\n",
    "\n",
    "    if (f - self.best[0]) > 0.0:\n",
    "        reward = (f-self.best[0])*f\n",
    "    else:\n",
    "        reward = 0\n",
    "#    reward = f #- self.best#- self.prev_score\n",
    "\n",
    "\n",
    "    # Soft constraints applied to the reward\n",
    "    if prefer_incoming_links_enabled:\n",
    "        g = nx.DiGraph(self.matrix)\n",
    "        g.remove_nodes_from(list(nx.isolates(g)))\n",
    "        # make sure the remaining nodes will be connected \n",
    "        inps=list(range(self.num_inputs)) \n",
    "        #outs=list(range(self.num_inputs, self.num_inputs+self.num_outputs))\n",
    "        for k in g.nodes():\n",
    "            if (k not in inps):# and (k not in outs):\n",
    "                # must be hidden\n",
    "                if (len(list(g.in_edges(nbunch=k))) != prefer_incoming_links ):\n",
    "                    reward *= 0.2\n",
    "\n",
    "\n",
    "    if min_extra_links_enabled:\n",
    "        if enable_disconnected_inputs:\n",
    "            if np.sum(self.matrix) < 1+min_extra_links:\n",
    "                reward = 0 # force at least one hidden node\n",
    "                f,fr = 0,0\n",
    "        else:\n",
    "            if np.sum(self.matrix) < self.num_inputs+min_extra_links:\n",
    "                reward = 0 # force at least one hidden node\n",
    "                f,fr = 0,0\n",
    "\n",
    "    if penalize_unsuccessful_move:\n",
    "        if move_attempt and (not move_success):\n",
    "            reward *= penalize_unsuccessful_move_mul\n",
    "    if penalize_unsuccessful_modify:\n",
    "        if modify_attempt and (not modify_success):\n",
    "            reward *= penalize_unsuccessful_modify_mul\n",
    "    if penalize_unsuccessful_struct:\n",
    "        if struct_attempt and (not struct_success):\n",
    "            reward *= penalize_unsuccessful_struct_mul\n",
    "\n",
    "    #reward = np.sum(self.matrix) - prev#rnd.uniform(-1,1)\n",
    "    #af = test_xor(self.matrix, self.weight_matrix, \n",
    "    #                 self.num_inputs, self.num_outputs, self.max_nodes)\n",
    "\n",
    "\n",
    "    if (f>self.best[0]) or (self.step_counter % display_every == 0):\n",
    "        clear_output(wait=True)\n",
    "        see_graph(self.matrix, pos=pos, \n",
    "                  wm=self.weight_matrix, nm=self.node_matrix)\n",
    "        plt.show()\n",
    "        print(\"%d steps, %3.4f / %3.4f, best: %3.4f / %3.4f\\nbest ever: %3.4f / %3.4f\" % \n",
    "              (self.step_counter, f, fr, \n",
    "               self.best[0], self.best[1], \n",
    "               self.best_ever[0], self.best_ever[1]))\n",
    "        print('Accuracy scores: %3.4f / %3.4f' % (facc, faccr))\n",
    "        print('Action: %s\\nReward: %3.5f, charge for n/l: %3.4f/%3.4f' \n",
    "              % (action_codes[action], reward, 1.0-self.newnode_charge/steps_to_charge_newnode, \n",
    "                 1.0-self.newlink_charge/steps_to_charge_newlink))\n",
    "        print('Caret row/col: %d/%d' % (self.caret_row, self.caret_col))\n",
    "\n",
    "        if display_matrix:\n",
    "            sm = np.copy(self.matrix)\n",
    "            sm[self.caret_row, self.caret_col] = 666\n",
    "            print(sm[0:num_nodes(self.matrix), \n",
    "                     self.num_inputs:num_nodes(self.matrix)])\n",
    "            #print(sm)\n",
    "        if display_weight_matrix:\n",
    "            print(self.weight_matrix[0:num_nodes(self.matrix), \n",
    "                     self.num_inputs:num_nodes(self.matrix)])\n",
    "            #print(self.weight_matrix)\n",
    "\n",
    "        if display_sleep > 0:\n",
    "            time.sleep(display_sleep)\n",
    "\n",
    "    if f > self.best[0]:\n",
    "        self.best = f,fr\n",
    "        self.stag = 0\n",
    "    else:\n",
    "        self.stag += 1\n",
    "\n",
    "    if f > self.best_ever[0]:\n",
    "        self.best_ever = f,fr            \n",
    "\n",
    "    if (self.step_counter > max_steps) or (self.stag > max_stag):\n",
    "        done = True\n",
    "\n",
    "    observation = make_input(self.matrix, self.weight_matrix, self.node_matrix,\n",
    "                             self.caret_row, self.caret_col, self.node_caret,\n",
    "                             self.max_nodes, self.max_links, \n",
    "                             self.num_inputs, self.num_outputs, \n",
    "                             la=float(move_success or modify_success or struct_success),\n",
    "                             ls=f, \n",
    "                             nnc=1.0-self.newnode_charge/steps_to_charge_newnode,\n",
    "                             nlc=1.0-self.newlink_charge/steps_to_charge_newlink, )\n",
    "\n",
    "    self.prev_score = f,fr,facc,faccr\n",
    "    self.prev_reward = reward\n",
    "\n",
    "    return observation, reward, done, info\n",
    "\n",
    "GraphMakerEnv.step = step"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "#### reset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def reset(self):\n",
    "\n",
    "    print('Env reset')\n",
    "    self.step_counter = 0\n",
    "    self.best = 0,0\n",
    "    self.stag = 0\n",
    "\n",
    "    self.matrix = zeros((self.max_nodes, self.max_nodes))\n",
    "    self.weight_matrix = zeros((self.max_nodes, self.max_nodes))\n",
    "    self.node_matrix = zeros((self.max_nodes-self.num_inputs, len(funcs)))\n",
    "    if enable_disconnected_inputs:\n",
    "        cn = rnd.randint(0,self.num_inputs-1)\n",
    "        self.matrix[cn, \n",
    "                    self.num_inputs:self.num_inputs+self.num_outputs] = 1\n",
    "        if enforce_bias_link:\n",
    "            self.matrix[self.num_inputs-1, \n",
    "                        self.num_inputs:self.num_inputs+self.num_outputs] = 1 # bias\n",
    "    else:\n",
    "        self.matrix[0:self.num_inputs, \n",
    "                    self.num_inputs:self.num_inputs+self.num_outputs] = 1\n",
    "\n",
    "    for i in range(self.max_nodes):\n",
    "        for j in range(self.max_nodes):\n",
    "            if self.matrix[i,j]==1:\n",
    "                if random_init_structural:\n",
    "                    self.weight_matrix[i,j] = rnd.uniform(-max_init_weight,max_init_weight)\n",
    "                else:\n",
    "                    self.weight_matrix[i,j] = init_structural_with\n",
    "    for i in range(self.num_inputs, num_nodes(self.matrix)):\n",
    "        self.node_matrix[i-self.num_inputs, rnd.choice(arange(len(funcs)))]=1                    \n",
    "\n",
    "    f,fr, facc, faccr = test_xor(self.matrix, self.weight_matrix, self.node_matrix,\n",
    "                     self.num_inputs, self.num_outputs, self.max_nodes)\n",
    "\n",
    "    self.caret_row = rnd.randint(0, self.num_inputs-1)\n",
    "    self.caret_col = self.num_inputs\n",
    "    self.node_caret = 0\n",
    "\n",
    "    # these increase and when threshold is hit, then it can add new links/nodes\n",
    "    self.newnode_charge = 0\n",
    "    self.newlink_charge = 0\n",
    "\n",
    "    self.prev_score = f,fr,facc, faccr\n",
    "    self.prev_reward = 0\n",
    "\n",
    "    self.saved_matrix = np.copy(self.matrix)\n",
    "    self.saved_weight_matrix = np.copy(self.weight_matrix)\n",
    "    self.saved_caret = (self.caret_row, self.caret_col)\n",
    "\n",
    "    observation = make_input(self.matrix, self.weight_matrix, self.node_matrix,\n",
    "                             self.caret_row, self.caret_col, self.node_caret,\n",
    "                             self.max_nodes, self.max_links, \n",
    "                             self.num_inputs, self.num_outputs, \n",
    "                             la=1, ls=0, nnc=1.0, nlc=1.0)\n",
    "    return observation\n",
    "\n",
    "GraphMakerEnv.reset = reset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "#%debug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# multiprocess environment or not\n",
    "if 0:\n",
    "    n_cpu = 8\n",
    "    env = SubprocVecEnv([lambda: GraphMakerEnv(max_nodes, max_links, num_inputs, num_outputs) for i in range(n_cpu)])\n",
    "else:\n",
    "    env = GraphMakerEnv(max_nodes, max_links, num_inputs, num_outputs)\n",
    "    env = DummyVecEnv([lambda: env])\n",
    "\n",
    "t = 0\n",
    "\n",
    "# Custom MLP policy of two layers of size 32 each with tanh activation function\n",
    "policy_kwargs = dict(act_fun=tf.nn.relu, net_arch=[#'lstm', \n",
    "                                                   32, #256, \n",
    "                                                   16, #128\n",
    "                                                  ])\n",
    "\n",
    "tblog = 'C:/Users/spook/Dropbox/Laptop/tblog'\n",
    "    \n",
    "[shutil.rmtree(tblog+'/'+x) for x in os.listdir(tblog) if x and ('_' in x)]\n",
    "time.sleep(10)\n",
    "\n",
    "\n",
    "# model = PPO2(MlpPolicy, env, n_steps=128, \n",
    "#              verbose=0, nminibatches=1, \n",
    "#  #            policy_kwargs=policy_kwargs, \n",
    "#              gamma=0.99,#0.99,\n",
    "#              ent_coef=0.01,#0.01,\n",
    "#              learning_rate=0.0005,\n",
    "#              vf_coef=0.5,\n",
    "#              max_grad_norm=0.5,\n",
    "#              lam=0.95,\n",
    "#              tensorboard_log=tblog)\n",
    "\n",
    "# model = ACKTR(MlpPolicy, env, verbose=0, #n_steps=5, \n",
    "# #              policy_kwargs=policy_kwargs, \n",
    "#               tensorboard_log='/home/peter/tblog')\n",
    "\n",
    "model = A2C(MlpPolicy, env, verbose=0, n_steps=64, \n",
    "#              policy_kwargs=policy_kwargs, \n",
    "              tensorboard_log=tblog)\n",
    "\n",
    "# model = TRPO(MlpPolicy, env, verbose=0,# n_steps=50, \n",
    "#               #policy_kwargs=policy_kwargs, \n",
    "#               tensorboard_log=tblog)\n",
    "\n",
    "# model = ACER(MlpPolicy, env, verbose=0,# n_steps=50, \n",
    "#               #policy_kwargs=policy_kwargs, \n",
    "#               tensorboard_log=tblog)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "#%%prun\n",
    "\n",
    "try:\n",
    "    model.learn(total_timesteps=100000000,\n",
    "                log_interval=10)\n",
    "except KeyboardInterrupt:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "m,wm = env.get_attr('matrix')[0], env.get_attr('weight_matrix')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "see_graph(m);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "np.sum(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# mytest = bdx_val\n",
    "# testPredictions = ogp.get_predictions(bdx_val, bdy_val, ensemble)\n",
    "# pdtest = pd.DataFrame({'PassengerId': bdx_test_df.PassengerId.astype(int),\n",
    "#                         'Survived': testPredictions.astype(int)})\n",
    "# pdtest.to_csv('titanic_gp.csv', index=False) # make submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc-autonumbering": false,
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}