TAGCN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/krohak/anaconda3/lib/python3.6/importlib/_bootstrap.py:205: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "flags = tf.app.flags\n",
    "FLAGS = flags.FLAGS\n",
    "flags.DEFINE_string('dataset', 'cora', 'Dataset string.')  # 'cora', 'citeseer', 'pubmed'\n",
    "flags.DEFINE_string('model', 'gcn', 'Model string.')  # 'gcn', 'gcn_cheby', 'dense'\n",
    "flags.DEFINE_float('learning_rate', 0.01, 'Initial learning rate.')\n",
    "flags.DEFINE_integer('epochs', 200, 'Number of epochs to train.')\n",
    "flags.DEFINE_integer('hidden1', 16, 'Number of units in hidden layer 1.')\n",
    "flags.DEFINE_float('dropout', 0.5, 'Dropout rate (1 - keep probability).')\n",
    "flags.DEFINE_float('weight_decay', 5e-4, 'Weight for L2 loss on embedding matrix.')\n",
    "flags.DEFINE_integer('early_stopping', 10, 'Tolerance for early stopping (# of epochs).')\n",
    "flags.DEFINE_integer('max_degree', 3, 'Maximum Chebyshev polynomial degree.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask = load_data(FLAGS.dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "adj = normalize_adj(adj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "adj = adj.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(adj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "G = nx.Graph(adj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "paths = nx.all_simple_paths(G, source=0, target=138,cutoff=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paths=[p for p in paths]\n",
    "len(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2581988897471611"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adj[1][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([[0, 633, 1701, 1799, 306, 138],\n",
       "  [0, 633, 1701, 1856, 306, 138],\n",
       "  [0, 1862, 1701, 1799, 306, 138],\n",
       "  [0, 1862, 1701, 1856, 306, 138]],\n",
       " 0.33333333333333331,\n",
       " 0.06711560552140243,\n",
       " 0.058123819371909642,\n",
       " 0.056613851707229788,\n",
       " 0.065372045046061339)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paths, adj[633][0], adj[633][1701], adj[1701][1799], adj[1799][306], adj[306][138]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path_weight = 0\n",
    "for path in paths:\n",
    "    i = 1\n",
    "    weight = 1\n",
    "    while i < len(path):\n",
    "        weight *= adj[path[i-1]][path[i]]\n",
    "        i+=1\n",
    "    path_weight += weight   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.5159390159390157e-05"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def memoize(f):\n",
    "    memo={}\n",
    "\n",
    "    def helper(g,x,y,z):\n",
    "        if (x,y,z) not in memo:\n",
    "            if (y,x,z) not in memo:\n",
    "                memo[(x,y,z)]=f(g,i=x,j=y,cutoff=z)\n",
    "                #print(memo)\n",
    "                return memo[(x,y,z)]\n",
    "            \n",
    "            return memo[(y,x,z)]\n",
    "        return memo[(x,y,z)]\n",
    "\n",
    "    return helper\n",
    "\n",
    "@memoize\n",
    "def calculate_path_weight(G, i, j, cutoff):\n",
    "    paths = nx.all_simple_paths(G, source=i, target=j,cutoff=cutoff)\n",
    "    \n",
    "    path_weight = 0\n",
    "    for path in paths:\n",
    "        i = 1\n",
    "        weight = 1\n",
    "        while i < len(path):\n",
    "            weight *= adj[path[i-1]][path[i]]\n",
    "            i+=1\n",
    "        path_weight += weight  \n",
    "    \n",
    "    return path_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path_w = calculate_path_weight(G,0,138,5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.5159390159390157e-05"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.,  1.,  1., ...,  1.,  1.,  1.], dtype=float32)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features.toarray().max(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def preprocess_features(features):\n",
    "    \"\"\"Row-normalize feature matrix and convert to tuple representation\"\"\"\n",
    "    rowsum = np.array(features.sum(1))\n",
    "    r_inv = np.power(rowsum, -1).flatten()\n",
    "    r_inv[np.isinf(r_inv)] = 0.\n",
    "    r_mat_inv = sp.diags(r_inv)\n",
    "    features = r_mat_inv.dot(features)\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = preprocess_features(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.05555556,  0.08333334,  0.11111111, ...,  0.14285715,\n",
       "        0.11111111,  0.07142857], dtype=float32)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f.toarray().max(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((2708, 1433), (2708, 1433))"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f.shape, features.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating the weight matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "G = nx.Graph(adj)\n",
    "#G.add_path([0,1,2])\n",
    "#list(nx.bfs_edges([0,1,2],0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "paths = nx.all_simple_paths(G, source=0, target=1378,cutoff=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "paths=[p for p in paths]\n",
    "len(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def memoize(f):\n",
    "    memo={}\n",
    "\n",
    "    def helper(g,x,y,z):\n",
    "        if (x,y,z) not in memo:\n",
    "            if (y,x,z) not in memo:\n",
    "                memo[(x,y,z)]=f(g,i=x,j=y,cutoff=z)\n",
    "                #print(memo)\n",
    "                return memo[(x,y,z)]\n",
    "            \n",
    "            return memo[(y,x,z)]\n",
    "        return memo[(x,y,z)]\n",
    "\n",
    "    return helper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "@memoize\n",
    "def calculate_path_weight(G, i, j, cutoff):\n",
    "    \"\"\"https://networkx.github.io/documentation/networkx-1.10/reference/generated/networkx.algorithms.simple_paths.all_simple_paths.html\"\"\"\n",
    "    # put G = nx.Graph(adj) before calling the function\n",
    "    paths = nx.all_simple_paths(G, source=i, target=j,cutoff=cutoff)\n",
    "    paths=[p for p in paths]\n",
    "    return len(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path_len = calculate_path_weight(G,0,1378,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lets create Weight matrix (k=1 and k=2) for i=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "w_k1 = []\n",
    "w_k2 = []\n",
    "for j in range(features.shape[0]):\n",
    "    w_k1.append(calculate_path_weight(G,0,j,1))\n",
    "    w_k2.append(calculate_path_weight(G,0,j,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "w_k1=np.asarray(w_k1,np.float32)\n",
    "w_k2=np.asarray(w_k2,np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "w_k1 = w_k1.reshape([w_k1.shape[0],1])\n",
    "w_k2 = w_k2.reshape([w_k2.shape[0],1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "w_k1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "w_k2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(w_k1*w_k2).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tensor = tf.matmul(w_k1.T,w_k2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    print(tensor.eval())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lets multiply it by the feature vector, Xc for c = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#features = preprocess_features(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f=features.todense()[0:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s = w_k1.T*f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ".\n",
    "\n",
    "\n",
    "\n",
    "."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SECTION 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ".\n",
    "multi-filter:\n",
    "\n",
    "https://github.com/krohak/CADL/blob/master/session-3/lecture-3.ipynb\n",
    "\n",
    "https://github.com/krohak/CADL/blob/master/session-3/session-3.ipynb\n",
    "\n",
    "https://github.com/krohak/CADL/blob/master/session-3/libs/utils.py\n",
    "\n",
    "optimizer:\n",
    "\n",
    "https://github.com/krohak/CADL/blob/master/session-2/lecture-2.ipynb\n",
    "\n",
    "https://github.com/krohak/CADL/blob/master/session-2/session-2.ipynb\n",
    "."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "var_gs = {}\n",
    "name = '01'\n",
    "Fl = 8\n",
    "Kl = 2\n",
    "Cl = features.shape[1]\n",
    "Nl = features.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from inits import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope( name + '_vars'):\n",
    "        for k in range(Kl):\n",
    "                for c in range(Cl):\n",
    "                    var_gs['weights_' + str(k+1) + '_' + str(c+1)] = glorot([1,Fl],\n",
    "                                                            name=('weights_' + str(k+1) + '_' + str(c+1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "var_gs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To get a list of current nodes in graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "[n.name for n in tf.get_default_graph().as_graph_def().node]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Clear the graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "var_gs = {}\n",
    "name = '01'\n",
    "Fl = 8\n",
    "Kl = 2\n",
    "Cl = features.shape[1]\n",
    "Nl = features.shape[0]\n",
    "from tensorflow.python.framework.ops import reset_default_graph\n",
    "reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " '01_vars/strided_slice/stack',\n",
       " '01_vars/strided_slice/stack_1',\n",
       " '01_vars/strided_slice/stack_2',\n",
       " '01_vars/strided_slice',\n",
       " '01_vars/MatMul',\n",
       " '01_vars/G_k_0/Initializer/random_uniform/shape',\n",
       " '01_vars/G_k_0/Initializer/random_uniform/min',\n",
       " '01_vars/G_k_0/Initializer/random_uniform/max',\n",
       " '01_vars/G_k_0/Initializer/random_uniform/RandomUniform',\n",
       " '01_vars/G_k_0/Initializer/random_uniform/sub',\n",
       " '01_vars/G_k_0/Initializer/random_uniform/mul',\n",
       " '01_vars/G_k_0/Initializer/random_uniform',\n",
       " '01_vars/G_k_0',\n",
       " '01_vars/G_k_0/Assign',\n",
       " '01_vars/G_k_0/read',\n",
       " '01_vars/MatMul_1',\n",
       " '01_vars/Add/x',\n",
       " '01_vars/Add',\n",
       " '01_vars/strided_slice_1/stack',\n",
       " '01_vars/strided_slice_1/stack_1',\n",
       " '01_vars/strided_slice_1/stack_2',\n",
       " '01_vars/strided_slice_1',\n",
       " '01_vars/MatMul_2',\n",
       " '01_vars/G_k_1/Initializer/random_uniform/shape',\n",
       " '01_vars/G_k_1/Initializer/random_uniform/min',\n",
       " '01_vars/G_k_1/Initializer/random_uniform/max',\n",
       " '01_vars/G_k_1/Initializer/random_uniform/RandomUniform',\n",
       " '01_vars/G_k_1/Initializer/random_uniform/sub',\n",
       " '01_vars/G_k_1/Initializer/random_uniform/mul',\n",
       " '01_vars/G_k_1/Initializer/random_uniform',\n",
       " '01_vars/G_k_1',\n",
       " '01_vars/G_k_1/Assign',\n",
       " '01_vars/G_k_1/read',\n",
       " '01_vars/MatMul_3',\n",
       " '01_vars/Add_1',\n",
       " 'zeros',\n",
       " 'bias',\n",
       " 'bias/Assign',\n",
       " 'bias/read',\n",
       " 'Relu',\n",
       " '01_vars_1/strided_slice/stack',\n",
       " '01_vars_1/strided_slice/stack_1',\n",
       " '01_vars_1/strided_slice/stack_2',\n",
       " '01_vars_1/strided_slice',\n",
       " '01_vars_1/MatMul',\n",
       " '01_vars/G_k2_0/Initializer/random_uniform/shape',\n",
       " '01_vars/G_k2_0/Initializer/random_uniform/min',\n",
       " '01_vars/G_k2_0/Initializer/random_uniform/max',\n",
       " '01_vars/G_k2_0/Initializer/random_uniform/RandomUniform',\n",
       " '01_vars/G_k2_0/Initializer/random_uniform/sub',\n",
       " '01_vars/G_k2_0/Initializer/random_uniform/mul',\n",
       " '01_vars/G_k2_0/Initializer/random_uniform',\n",
       " '01_vars/G_k2_0',\n",
       " '01_vars/G_k2_0/Assign',\n",
       " '01_vars/G_k2_0/read',\n",
       " '01_vars_1/MatMul_1',\n",
       " '01_vars_1/Add',\n",
       " '01_vars_1/strided_slice_1/stack',\n",
       " '01_vars_1/strided_slice_1/stack_1',\n",
       " '01_vars_1/strided_slice_1/stack_2',\n",
       " '01_vars_1/strided_slice_1',\n",
       " '01_vars_1/MatMul_2',\n",
       " '01_vars/G_k2_1/Initializer/random_uniform/shape',\n",
       " '01_vars/G_k2_1/Initializer/random_uniform/min',\n",
       " '01_vars/G_k2_1/Initializer/random_uniform/max',\n",
       " '01_vars/G_k2_1/Initializer/random_uniform/RandomUniform',\n",
       " '01_vars/G_k2_1/Initializer/random_uniform/sub',\n",
       " '01_vars/G_k2_1/Initializer/random_uniform/mul',\n",
       " '01_vars/G_k2_1/Initializer/random_uniform',\n",
       " '01_vars/G_k2_1',\n",
       " '01_vars/G_k2_1/Assign',\n",
       " '01_vars/G_k2_1/read',\n",
       " '01_vars_1/MatMul_3',\n",
       " '01_vars_1/Add_1',\n",
       " 'zeros_1',\n",
       " 'bias_1',\n",
       " 'bias_1/Assign',\n",
       " 'bias_1/read',\n",
       " 'Relu_1',\n",
       " 'zeros_2',\n",
       " 'bias_2',\n",
       " 'bias_2/Assign',\n",
       " 'bias_2/read',\n",
       " 'Relu_2',\n",
       " '01_vars_2/strided_slice/stack',\n",
       " '01_vars_2/strided_slice/stack_1',\n",
       " '01_vars_2/strided_slice/stack_2',\n",
       " '01_vars_2/strided_slice',\n",
       " '01_vars_2/MatMul',\n",
       " '01_vars_3/strided_slice/stack',\n",
       " '01_vars_3/strided_slice/stack_1',\n",
       " '01_vars_3/strided_slice/stack_2',\n",
       " '01_vars_3/strided_slice',\n",
       " '01_vars_3/MatMul',\n",
       " '01_vars/G_k3_1/Initializer/random_uniform/shape',\n",
       " '01_vars/G_k3_1/Initializer/random_uniform/min',\n",
       " '01_vars/G_k3_1/Initializer/random_uniform/max',\n",
       " '01_vars/G_k3_1/Initializer/random_uniform/RandomUniform',\n",
       " '01_vars/G_k3_1/Initializer/random_uniform/sub',\n",
       " '01_vars/G_k3_1/Initializer/random_uniform/mul',\n",
       " '01_vars/G_k3_1/Initializer/random_uniform',\n",
       " '01_vars/G_k3_1',\n",
       " '01_vars/G_k3_1/Assign',\n",
       " '01_vars/G_k3_1/read',\n",
       " '01_vars_3/MatMul_1',\n",
       " 'zeros_3',\n",
       " 'bias_3',\n",
       " 'bias_3/Assign',\n",
       " 'bias_3/read',\n",
       " 'Relu_3',\n",
       " 'init']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g = tf.get_default_graph()\n",
    "[op.name for op in g.get_operations()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#s = tf.constant(s[0,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "t = s[0,0]*var_gs['weights_' + str(1) + '_' + str(1)]\n",
    "z = s[0,0]*var_gs['weights_' + str(1) + '_' + str(1)]\n",
    "#with sess.as_default():\n",
    "#t.eval()\n",
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "sess.run((t,z,t+z))\n",
    "type(t.eval(session=sess))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def dot(x, y, sparse=False):\n",
    "    \"\"\"Wrapper for tf.matmul (sparse vs dense).\"\"\"\n",
    "    if sparse:\n",
    "        res = tf.sparse_tensor_dense_matmul(x, y)\n",
    "    else:\n",
    "        res = tf.matmul(x, y)\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dot(s[0,0], var_gs['weights_' + str(1) + '_' + str(1)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ".\n",
    "\n",
    "\n",
    "\n",
    "."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "G = nx.Graph(adj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def memoize(f):\n",
    "    memo={}\n",
    "\n",
    "    def helper(g,x,y,z):\n",
    "        if (x,y,z) not in memo:\n",
    "            if (y,x,z) not in memo:\n",
    "                p=f(g,i=x,j=y,cutoff=z)\n",
    "                memo[(x,y,z)]=p\n",
    "                print(x,y,p)\n",
    "                return memo[(x,y,z)]\n",
    "            \n",
    "            return memo[(y,x,z)]\n",
    "        return memo[(x,y,z)]\n",
    "\n",
    "    return helper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "@memoize\n",
    "def calculate_path_weight(G, i, j, cutoff):\n",
    "    \"\"\"https://networkx.github.io/documentation/networkx-1.10/reference/generated/networkx.algorithms.simple_paths.all_simple_paths.html\"\"\"\n",
    "    # put G = nx.Graph(adj) before calling the function\n",
    "    paths = nx.all_simple_paths(G, source=i, target=j,cutoff=cutoff)\n",
    "    paths=[p for p in paths]\n",
    "    return len(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range(features.shape[0]):\n",
    "    for k in range(1,Kl):\n",
    "        for j in range(features.shape[0]):\n",
    "            calculate_path_weight(G,i,j,k)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ".\n",
    "\n",
    "\n",
    "\n",
    "."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a = np.array([1, 2, 3])\n",
    "b = np.array([2, 3, 4])\n",
    "a = np.vstack((a,b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a = np.vstack((a,a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 2, 3],\n",
       "       [2, 3, 4],\n",
       "       [1, 2, 3],\n",
       "       [2, 3, 4]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SECTION 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/krohak/anaconda3/lib/python3.6/importlib/_bootstrap.py:205: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from utils import *\n",
    "from metrics import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "flags = tf.app.flags\n",
    "FLAGS = flags.FLAGS\n",
    "flags.DEFINE_string('dataset', 'cora', 'Dataset string.')  # 'cora', 'citeseer', 'pubmed'\n",
    "flags.DEFINE_string('model', 'gcn', 'Model string.')  # 'gcn', 'gcn_cheby', 'dense'\n",
    "flags.DEFINE_float('learning_rate', 0.01, 'Initial learning rate.')\n",
    "flags.DEFINE_integer('epochs', 200, 'Number of epochs to train.')\n",
    "flags.DEFINE_integer('hidden1', 16, 'Number of units in hidden layer 1.')\n",
    "flags.DEFINE_float('dropout', 0.5, 'Dropout rate (1 - keep probability).')\n",
    "flags.DEFINE_float('weight_decay', 5e-4, 'Weight for L2 loss on embedding matrix.')\n",
    "flags.DEFINE_integer('early_stopping', 10, 'Tolerance for early stopping (# of epochs).')\n",
    "flags.DEFINE_integer('max_degree', 3, 'Maximum Chebyshev polynomial degree.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask = load_data(FLAGS.dataset)\n",
    "#features = preprocess_features(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = features.todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path_weight_matrix = np.load(\"path_weights.dat\")\n",
    "path_weight_matrix.shape\n",
    "path_weight_matrix = path_weight_matrix.astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2708,)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_weight_matrix[1,:,1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(path_weight_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(dtype('float32'), (2708, 2708, 2))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f=features\n",
    "output=[]\n",
    "output = np.asarray(output,dtype=np.float32)\n",
    "path_weight_matrix.dtype, path_weight_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2708, 1433)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g = tf.get_default_graph()\n",
    "[op.name for op in g.get_operations()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from inits import *\n",
    "var_gs = {}\n",
    "name = '01'\n",
    "Fl = 8\n",
    "Kl = 2\n",
    "Cl = features.shape[1]\n",
    "Nl = features.shape[0]\n",
    "from tensorflow.python.framework.ops import reset_default_graph\n",
    "reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define wieghts, biases\n",
    "\n",
    "vars_F={}\n",
    "\n",
    "with tf.variable_scope(name + '_vars'):\n",
    "            \n",
    "    for k in range(Kl):\n",
    "\n",
    "        vars_F['weights_' + str(0) + '_' + str(k)] = tf.get_variable(name=('weights_' + str(0) + '_' + str(k)),\n",
    "                                                                     initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                                                                     shape=[Cl,Fl])\n",
    "    initial = tf.zeros([Nl,Fl], dtype=tf.float32)\n",
    "    vars_F['bias_' + str(0)] = tf.Variable(initial, name='bias')\n",
    "\n",
    "\n",
    "    for k in range(Kl):\n",
    "\n",
    "        vars_F['weights_' + str(1) + '_' + str(k)] = tf.get_variable(name=('weights_' + str(1) + '_' + str(k)),\n",
    "                                                                     initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                                                                     shape=[Fl,Fl])\n",
    "    initial = tf.zeros([Nl,Fl], dtype=tf.float32)\n",
    "    vars_F['bias_' + str(1)] = tf.Variable(initial, name='bias')\n",
    "    \n",
    "    \n",
    "\n",
    "\n",
    "    vars_F['weights_' + str(2)] = tf.get_variable(name=('weights_' + str(2)),\n",
    "                                                                     initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                                                                     shape=[Fl,7])\n",
    "    initial = tf.zeros([Nl,7], dtype=tf.float32)\n",
    "    vars_F['bias_' + str(2)] = tf.Variable(initial, name='bias')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bias_0': <tf.Variable '01_vars/bias:0' shape=(2708, 8) dtype=float32_ref>,\n",
       " 'bias_1': <tf.Variable '01_vars/bias_1:0' shape=(2708, 8) dtype=float32_ref>,\n",
       " 'bias_2': <tf.Variable '01_vars/bias_2:0' shape=(2708, 7) dtype=float32_ref>,\n",
       " 'weights_0_0': <tf.Variable '01_vars/weights_0_0:0' shape=(1433, 8) dtype=float32_ref>,\n",
       " 'weights_0_1': <tf.Variable '01_vars/weights_0_1:0' shape=(1433, 8) dtype=float32_ref>,\n",
       " 'weights_1_0': <tf.Variable '01_vars/weights_1_0:0' shape=(8, 8) dtype=float32_ref>,\n",
       " 'weights_1_1': <tf.Variable '01_vars/weights_1_1:0' shape=(8, 8) dtype=float32_ref>,\n",
       " 'weights_2': <tf.Variable '01_vars/weights_2:0' shape=(8, 7) dtype=float32_ref>}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vars_F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2708, 8)\n",
      "(2708, 8)\n"
     ]
    }
   ],
   "source": [
    "# features_m = tf.sparse_placeholder(tf.float32, shape=[features[2][0], features[2][1]])\n",
    "features_m = tf.placeholder(tf.float32, shape=[features.shape[0], features.shape[1]])\n",
    "pwm = tf.placeholder(tf.float32, [Nl, Nl, Kl])\n",
    "outputs=[]\n",
    "\n",
    "conv = np.zeros([Nl,Fl],dtype=np.float32)\n",
    "#tf.placeholder(tf.float32, shape=[Nl,Fl])\n",
    "\n",
    "# droupout\n",
    "features_m = tf.nn.dropout(features_m, 1-FLAGS.dropout)\n",
    "\n",
    "with tf.variable_scope( name + '_vars'):    \n",
    "     \n",
    "    for k in range(Kl):\n",
    "        \n",
    "        w_k = pwm[:,:,k]\n",
    "        \n",
    "#         s = tf.sparse_tensor_dense_matmul(sp_a=features_m, b=w_k)\n",
    "        s = tf.matmul(w_k,features_m)\n",
    "    \n",
    "        G_k = vars_F['weights_' + str(0) + '_' + str(k)]\n",
    "        \n",
    "        res = tf.matmul(s,G_k)\n",
    "        \n",
    "        outputs.append(res)\n",
    "        \n",
    "        conv = tf.add(conv,res)\n",
    "        \n",
    "        print(conv.shape)\n",
    "    \n",
    "        outputs.append(conv)\n",
    "    \n",
    "# add bias\n",
    "bias = vars_F['bias_' + str(0)]\n",
    "\n",
    "conv = tf.add(conv,bias)\n",
    "\n",
    "# apply non-linearity\n",
    "conv = tf.nn.relu(conv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2708, 8)\n",
      "(2708, 8)\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope( name + '_vars'):    \n",
    "     \n",
    "    for k in range(Kl):\n",
    "        \n",
    "        w_k = pwm[:,:,k]\n",
    "        \n",
    "        s = tf.matmul(w_k,conv)\n",
    "    \n",
    "        G_k = vars_F['weights_' + str(1) + '_' + str(k)]\n",
    "        \n",
    "        res = tf.matmul(s,G_k)\n",
    "        \n",
    "        outputs.append(res)\n",
    "        \n",
    "        conv = tf.add(conv,res)\n",
    "        \n",
    "        print(conv.shape)\n",
    "    \n",
    "        outputs.append(conv)\n",
    "\n",
    "# add bias\n",
    "bias = vars_F['bias_' + str(1)]\n",
    "\n",
    "conv = tf.add(conv,bias)\n",
    "\n",
    "# apply non-linearity\n",
    "conv = tf.nn.relu(conv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2708, 7)\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope( name + '_vars'):    \n",
    "             \n",
    "    w_k = pwm[:,:,k]\n",
    "\n",
    "    s = tf.matmul(w_k,conv)\n",
    "\n",
    "    G_k = vars_F['weights_' + str(2)]\n",
    "\n",
    "    conv = tf.matmul(s,G_k)\n",
    "    \n",
    "\n",
    "print(conv.shape)    \n",
    "# add bias\n",
    "bias = vars_F['bias_' + str(2)]\n",
    "conv = tf.add(conv,bias)\n",
    "\n",
    "\n",
    "# apply non-linearity\n",
    "conv = tf.nn.relu(conv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor '01_vars_1/MatMul_1:0' shape=(2708, 8) dtype=float32>,\n",
       " <tf.Tensor '01_vars_1/Add:0' shape=(2708, 8) dtype=float32>,\n",
       " <tf.Tensor '01_vars_1/MatMul_3:0' shape=(2708, 8) dtype=float32>,\n",
       " <tf.Tensor '01_vars_1/Add_1:0' shape=(2708, 8) dtype=float32>,\n",
       " <tf.Tensor '01_vars_2/MatMul_1:0' shape=(2708, 8) dtype=float32>,\n",
       " <tf.Tensor '01_vars_2/Add:0' shape=(2708, 8) dtype=float32>,\n",
       " <tf.Tensor '01_vars_2/MatMul_3:0' shape=(2708, 8) dtype=float32>,\n",
       " <tf.Tensor '01_vars_2/Add_1:0' shape=(2708, 8) dtype=float32>]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#conv = tf.nn.softmax(conv)\n",
    "\n",
    "accuracy = masked_accuracy(conv,y_train, train_mask)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2708, 7)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# one hot encoded\n",
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss= 0\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate)\n",
    "\n",
    "loss += masked_softmax_cross_entropy(conv, y_train, train_mask)\n",
    "\n",
    "opt_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "evals = sess.run([opt_op,accuracy,loss],feed_dict={features_m:features,pwm:path_weight_matrix})\n",
    "#type(t.eval(session=sess))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.17142858, 546057.5)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evals[1], evals[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for epoch in range(FLAGS.epochs):\n",
    "    \n",
    "    evals = sess.run([opt_op,accuracy,loss],feed_dict={features_m:features,pwm:path_weight_matrix})\n",
    "    print(evals[1], evals[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((2708, 7), (2708, 7), (2708, 7), (2708, 1433), (2708,), (2708,), (2708,))"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.shape, y_train.shape, y_val.shape, features.shape, train_mask.shape, test_mask.shape, val_mask.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Counter({False: 2568, True: 140}),\n",
       " Counter({False: 2208, True: 500}),\n",
       " Counter({False: 1708, True: 1000}))"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "Counter(train_mask), Counter(val_mask), Counter(test_mask)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reusing variables:\n",
    "https://www.tensorflow.org/versions/r0.12/how_tos/variable_scope/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f=features.todense()\n",
    "output=[]\n",
    "output = np.asarray(output,dtype=np.float32)\n",
    "with tf.variable_scope( name + '_vars') as scope:\n",
    "    scope.reuse_variables()\n",
    "    for i in range(features.shape[0]):\n",
    "        conv = tf.get_variable(\"conv\",shape=())\n",
    "        for k in range(Kl):\n",
    "            for c in range(Cl):\n",
    "                var_gs['weights_' + str(k+1) + '_' + str(c+1)] = glorot([1,Fl],name=('weights_' + str(k+1) + '_' + str(c+1)))\n",
    "                w_k = []\n",
    "                w_k = path_weight_matrix[i,:,k]\n",
    "                w_k = np.asarray(w_k,np.float32)\n",
    "                w_k = w_k.reshape([w_k.shape[0],1])\n",
    "                x_c = f[0:,c]\n",
    "                s = tf.matmul(w_k.T,x_c)\n",
    "                conv = tf.add(conv,tf.multiply(s[0,0],var_gs['weights_' + str(k+1) + '_' + str(c+1)]))  \n",
    "        print(conv)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SECTION 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ".\n",
    "\n",
    "Installing TF with GPU:\n",
    "https://www.tensorflow.org/install/install_linux\n",
    "\n",
    "Check available GPUs on TF:\n",
    "https://stackoverflow.com/questions/38559755/how-to-get-current-available-gpus-in-tensorflow\n",
    "\n",
    "Change port of SSH:\n",
    "https://wiki.archlinux.org/index.php/Secure_Shell\n",
    "\n",
    "Check GPU info:\n",
    "https://askubuntu.com/questions/5417/how-to-get-the-gpu-info\n",
    "\n",
    "Distributed TF:\n",
    "https://www.tensorflow.org/deploy/distributed\n",
    "\n",
    "."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Xavier (Glorot) initializer: https://www.tensorflow.org/api_docs/python/tf/contrib/layers/xavier_initializer\n",
    "\n",
    "Mutmul exceptions: https://stackoverflow.com/questions/34908033/tensorflow-exception-with-matmul\n",
    "\n",
    "tf.Variable: https://www.tensorflow.org/api_docs/python/tf/Variable\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ".\n",
    "\n",
    "\n",
    "\n",
    "."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(list(nx.bfs_tree(G,0,0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(nx.shortest_path_length(G,source=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "one_path = nx.all_pairs_shortest_path(G,cutoff=1)[0]\n",
    "two_path = nx.all_pairs_shortest_path(G,cutoff=2)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(len(two_path), len(one_path)-1, len(two_path)-len(one_path)-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "one_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "two_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "type(two_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = preprocess_features(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#print(features.shape)\n",
    "features[1].shape\n",
    "#2708*1433"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(adj.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import time\n",
    "import tensorflow as tf\n",
    "\n",
    "from utils import *\n",
    "from models import GCN, MLP\n",
    "\n",
    "# Set random seed\n",
    "seed = 123\n",
    "np.random.seed(seed)\n",
    "tf.set_random_seed(seed)\n",
    "\n",
    "# Settings\n",
    "flags = tf.app.flags\n",
    "FLAGS = flags.FLAGS\n",
    "flags.DEFINE_string('dataset', 'pubmed', 'Dataset string.')  # 'cora', 'citeseer', 'pubmed'\n",
    "flags.DEFINE_string('model', 'gcn', 'Model string.')  # 'gcn', 'gcn_cheby', 'dense'\n",
    "flags.DEFINE_float('learning_rate', 0.01, 'Initial learning rate.')\n",
    "flags.DEFINE_integer('epochs', 200, 'Number of epochs to train.')\n",
    "flags.DEFINE_integer('hidden1', 16, 'Number of units in hidden layer 1.')\n",
    "flags.DEFINE_float('dropout', 0.5, 'Dropout rate (1 - keep probability).')\n",
    "flags.DEFINE_float('weight_decay', 5e-4, 'Weight for L2 loss on embedding matrix.')\n",
    "flags.DEFINE_integer('early_stopping', 10, 'Tolerance for early stopping (# of epochs).')\n",
    "flags.DEFINE_integer('max_degree', 3, 'Maximum Chebyshev polynomial degree.')\n",
    "\n",
    "# Load data\n",
    "adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask = load_data(FLAGS.dataset)\n",
    "\n",
    "# Some preprocessing\n",
    "features = preprocess_features(features)\n",
    "if FLAGS.model == 'gcn':\n",
    "    support = [preprocess_adj(adj)]\n",
    "    num_supports = 1\n",
    "    model_func = GCN\n",
    "elif FLAGS.model == 'gcn_cheby':\n",
    "    support = chebyshev_polynomials(adj, FLAGS.max_degree)\n",
    "    num_supports = 1 + FLAGS.max_degree\n",
    "    model_func = GCN\n",
    "elif FLAGS.model == 'dense':\n",
    "    support = [preprocess_adj(adj)]  # Not used\n",
    "    num_supports = 1\n",
    "    model_func = MLP\n",
    "else:\n",
    "    raise ValueError('Invalid argument for model: ' + str(FLAGS.model))\n",
    "\n",
    "# Define placeholders\n",
    "placeholders = {\n",
    "    'support': [tf.sparse_placeholder(tf.float32) for _ in range(num_supports)],\n",
    "    'features': tf.sparse_placeholder(tf.float32, shape=tf.constant(features[2], dtype=tf.int64)),\n",
    "    'labels': tf.placeholder(tf.float32, shape=(None, y_train.shape[1])),\n",
    "    'labels_mask': tf.placeholder(tf.int32),\n",
    "    'dropout': tf.placeholder_with_default(0., shape=()),\n",
    "    'num_features_nonzero': tf.placeholder(tf.int32)  # helper variable for sparse dropout\n",
    "}\n",
    "\n",
    "# Create model\n",
    "model = model_func(placeholders, input_dim=features[2][1], logging=True)\n",
    "\n",
    "# Initialize session\n",
    "sess = tf.Session()\n",
    "\n",
    "\n",
    "# Define model evaluation function\n",
    "def evaluate(features, support, labels, mask, placeholders):\n",
    "    t_test = time.time()\n",
    "    feed_dict_val = construct_feed_dict(features, support, labels, mask, placeholders)\n",
    "    outs_val = sess.run([model.loss, model.accuracy], feed_dict=feed_dict_val)\n",
    "    return outs_val[0], outs_val[1], (time.time() - t_test)\n",
    "\n",
    "\n",
    "# Init variables\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "cost_val = []\n",
    "\n",
    "# Construct feed dictionary\n",
    "feed_dict = construct_feed_dict(features, support, y_train, train_mask, placeholders)\n",
    "feed_dict.update({placeholders['dropout']: FLAGS.dropout})\n",
    "\n",
    "\n",
    "# Train model\n",
    "for epoch in range(FLAGS.epochs):\n",
    "\n",
    "    t = time.time()\n",
    "  \n",
    "\n",
    "    # Training step\n",
    "    outs = sess.run([model.opt_op, model.loss, model.accuracy], feed_dict=feed_dict)\n",
    "\n",
    "    # Validation\n",
    "    cost, acc, duration = evaluate(features, support, y_val, val_mask, placeholders)\n",
    "    cost_val.append(cost)\n",
    "\n",
    "    # Print results\n",
    "    print(\"Epoch:\", '%04d' % (epoch + 1), \"train_loss=\", \"{:.5f}\".format(outs[1]),\n",
    "          \"train_acc=\", \"{:.5f}\".format(outs[2]), \"val_loss=\", \"{:.5f}\".format(cost),\n",
    "          \"val_acc=\", \"{:.5f}\".format(acc), \"time=\", \"{:.5f}\".format(time.time() - t))\n",
    "\n",
    "    if epoch > FLAGS.early_stopping and cost_val[-1] > np.mean(cost_val[-(FLAGS.early_stopping+1):-1]):\n",
    "        print(\"Early stopping...\")\n",
    "        break\n",
    "\n",
    "print(\"Optimization Finished!\")\n",
    "\n",
    "# Testing\n",
    "test_cost, test_acc, test_duration = evaluate(features, support, y_test, test_mask, placeholders)\n",
    "print(\"Test set results:\", \"cost=\", \"{:.5f}\".format(test_cost),\n",
    "      \"accuracy=\", \"{:.5f}\".format(test_acc), \"time=\", \"{:.5f}\".format(test_duration))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}