algo_check.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from distributions.distribution_by_sequence import DistributionBySequence\n",
    "from distributions.distribution import Distribution\n",
    "from distributions.sequence import Sequence\n",
    "from algorithms.semi_bandit_exp3 import SemiBanditExp3\n",
    "from algorithms.full_bandit_exp3 import FullBanditExp3\n",
    "from algorithms.semi_bandit_ftrl import SemiBanditFTRL\n",
    "from algorithms.semi_bandit_ftrl_inv import SemiBanditFTRLInv\n",
    "from algorithms.full_bandit_exp3_inv import FullBanditExp3Inv\n",
    "from algorithms.real_lin_exp3 import RealLinExp3\n",
    "from algorithms.uniform_random import UniformRandom\n",
    "from algorithms.non_contextual_exp3 import NonContextualExp3\n",
    "from algorithms.one_per_context import OnePerContext\n",
    "from experiment_manager.experiment_manager import ExperimentManager\n",
    "\n",
    "from algorithms.shortest_path import ShortestPath\n",
    "from algorithms.bubeck import Bubeck\n",
    "from algorithms.one_per_context_bubeck import OnePerContextBubeck\n",
    "\n",
    "from misc.tensor_helpers import *\n",
    "\n",
    "from distributions.actionsets.msets import MSets\n",
    "\n",
    "from distributions.contexts.binary_context import BinaryContext\n",
    "from distributions.thetas.single_hole import SingleHole\n",
    "from distributions.thetas.independent_bernoulli import IndependentBernoulli\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dist(rng, d, K, m):\n",
    "    p = np.zeros((d, K)) + 0.5\n",
    "    for i in range(d):\n",
    "        placed_already = []\n",
    "        while len(placed_already) < m:\n",
    "\n",
    "            index = rng.integers(K)\n",
    "            if index not in placed_already:\n",
    "                placed_already.append(index)\n",
    "                p[i, index] = 0.4\n",
    "    \n",
    "    return IndependentBernoulli(d, K, p)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sigma: 1.4142135623730951 m: 1 beta: 0.4999999999999999 gamma: 0.017320508075688773 eta 0.0060514799530586175\n",
      "Bubeck 457.0 0.017320508075688773 0.0060514799530586175\n",
      "OnePerContextBubeck 470.0 None None\n",
      "ShortestPath 552.0 0.05256521769756932 0.0029202898720871845\n",
      "SemiBanditFTRLInv 710.0 0.21538207996457961 0.0006740662842713811\n",
      "UniformRandom 731.0 None None\n",
      "RealLinExp3NoLog 249.0 0.01 0.0030257399765293088\n",
      "RealLinExp3 608.0 0.035603477744141666 0.0008498439389189096\n",
      "OnePerContext 550.0 None None\n",
      "FullBanditExp3Inv 981.0 0.15583524465641818 0.0007044865450771258\n"
     ]
    }
   ],
   "source": [
    "rng = np.random.default_rng()\n",
    "length = 10000\n",
    "d = 4\n",
    "number_of_ones = 2\n",
    "K = 3\n",
    "m  = 1\n",
    "actionset = MSets(K, m)\n",
    "\n",
    "epsilon = 0.25 * np.min([np.sqrt(K / length), 1])\n",
    "p = np.zeros((d, K)) + 0.5\n",
    "for i in range(d):\n",
    "    p[i, 0] = 0.45\n",
    "\n",
    "dist_lower_bound = Distribution(BinaryContext(d, number_of_ones), get_dist(rng, d, K, m), actionset)\n",
    "\n",
    "\n",
    "seq = dist_lower_bound.generate(length, rng, rng)\n",
    "algo = Bubeck()\n",
    "algo.set_constants(rng, seq)\n",
    "print(\"sigma:\", seq.sigma, \"m:\", seq.m, \"beta:\", algo.beta, \"gamma:\", algo.gamma, \"eta\", algo.eta)\n",
    "#algo.gamma = 0.06887868239885406\n",
    "#algo.eta = 0.0010063304877787227\n",
    "loss, _, _, _ = algo.run_on_sequence(rng, seq)\n",
    "loss_of_optimal_policy, _, _ = seq.find_optimal_policy()\n",
    "print(\"Bubeck\", loss - loss_of_optimal_policy, algo.gamma, algo.eta)\n",
    "\n",
    "algo = OnePerContextBubeck()\n",
    "algo.set_constants(rng, seq)\n",
    "loss, _, _, _ = algo.run_on_sequence(rng, seq)\n",
    "print(\"OnePerContextBubeck\", loss - loss_of_optimal_policy, algo.gamma, algo.eta)\n",
    "\n",
    "algo = ShortestPath()\n",
    "algo.set_constants(rng, seq)\n",
    "loss, _, _, _ = algo.run_on_sequence(rng, seq)\n",
    "print(\"ShortestPath\", loss - loss_of_optimal_policy, algo.gamma, algo.eta)\n",
    "\n",
    "algo = SemiBanditFTRLInv()\n",
    "algo.set_constants(rng, seq)\n",
    "loss, _, _, _ = algo.run_on_sequence(rng, seq)\n",
    "print(\"SemiBanditFTRLInv\", loss - loss_of_optimal_policy, algo.gamma, algo.eta)\n",
    "\n",
    "algo = UniformRandom()\n",
    "algo.set_constants(rng, seq)\n",
    "loss, _, _, _ = algo.run_on_sequence(rng, seq)\n",
    "print(\"UniformRandom\", loss - loss_of_optimal_policy, algo.gamma, algo.eta)\n",
    "\n",
    "algo = RealLinExp3()\n",
    "algo.set_constants(rng, seq)\n",
    "algo.gamma = np.sqrt(1 / seq.length)\n",
    "algo.eta = np.sqrt(np.log(len(seq.actionset.actionset)) / (seq.d * len(seq.actionset.actionset) * seq.length))\n",
    "loss, _, _, _ = algo.run_on_sequence(rng, seq)\n",
    "print(\"RealLinExp3NoLog\", loss - loss_of_optimal_policy, algo.gamma, algo.eta)\n",
    "\n",
    "algo = RealLinExp3()\n",
    "algo.set_constants(rng, seq)\n",
    "loss, _, _, _ = algo.run_on_sequence(rng, seq)\n",
    "print(\"RealLinExp3\", loss - loss_of_optimal_policy, algo.gamma, algo.eta)\n",
    "\n",
    "algo = OnePerContext()\n",
    "algo.set_constants(rng, seq)\n",
    "loss, _, _, _ = algo.run_on_sequence(rng, seq)\n",
    "context_algo = algo.context_algos[list(algo.context_algos.keys())[0]]\n",
    "print(\"OnePerContext\", loss - loss_of_optimal_policy, algo.gamma, algo.eta)\n",
    "\n",
    "algo = FullBanditExp3Inv()\n",
    "algo.set_constants(rng, seq)\n",
    "loss, _, _, _ = algo.run_on_sequence(rng, seq)\n",
    "print(\"FullBanditExp3Inv\", loss - loss_of_optimal_policy, algo.gamma, algo.eta)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start next\n",
      "[1. 0. 0. 1.] [0.33333333 0.33333333 0.33333333]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [2. 1. 0.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[ 0.  6.  0.]\n",
      " [ 0. -3.  0.]\n",
      " [ 0. -3.  0.]\n",
      " [ 0.  6.  0.]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 1. 0. 0.] [0.33350957 0.33298086 0.33350957]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "2.0 [2. 0. 1.]\n",
      "2.0 [2. 0. 0.]\n",
      "[[11.98891127  6.          0.        ]\n",
      " [11.99841472 -3.          0.        ]\n",
      " [-5.9920736  -3.          0.        ]\n",
      " [-6.00157705  6.          0.        ]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.3333334  0.33227554 0.33439107]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "0.0 [0. 1. 0.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[11.98891127  6.          0.        ]\n",
      " [11.99841472 -3.          0.        ]\n",
      " [-5.9920736  -3.          0.        ]\n",
      " [-6.00157705  6.          0.        ]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.33227554 0.33439219 0.33333228]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "1.0 [2. 0. 1.]\n",
      "1.0 [0. 0. 1.]\n",
      "[[11.98891127  6.         -2.9999983 ]\n",
      " [11.99841472 -3.          5.99051616]\n",
      " [-5.9920736  -3.          6.000016  ]\n",
      " [-6.00157705  6.         -2.99049847]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.33509905 0.33245131 0.33244964]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "1.0 [0. 0. 1.]\n",
      "1.0 [0. 0. 1.]\n",
      "[[11.98891127  6.         -6.00559379]\n",
      " [11.99841472 -3.          2.99922527]\n",
      " [-5.9920736  -3.         12.01357075]\n",
      " [-6.00157705  6.          3.00875169]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33315652 0.33209937 0.33474411]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 2. 0.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[ 1.80040304e+01  6.00000000e+00 -6.00559379e+00]\n",
      " [ 9.00872291e+00 -3.00000000e+00  2.99922527e+00]\n",
      " [-9.00557452e+00 -3.00000000e+00  1.20135707e+01]\n",
      " [-1.02670190e-02  6.00000000e+00  3.00875169e+00]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33174779 0.33280018 0.33545203]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "2.0 [0. 2. 2.]\n",
      "2.0 [0. 0. 2.]\n",
      "[[ 1.80040304e+01  6.00000000e+00  5.93751816e+00]\n",
      " [ 9.00872291e+00 -3.00000000e+00 -3.01966306e+00]\n",
      " [-9.00557452e+00 -3.00000000e+00  6.03252766e+00]\n",
      " [-1.02670190e-02  6.00000000e+00  1.49897089e+01]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33315274 0.33421081 0.33263645]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "1.0 [2. 1. 1.]\n",
      "1.0 [0. 0. 1.]\n",
      "[[ 1.80040304e+01  6.00000000e+00  1.19405810e+01]\n",
      " [ 9.00872291e+00 -3.00000000e+00 -6.03554134e+00]\n",
      " [-9.00557452e+00 -3.00000000e+00  3.03578717e+00]\n",
      " [-1.02670190e-02  6.00000000e+00  2.10119095e+01]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 1. 0. 0.] [0.3306885  0.33491312 0.33439838]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "0.0 [0. 0. 1.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[ 1.80040304e+01  6.00000000e+00  1.19405810e+01]\n",
      " [ 9.00872291e+00 -3.00000000e+00 -6.03554134e+00]\n",
      " [-9.00557452e+00 -3.00000000e+00  3.03578717e+00]\n",
      " [-1.02670190e-02  6.00000000e+00  2.10119095e+01]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33385607 0.334917   0.33122693]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "1.0 [1. 2. 1.]\n",
      "1.0 [0. 0. 1.]\n",
      "[[ 1.80040304e+01  6.00000000e+00  1.79595574e+01]\n",
      " [ 9.00872291e+00 -3.00000000e+00 -9.05466269e+00]\n",
      " [-9.00557452e+00 -3.00000000e+00  3.59126870e-02]\n",
      " [-1.02670190e-02  6.00000000e+00  2.70501328e+01]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 0. 1.] [0.33350735 0.33456769 0.33192497]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 1.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[ 1.80040304e+01  3.00399434e+00  1.79595574e+01]\n",
      " [ 9.00872291e+00  2.98023809e+00 -9.05466269e+00]\n",
      " [-9.00557452e+00 -5.98181809e+00  3.59126870e-02]\n",
      " [-1.02670190e-02  1.19755289e+01  2.70501328e+01]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.33634104 0.33367799 0.32998097]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "0.0 [0. 1. 1.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[ 1.80040304e+01  3.00399434e+00  1.79595574e+01]\n",
      " [ 9.00872291e+00  2.98023809e+00 -9.05466269e+00]\n",
      " [-9.00557452e+00 -5.98181809e+00  3.59126870e-02]\n",
      " [-1.02670190e-02  1.19755289e+01  2.70501328e+01]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 1. 0.] [0.33315393 0.33527238 0.33157369]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "0.0 [0. 1. 1.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[ 1.80040304e+01  3.00399434e+00  1.79595574e+01]\n",
      " [ 9.00872291e+00  2.98023809e+00 -9.05466269e+00]\n",
      " [-9.00557452e+00 -5.98181809e+00  3.59126870e-02]\n",
      " [-1.02670190e-02  1.19755289e+01  2.70501328e+01]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.33262671 0.33315553 0.33421776]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 0. 0.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[15.02149931  3.00399434 17.95955743]\n",
      " [15.02153064  2.98023809 -9.05466269]\n",
      " [-3.01188367 -5.98181809  0.03591269]\n",
      " [-3.02145481 11.97552892 27.0501328 ]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 1. 0. 0.] [0.33068403 0.33491674 0.33439923]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "2.0 [2. 2. 1.]\n",
      "2.0 [0. 2. 0.]\n",
      "[[ 15.02149931  14.94257133  17.95955743]\n",
      " [ 15.02153064  14.94711432  -9.05466269]\n",
      " [ -3.01188367 -11.96765818   0.03591269]\n",
      " [ -3.02145481   5.99911534  27.0501328 ]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.33156851 0.33315457 0.33527692]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 2.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[ 12.03425077  14.94257133  17.95955743]\n",
      " [ 21.03916166  14.94711432  -9.05466269]\n",
      " [  2.99609053 -11.96765818   0.03591269]\n",
      " [ -6.02794135   5.99911534  27.0501328 ]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 0. 1.] [0.33385765 0.33281133 0.33333102]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 0.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[ 9.02717854e+00  1.49425713e+01  1.79595574e+01]\n",
      " [ 2.70559926e+01  1.49471143e+01 -9.05466269e+00]\n",
      " [ 3.15675386e-03 -1.19676582e+01  3.59126870e-02]\n",
      " [-5.39884406e-02  5.99911534e+00  2.70501328e+01]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 1. 0. 0.] [0.33137159 0.33245754 0.33617087]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "0.0 [0. 2. 1.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[ 9.02717854e+00  1.49425713e+01  1.79595574e+01]\n",
      " [ 2.70559926e+01  1.49471143e+01 -9.05466269e+00]\n",
      " [ 3.15675386e-03 -1.19676582e+01  3.59126870e-02]\n",
      " [-5.39884406e-02  5.99911534e+00  2.70501328e+01]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32980879 0.33403088 0.33616034]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 2. 1.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[  6.02885203  14.94257133  17.95955743]\n",
      " [ 33.1008426   14.94711432  -9.05466269]\n",
      " [  6.02864533 -11.96765818   0.03591269]\n",
      " [ -3.06208582   5.99911534  27.0501328 ]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.33421556 0.33580086 0.32998358]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "0.0 [0. 1. 2.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[  6.02885203  14.94257133  17.95955743]\n",
      " [ 33.1008426   14.94711432  -9.05466269]\n",
      " [  6.02864533 -11.96765818   0.03591269]\n",
      " [ -3.06208582   5.99911534  27.0501328 ]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32840182 0.33473152 0.33686667]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "0.0 [0. 0. 1.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[  6.02885203  14.94257133  17.95955743]\n",
      " [ 33.1008426   14.94711432  -9.05466269]\n",
      " [  6.02864533 -11.96765818   0.03591269]\n",
      " [ -3.06208582   5.99911534  27.0501328 ]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32840182 0.33473152 0.33686667]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 1.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[ 6.02885203 11.95504491 17.95955743]\n",
      " [33.1008426  20.93617806 -9.05466269]\n",
      " [ 6.02864533 -5.99747532  0.03591269]\n",
      " [-3.06208582  3.00216813 27.0501328 ]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33650918 0.33437828 0.32911253]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [2. 1. 2.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[ 6.02885203 17.94569727 17.95955743]\n",
      " [33.1008426  17.95025419 -9.05466269]\n",
      " [ 6.02864533 -9.00232719  0.03591269]\n",
      " [-3.06208582  8.98337619 27.0501328 ]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33722314 0.33297032 0.32980654]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "0.0 [2. 2. 0.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[ 6.02885203 17.94569727 17.95955743]\n",
      " [33.1008426  17.95025419 -9.05466269]\n",
      " [ 6.02864533 -9.00232719  0.03591269]\n",
      " [-3.06208582  8.98337619 27.0501328 ]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33722314 0.33297032 0.32980654]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "2.0 [0. 2. 0.]\n",
      "2.0 [0. 2. 0.]\n",
      "[[  6.02885203  29.95866433  17.95955743]\n",
      " [ 33.1008426   11.97219599  -9.05466269]\n",
      " [  6.02864533 -15.01845615   0.03591269]\n",
      " [ -3.06208582  20.97734751  27.0501328 ]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 0. 1.] [0.33279473 0.33228313 0.33492215]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [0. 1. 1.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[  6.02885203  26.97315613  17.95955743]\n",
      " [ 33.1008426   17.98165719  -9.05466269]\n",
      " [  6.02864533 -18.04219366   0.03591269]\n",
      " [ -3.06208582  26.99636119  27.0501328 ]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 0. 1.] [0.33349633 0.3308742  0.33562947]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "2.0 [0. 2. 2.]\n",
      "2.0 [0. 0. 2.]\n",
      "[[  6.02885203  26.97315613  11.96763279]\n",
      " [ 33.1008426   17.98165719   2.83951194]\n",
      " [  6.02864533 -18.04219366  -5.94648365]\n",
      " [ -3.06208582  26.99636119  39.04870134]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.33545461 0.33439261 0.33015279]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 0. 1.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[  3.01454014  26.97315613  11.96763279]\n",
      " [ 30.12448185  17.98165719   2.83951194]\n",
      " [ 12.03353164 -18.04219366  -5.94648365]\n",
      " [  2.88596555  26.99636119  39.04870134]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32821779 0.33561995 0.33616227]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 0. 1.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[ 2.34844926e-02  2.69731561e+01  1.19676328e+01]\n",
      " [ 3.61670195e+01  1.79816572e+01  2.83951194e+00]\n",
      " [ 1.80806874e+01 -1.80421937e+01 -5.94648365e+00]\n",
      " [-1.19797122e-01  2.69963612e+01  3.90487013e+01]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.33368818 0.33528301 0.33102881]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [0. 1. 0.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[ 2.34844926e-02  2.39835006e+01  1.19676328e+01]\n",
      " [ 3.61670195e+01  1.49683616e+01  2.83951194e+00]\n",
      " [ 1.80806874e+01 -1.20747080e+01 -5.94648365e+00]\n",
      " [-1.19797122e-01  3.29970929e+01  3.90487013e+01]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.33439675 0.33387306 0.33173019]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "2.0 [2. 0. 1.]\n",
      "2.0 [2. 0. 0.]\n",
      "[[ -6.02428192  23.98350059  11.96763279]\n",
      " [ 30.21459055  14.96836159   2.83951194]\n",
      " [ 30.12881622 -12.074708    -5.94648365]\n",
      " [ 11.79499313  32.9970929   39.04870134]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 1. 0. 0.] [0.33364371 0.33105116 0.33530513]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 2.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[ -0.04442074  23.98350059  11.96763279]\n",
      " [ 36.24236523  14.96836159   2.83951194]\n",
      " [ 27.15593723 -12.074708    -5.94648365]\n",
      " [  8.75510247  32.9970929   39.04870134]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32592325 0.33650194 0.33757481]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "2.0 [0. 2. 0.]\n",
      "2.0 [0. 2. 0.]\n",
      "[[-0.04442074 17.99237752 11.96763279]\n",
      " [36.24236523 26.93079897  2.83951194]\n",
      " [27.15593723 -0.15021285 -5.94648365]\n",
      " [ 8.75510247 26.98718694 39.04870134]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 1. 0.] [0.33155092 0.333179   0.33527008]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "0.0 [0. 0. 2.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[-0.04442074 17.99237752 11.96763279]\n",
      " [36.24236523 26.93079897  2.83951194]\n",
      " [27.15593723 -0.15021285 -5.94648365]\n",
      " [ 8.75510247 26.98718694 39.04870134]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.3379706  0.33154211 0.33048729]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "2.0 [2. 0. 0.]\n",
      "2.0 [2. 0. 0.]\n",
      "[[11.89518461 17.99237752 11.96763279]\n",
      " [30.22022446 26.93079897  2.83951194]\n",
      " [21.15219142 -0.15021285 -5.94648365]\n",
      " [20.67534314 26.98718694 39.04870134]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33514929 0.33295563 0.33189508]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "0.0 [0. 1. 1.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[11.89518461 17.99237752 11.96763279]\n",
      " [30.22022446 26.93079897  2.83951194]\n",
      " [21.15219142 -0.15021285 -5.94648365]\n",
      " [20.67534314 26.98718694 39.04870134]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.33194115 0.33458316 0.33347569]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 1.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[ 8.87256866 17.99237752 11.96763279]\n",
      " [27.21724195 26.93079897  2.83951194]\n",
      " [27.19709075 -0.15021285 -5.94648365]\n",
      " [26.68132907 26.98718694 39.04870134]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.33053053 0.33528996 0.33417951]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 1.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[ 5.846692   17.99237752 11.96763279]\n",
      " [24.21111088 26.93079897  2.83951194]\n",
      " [33.25814989 -0.15021285 -5.94648365]\n",
      " [32.70325252 26.98718694 39.04870134]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32798918 0.33334408 0.33866675]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "2.0 [0. 1. 2.]\n",
      "2.0 [0. 0. 2.]\n",
      "[[ 5.846692   17.99237752  5.99594726]\n",
      " [24.21111088 26.93079897 14.706611  ]\n",
      " [33.25814989 -0.15021285  5.92124871]\n",
      " [32.70325252 26.98718694 33.09611023]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 1. 0.] [0.3304936  0.3342281  0.33527831]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "0.0 [0. 1. 0.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[ 5.846692   17.99237752  5.99594726]\n",
      " [24.21111088 26.93079897 14.706611  ]\n",
      " [33.25814989 -0.15021285  5.92124871]\n",
      " [32.70325252 26.98718694 33.09611023]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33374284 0.33261002 0.33364714]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [2. 1. 0.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[ 5.846692   24.00525888  5.99594726]\n",
      " [24.21111088 23.94837305 14.706611  ]\n",
      " [33.25814989 -3.16141561  5.92124871]\n",
      " [32.70325252 32.98101176 33.09611023]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32903071 0.33547051 0.33549879]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 1.]\n",
      "1.0 [0. 0. 1.]\n",
      "[[ 5.846692   24.00525888  3.00535915]\n",
      " [24.21111088 23.94837305 20.67282451]\n",
      " [33.25814989 -3.16141561 11.88777866]\n",
      " [32.70325252 32.98101176 30.11518589]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.32981209 0.33617511 0.3340128 ]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "0.0 [0. 1. 2.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[ 5.846692   24.00525888  3.00535915]\n",
      " [24.21111088 23.94837305 20.67282451]\n",
      " [33.25814989 -3.16141561 11.88777866]\n",
      " [32.70325252 32.98101176 30.11518589]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32972519 0.33618238 0.33409244]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 1.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[ 2.82909043 24.00525888  3.00535915]\n",
      " [30.24235228 23.94837305 20.67282451]\n",
      " [39.31828151 -3.16141561 11.88777866]\n",
      " [29.69507027 32.98101176 30.11518589]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 1. 0.] [0.33048732 0.33422895 0.33528373]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 1.]\n",
      "1.0 [0. 0. 1.]\n",
      "[[ 2.82909043 24.00525888  8.9728816 ]\n",
      " [30.24235228 23.94837305 17.6829012 ]\n",
      " [39.31828151 -3.16141561 17.85987024]\n",
      " [29.69507027 32.98101176 27.13070166]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 1. 0.] [0.33118556 0.33493729 0.33387715]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "2.0 [2. 2. 0.]\n",
      "2.0 [2. 0. 0.]\n",
      "[[14.84852526 24.00525888  8.9728816 ]\n",
      " [24.22383039 23.94837305 17.6829012 ]\n",
      " [51.43500134 -3.16141561 17.85987024]\n",
      " [23.65567297 32.98101176 27.13070166]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32777934 0.33742482 0.33479584]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "0.0 [0. 1. 2.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[14.84852526 24.00525888  8.9728816 ]\n",
      " [24.22383039 23.94837305 17.6829012 ]\n",
      " [51.43500134 -3.16141561 17.85987024]\n",
      " [23.65567297 32.98101176 27.13070166]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.3289208  0.33688815 0.33419105]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 1.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[14.84852526 21.01802543  8.9728816 ]\n",
      " [24.22383039 20.9516285  17.6829012 ]\n",
      " [51.43500134  2.77527346 17.85987024]\n",
      " [23.65567297 38.96502734 27.13070166]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 0. 1.] [0.33385853 0.33174079 0.33440068]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 1.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[11.80737196 21.01802543  8.9728816 ]\n",
      " [30.24600271 20.9516285  17.6829012 ]\n",
      " [48.43749749  2.77527346 17.85987024]\n",
      " [29.66271985 38.96502734 27.13070166]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 0. 1.] [0.33244523 0.33244384 0.33511093]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 1.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[11.80737196 18.04611844  8.9728816 ]\n",
      " [30.24600271 26.94610902 17.6829012 ]\n",
      " [48.43749749 -0.23996495 17.85987024]\n",
      " [29.66271985 44.97385654 27.13070166]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32777314 0.33690063 0.33532623]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "0.0 [0. 2. 0.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[11.80737196 18.04611844  8.9728816 ]\n",
      " [30.24600271 26.94610902 17.6829012 ]\n",
      " [48.43749749 -0.23996495 17.85987024]\n",
      " [29.66271985 44.97385654 27.13070166]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 1. 0. 0.] [0.33259838 0.33208225 0.33531937]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "0.0 [0. 0. 0.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[11.80737196 18.04611844  8.9728816 ]\n",
      " [30.24600271 26.94610902 17.6829012 ]\n",
      " [48.43749749 -0.23996495 17.85987024]\n",
      " [29.66271985 44.97385654 27.13070166]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 0. 1.] [0.33314487 0.33103742 0.33581772]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "1.0 [2. 1. 1.]\n",
      "1.0 [0. 0. 1.]\n",
      "[[11.80737196 18.04611844  5.99041285]\n",
      " [30.24600271 26.94610902 23.64289881]\n",
      " [48.43749749 -0.23996495 14.87725035]\n",
      " [29.66271985 44.97385654 33.09140946]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33445489 0.33066704 0.33487807]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "2.0 [0. 2. 1.]\n",
      "2.0 [0. 2. 0.]\n",
      "[[11.80737196 30.07999845  5.99041285]\n",
      " [30.24600271 20.98960852 23.64289881]\n",
      " [48.43749749 -6.26424448 14.87725035]\n",
      " [29.66271985 57.01714887 33.09140946]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.3272453  0.33849917 0.33425554]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "0.0 [1. 0. 0.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[11.80737196 30.07999845  5.99041285]\n",
      " [30.24600271 20.98960852 23.64289881]\n",
      " [48.43749749 -6.26424448 14.87725035]\n",
      " [29.66271985 57.01714887 33.09140946]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 1. 0.] [0.32889762 0.33528912 0.33581326]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "2.0 [2. 0. 2.]\n",
      "2.0 [0. 0. 2.]\n",
      "[[11.80737196 30.07999845 17.90184994]\n",
      " [30.24600271 20.98960852 17.68671563]\n",
      " [48.43749749 -6.26424448 26.80724637]\n",
      " [29.66271985 57.01714887 27.1177705 ]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32758916 0.33885817 0.33355267]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "2.0 [1. 1. 2.]\n",
      "2.0 [0. 0. 2.]\n",
      "[[11.80737196 30.07999845 11.93776098]\n",
      " [30.24600271 20.98960852 29.64601549]\n",
      " [48.43749749 -6.26424448 38.79551481]\n",
      " [29.66271985 57.01714887 21.12582668]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32896296 0.3402925  0.33074454]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "0.0 [1. 0. 2.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[11.80737196 30.07999845 11.93776098]\n",
      " [30.24600271 20.98960852 29.64601549]\n",
      " [48.43749749 -6.26424448 38.79551481]\n",
      " [29.66271985 57.01714887 21.12582668]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33549612 0.32750924 0.33699464]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 2.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[11.80737196 36.12080784 11.93776098]\n",
      " [30.24600271 18.01650326 29.64601549]\n",
      " [48.43749749 -9.28135964 38.79551481]\n",
      " [29.66271985 63.07240412 21.12582668]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.33084214 0.33512187 0.33403599]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "2.0 [2. 1. 1.]\n",
      "2.0 [2. 0. 0.]\n",
      "[[ 5.77138683 36.12080784 11.93776098]\n",
      " [24.24973982 18.01650326 29.64601549]\n",
      " [60.54836656 -9.28135964 38.79551481]\n",
      " [41.67529482 63.07240412 21.12582668]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32789586 0.34136902 0.33073511]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "2.0 [0. 1. 2.]\n",
      "2.0 [0. 0. 2.]\n",
      "[[ 5.77138683 36.12080784  5.98018484]\n",
      " [24.24973982 18.01650326 41.66864598]\n",
      " [60.54836656 -9.28135964 50.82832222]\n",
      " [41.67529482 63.07240412 15.12093267]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33476759 0.32575305 0.33947936]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "1.0 [1. 0. 1.]\n",
      "1.0 [0. 0. 1.]\n",
      "[[ 5.77138683 36.12080784 11.93381764]\n",
      " [24.24973982 18.01650326 38.66921442]\n",
      " [60.54836656 -9.28135964 47.83386208]\n",
      " [41.67529482 63.07240412 21.05145073]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.3285481  0.33706863 0.33438327]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "1.0 [2. 1. 1.]\n",
      "1.0 [0. 0. 1.]\n",
      "[[ 5.77138683 36.12080784  8.9200793 ]\n",
      " [24.24973982 18.01650326 35.68354733]\n",
      " [60.54836656 -9.28135964 53.85266553]\n",
      " [41.67529482 63.07240412 27.01362101]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 1. 0.] [0.33079825 0.3377835  0.33141825]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "1.0 [0. 1. 1.]\n",
      "1.0 [0. 0. 1.]\n",
      "[[ 5.77138683 36.12080784 14.91890869]\n",
      " [24.24973982 18.01650326 32.70201928]\n",
      " [60.54836656 -9.28135964 59.89454479]\n",
      " [41.67529482 63.07240412 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.3294154 0.3379647 0.3326199]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 1.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[ 2.75483658 36.12080784 14.91890869]\n",
      " [21.25323293 18.01650326 32.70201928]\n",
      " [66.61068531 -9.28135964 59.89454479]\n",
      " [47.69764236 63.07240412 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.32800786 0.33867561 0.33331653]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 0. 0.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[-0.26497339 36.12080784 14.91890869]\n",
      " [18.25358099 18.01650326 32.70201928]\n",
      " [72.68924954 -9.28135964 59.89454479]\n",
      " [53.73600672 63.07240412 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 1. 0. 0.] [0.33721271 0.33082351 0.33196379]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "0.0 [0. 0. 1.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[-0.26497339 36.12080784 14.91890869]\n",
      " [18.25358099 18.01650326 32.70201928]\n",
      " [72.68924954 -9.28135964 59.89454479]\n",
      " [53.73600672 63.07240412 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 1. 0. 0.] [0.33721271 0.33082351 0.33196379]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "2.0 [0. 2. 1.]\n",
      "2.0 [0. 2. 0.]\n",
      "[[ -0.26497339  48.17333992  14.91890869]\n",
      " [ 18.25358099  29.99193977  32.70201928]\n",
      " [ 72.68924954 -15.35560295  59.89454479]\n",
      " [ 53.73600672  57.21044961  23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33548886 0.32642106 0.33809007]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "0.0 [0. 0. 2.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[ -0.26497339  48.17333992  14.91890869]\n",
      " [ 18.25358099  29.99193977  32.70201928]\n",
      " [ 72.68924954 -15.35560295  59.89454479]\n",
      " [ 53.73600672  57.21044961  23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 1. 0. 0.] [0.33862798 0.32802095 0.33335107]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 2. 2.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[  5.68785051  48.17333992  14.91890869]\n",
      " [ 24.22595176  29.99193977  32.70201928]\n",
      " [ 69.69892825 -15.35560295  59.89454479]\n",
      " [ 50.70671275  57.21044961  23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.32659812 0.34045316 0.33294872]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 1.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[ 5.68785051 45.20337272 14.91890869]\n",
      " [24.22595176 26.98440558 32.70201928]\n",
      " [69.69892825 -9.48129384 59.89454479]\n",
      " [50.70671275 63.18820477 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33531729 0.32624094 0.33844176]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "0.0 [0. 2. 1.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[ 5.68785051 45.20337272 14.91890869]\n",
      " [24.22595176 26.98440558 32.70201928]\n",
      " [69.69892825 -9.48129384 59.89454479]\n",
      " [50.70671275 63.18820477 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.32729426 0.33904353 0.33366221]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "2.0 [0. 2. 1.]\n",
      "2.0 [0. 2. 0.]\n",
      "[[ 5.68785051 39.25741211 14.91890869]\n",
      " [24.22595176 20.96288855 32.70201928]\n",
      " [69.69892825  2.29767597 59.89454479]\n",
      " [50.70671275 75.17523012 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32907272 0.34162409 0.32930319]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 0. 2.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[ 2.66690528 39.25741211 14.91890869]\n",
      " [30.25125713 20.96288855 32.70201928]\n",
      " [75.76839978  2.29767597 59.89454479]\n",
      " [47.70993622 75.17523012 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.3363759  0.32519494 0.33842916]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 2. 1.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[ 8.63959327 39.25741211 14.91890869]\n",
      " [27.21755403 20.96288855 32.70201928]\n",
      " [72.77796545  2.29767597 59.89454479]\n",
      " [53.70630183 75.17523012 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32836565 0.34198589 0.32964847]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [0. 1. 2.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[ 8.63959327 36.25788851 14.91890869]\n",
      " [27.21755403 26.90960383 32.70201928]\n",
      " [72.77796545  8.18774043 59.89454479]\n",
      " [53.70630183 72.18491754 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 1. 0.] [0.3307698  0.33730497 0.33192523]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 0. 1.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[14.64651571 36.25788851 14.91890869]\n",
      " [24.19994945 26.90960383 32.70201928]\n",
      " [78.8442626   8.18774043 59.89454479]\n",
      " [50.6975163  72.18491754 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.32779072 0.33641745 0.33579184]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "0.0 [0. 1. 2.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[14.64651571 36.25788851 14.91890869]\n",
      " [24.19994945 26.90960383 32.70201928]\n",
      " [78.8442626   8.18774043 59.89454479]\n",
      " [50.6975163  72.18491754 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 1. 0. 0.] [0.33527939 0.33099503 0.33372558]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "2.0 [2. 2. 0.]\n",
      "2.0 [2. 0. 0.]\n",
      "[[26.6386971  36.25788851 14.91890869]\n",
      " [36.2033187  26.90960383 32.70201928]\n",
      " [72.85612778  8.18774043 59.89454479]\n",
      " [44.62067708 72.18491754 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33356949 0.32708987 0.33934064]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 1.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[26.6386971  42.29593289 14.91890869]\n",
      " [36.2033187  23.94350881 32.70201928]\n",
      " [72.85612778  5.16714972 59.89454479]\n",
      " [44.62067708 78.24712243 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 1. 0.] [0.32883754 0.33801275 0.33314971]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [0. 1. 1.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[26.6386971  48.29420146 14.91890869]\n",
      " [36.2033187  20.91860174 32.70201928]\n",
      " [72.85612778 11.07918117 59.89454479]\n",
      " [44.62067708 75.27904787 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 1. 0. 0.] [0.33280889 0.33169041 0.3355007 ]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 1.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[32.65549605 48.29420146 14.91890869]\n",
      " [42.23527162 20.91860174 32.70201928]\n",
      " [69.85910185 11.07918117 59.89454479]\n",
      " [41.57885671 75.27904787 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 0. 1. 1.] [0.33024946 0.3346559  0.33509464]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 0.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[32.65549605 45.31777662 14.91890869]\n",
      " [42.23527162 17.89890282 32.70201928]\n",
      " [69.85910185 17.01510559 59.89454479]\n",
      " [41.57885671 81.31643287 23.98955734]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 1. 0. 0.] [0.33104743 0.33309557 0.33585701]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "1.0 [0. 0. 1.]\n",
      "1.0 [0. 0. 1.]\n",
      "[[32.65549605 45.31777662 20.86018282]\n",
      " [42.23527162 17.89890282 38.67133073]\n",
      " [69.85910185 17.01510559 56.94533515]\n",
      " [41.57885671 81.31643287 20.98369585]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 0. 1.] [0.33280345 0.33010781 0.33708874]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "2.0 [2. 2. 2.]\n",
      "2.0 [2. 0. 0.]\n",
      "[[26.58117097 45.31777662 20.86018282]\n",
      " [54.30901869 17.89890282 38.67133073]\n",
      " [63.85525347 17.01510559 56.94533515]\n",
      " [53.60299478 81.31643287 20.98369585]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33373887 0.32566434 0.34059678]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "0.0 [1. 0. 1.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[26.58117097 45.31777662 20.86018282]\n",
      " [54.30901869 17.89890282 38.67133073]\n",
      " [63.85525347 17.01510559 56.94533515]\n",
      " [53.60299478 81.31643287 20.98369585]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 1. 0. 0.] [0.33104453 0.33415148 0.334804  ]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 1.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[32.6107047  45.31777662 20.86018282]\n",
      " [60.37321306 17.89890282 38.67133073]\n",
      " [60.84231224 17.01510559 56.94533515]\n",
      " [50.56417703 81.31643287 20.98369585]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32676334 0.34203556 0.3312011 ]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [0. 1. 1.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[32.6107047  42.30886881 20.86018282]\n",
      " [60.37321306 23.82213764 38.67133073]\n",
      " [60.84231224 22.92767445 56.94533515]\n",
      " [50.56417703 78.33539684 20.98369585]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 0. 1.] [0.329798   0.33132945 0.33887255]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "0.0 [0. 1. 0.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[32.6107047  42.30886881 20.86018282]\n",
      " [60.37321306 23.82213764 38.67133073]\n",
      " [60.84231224 22.92767445 56.94533515]\n",
      " [50.56417703 78.33539684 20.98369585]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32746309 0.34062389 0.33191302]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 1.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[32.6107047  39.29675867 20.86018282]\n",
      " [60.37321306 29.7608307  38.67133073]\n",
      " [60.84231224 28.85564159 56.94533515]\n",
      " [50.56417703 75.35131516 20.98369585]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 1. 0.] [0.33093166 0.33538897 0.33367937]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "0.0 [0. 1. 0.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[32.6107047  39.29675867 20.86018282]\n",
      " [60.37321306 29.7608307  38.67133073]\n",
      " [60.84231224 28.85564159 56.94533515]\n",
      " [50.56417703 75.35131516 20.98369585]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 0. 1.] [0.32996791 0.33098365 0.33904844]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "2.0 [1. 2. 1.]\n",
      "2.0 [0. 2. 0.]\n",
      "[[32.6107047  33.33968891 20.86018282]\n",
      " [60.37321306 41.74296072 38.67133073]\n",
      " [60.84231224 22.80771583 56.94533515]\n",
      " [50.56417703 87.45997294 20.98369585]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32851219 0.33850743 0.33298038]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "1.0 [0. 1. 0.]\n",
      "1.0 [0. 1. 0.]\n",
      "[[32.6107047  30.30854736 20.86018282]\n",
      " [60.37321306 47.71919228 38.67133073]\n",
      " [60.84231224 28.7543372  56.94533515]\n",
      " [50.56417703 84.4759289  20.98369585]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33270975 0.32720923 0.34008102]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 1. 0.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[38.64253455 30.30854736 20.86018282]\n",
      " [57.35131902 47.71919228 38.67133073]\n",
      " [57.82110435 28.7543372  56.94533515]\n",
      " [56.58650139 84.4759289  20.98369585]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33129665 0.32789842 0.34080494]\n",
      "1 [False  True False]\n",
      "\n",
      "\n",
      "0.0 [0. 0. 2.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[38.64253455 30.30854736 20.86018282]\n",
      " [57.35131902 47.71919228 38.67133073]\n",
      " [57.82110435 28.7543372  56.94533515]\n",
      " [56.58650139 84.4759289  20.98369585]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32991767 0.33673958 0.33334275]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "2.0 [2. 1. 1.]\n",
      "2.0 [2. 0. 0.]\n",
      "[[32.60259345 30.30854736 20.86018282]\n",
      " [69.46215484 47.71919228 38.67133073]\n",
      " [69.93332609 28.7543372  56.94533515]\n",
      " [50.52736065 84.4759289  20.98369585]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[0. 1. 1. 0.] [0.32709751 0.33815876 0.33474373]\n",
      "2 [False False  True]\n",
      "\n",
      "\n",
      "2.0 [1. 0. 2.]\n",
      "2.0 [0. 0. 2.]\n",
      "[[32.60259345 30.30854736 14.97035298]\n",
      " [69.46215484 47.71919228 50.54452484]\n",
      " [69.93332609 28.7543372  68.87634479]\n",
      " [50.52736065 84.4759289  15.01833903]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 1. 0. 0.] [0.32978361 0.33399774 0.33621865]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "0.0 [0. 1. 1.]\n",
      "0.0 [0. 0. 0.]\n",
      "[[32.60259345 30.30854736 14.97035298]\n",
      " [69.46215484 47.71919228 50.54452484]\n",
      " [69.93332609 28.7543372  68.87634479]\n",
      " [50.52736065 84.4759289  15.01833903]]\n",
      "\n",
      "\n",
      "\n",
      "Start next\n",
      "[1. 0. 0. 1.] [0.33200118 0.32650789 0.34149094]\n",
      "0 [ True False False]\n",
      "\n",
      "\n",
      "1.0 [1. 0. 2.]\n",
      "1.0 [1. 0. 0.]\n",
      "[[38.64747068 30.30854736 14.97035298]\n",
      " [66.43354095 47.71919228 50.54452484]\n",
      " [66.90541143 28.7543372  68.87634479]\n",
      " [56.56265727 84.4759289  15.01833903]]\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "ename": "ZeroDivisionError",
     "evalue": "division by zero",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mZeroDivisionError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-4-de74b1a1eb0f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     34\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mseq\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcurrent_index\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m100\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m         \u001b[1;36m1\u001b[0m\u001b[1;33m/\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mZeroDivisionError\u001b[0m: division by zero"
     ]
    }
   ],
   "source": [
    "algo = SemiBanditFTRLInv()\n",
    "algo.set_constants(rng, seq)\n",
    "\n",
    "seq.reset()\n",
    "context, _, _, done = seq.get_next(None)\n",
    "\n",
    "losses = []\n",
    "probability_array = []\n",
    "action_array = []\n",
    "while not done:\n",
    "    probabilities = algo.get_policy(context)\n",
    "    probability_array.append(probabilities)\n",
    "\n",
    "    print(\"Start next\")\n",
    "    print(context, probabilities)\n",
    "    action_index = rng.choice(np.arange(seq.actionset.number_of_actions), p=probabilities)\n",
    "    print(action_index, seq.actionset[action_index])\n",
    "    action_array.append(action_index)\n",
    "\n",
    "    print(\"\\n\")\n",
    "    next_context, loss, loss_vec, done = seq.get_next(seq.actionset[action_index])\n",
    "    print(loss, loss_vec)\n",
    "    loss_vec[~seq.actionset[action_index]] = 0\n",
    "    print(loss, loss_vec)\n",
    "    if algo.full_bandit:\n",
    "        algo.observe_loss(loss, context, action_index)\n",
    "    else:\n",
    "        algo.observe_loss_vec(loss_vec, context, action_index)\n",
    "\n",
    "    print(algo.theta_estimate)\n",
    "    print(\"\\n\\n\")\n",
    "    losses.append(loss)\n",
    "    context = next_context\n",
    "\n",
    "    if seq.current_index == 100:\n",
    "        1/0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.6 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "52279f95a71c719c29b46e6649e7c2848e29d27590516e424c9feb43371b1514"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}