Overfitters.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Overfitters.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EkZLWXVbfDDF"
      },
      "source": [
        "### Kaggle competition_ overfitters\r\n",
        "Sanaz Kaviani, sanaz.kaviani@umontreal.ca, marticule: 2111567 \r\n",
        "Mersede Mokri, mersede.mokri@umontreal.ca, marticule: 2111556\r\n",
        "Hamed Naseri, hamed.naseri@polymtl.ca, marticule: 2051414\r\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "htnP8IKcWpqo"
      },
      "source": [
        "### In the following part, the required standard libraries are imported, and the preprocessing function is modeled."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zHMPYANGNwOq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bbb9b40b-6b22-4486-96ed-391199aa6a0c"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn import preprocessing\n",
        "from sklearn.model_selection import train_test_split\n",
        "import math\n",
        "from sklearn.metrics import accuracy_score\n",
        "import os\n",
        "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'  # used to Disable Tensorflow debugging informatio\n",
        "import tensorflow as tf\n",
        "from collections import Counter\n",
        "import matplotlib.pyplot as plt\n",
        "from skimage import measure, morphology\n",
        "import keras\n",
        "from keras.utils import to_categorical\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Conv2D\n",
        "from keras.layers import MaxPooling2D\n",
        "from keras.layers import Dense\n",
        "from keras.layers import Flatten\n",
        "from keras.layers import Dropout\n",
        "from keras.optimizers import SGD\n",
        "from keras.layers import BatchNormalization\n",
        "from keras.layers import LeakyReLU\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OR_Y-AzOabrO"
      },
      "source": [
        "## Pre-Processing \r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v6D8zqB6th4K",
        "outputId": "d2082003-ed2a-4a04-82c1-4259dfe57b77"
      },
      "source": [
        "\"\"\" Pre-Processing \"\"\"\n",
        "def PreProcessing_1(dataSample, filteredImgSize, binarizingTH):\n",
        "    dataSize = dataSample.shape[0]               #Number of Examples\n",
        "    NumPixels = dataSample[0][1].shape[0]       #Original Feature Size which is imageLength*imageWidth\n",
        "    originalImgSize = np.sqrt(NumPixels).astype(int)   #Original Image Size\n",
        "    \"\"\" Initialization \"\"\"\n",
        "    filteredImgCenter = int(filteredImgSize/2)   #Center of the Filtered Image (25 here)\n",
        "    featureSize = filteredImgSize**2\n",
        "    processedImageSet = np.zeros((dataSize, featureSize))\n",
        "    \"\"\" First Train Set: to find Vocabs of SIFT, SURF and anyother feature \"\"\"\n",
        "    for i in range(dataSize):\n",
        "        if (i+1)%1000 == 0:\n",
        "            print(i , ' of data have been processed')\n",
        "        originalImg = dataSample[i,1].reshape(originalImgSize,originalImgSize)\n",
        "        originalImg[originalImg > 255] = 255\n",
        "        binarizedImg = originalImg > binarizingTH  #this needs to be tunned\n",
        "        origImgSegments= measure.label(binarizedImg, background = 0)\n",
        "        mostCommonLabel = Counter(origImgSegments.flatten()).most_common(3)\n",
        "        filterMask = (origImgSegments == mostCommonLabel[1][0]) #+ (origImgSegments ==mostCommonLabel[2][0])\n",
        "        filteredImg = filterMask * originalImg\n",
        "        regionImg = measure.regionprops(filterMask.astype(int))[0]\n",
        "        originalImgCenter = [int(regionImg.bbox[0]+((regionImg.bbox[2]-regionImg.bbox[0])/2)), int(regionImg.bbox[1]+((regionImg.bbox[3]-regionImg.bbox[1])/2))]\n",
        "        deltaX = regionImg.bbox[2]-regionImg.bbox[0]\n",
        "        deltaY = regionImg.bbox[3]-regionImg.bbox[1]\n",
        "        deltaX = int( min(1.5*deltaX , filteredImgSize))\n",
        "        deltaX += deltaX%2\n",
        "        deltaY = int( min(1.5*deltaY , filteredImgSize))\n",
        "        deltaY += deltaY%2\n",
        "        grabbedImg = []\n",
        "        grabbedImg = filteredImg[max(0,originalImgCenter[0]-int(deltaX/2)):min(originalImgSize-1,originalImgCenter[0]+int(deltaX/2)), max(0,originalImgCenter[1]-int(deltaY/2)):min(originalImgSize-1,originalImgCenter[1]+int(deltaY/2))]\n",
        "        tmpXsize = grabbedImg.shape[0]\n",
        "        tmpYsize = grabbedImg.shape[1]\n",
        "        tmpImg = np.zeros((filteredImgSize , filteredImgSize))\n",
        "        tmpImg[filteredImgCenter-int(tmpXsize/2) : filteredImgCenter+int(tmpXsize/2)+tmpXsize%2, filteredImgCenter-int(tmpYsize/2): filteredImgCenter+int(tmpYsize/2)+tmpYsize%2] = grabbedImg\n",
        "        tmpImgSegments= measure.label(tmpImg>binarizingTH, background = 0)\n",
        "        mostCommonLabel = Counter(tmpImgSegments.flatten()).most_common()\n",
        "        finalMask = morphology.remove_small_objects(tmpImg>binarizingTH, min(40, mostCommonLabel[1][1]-1), connectivity=2)\n",
        "        tmpImg = tmpImg*finalMask\n",
        "        outImg = tmpImg / np.max(tmpImg)\n",
        "        processedImageSet[i,:] = outImg.flatten()\n",
        "    return processedImageSet"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nX5ehIRaaq0f"
      },
      "source": [
        "## Loading Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E7JQLDvOapEZ"
      },
      "source": [
        "gpu_devices = tf.config.experimental.list_physical_devices('GPU')\n",
        "tf.config.experimental.set_memory_growth(gpu_devices[0], True)\n",
        "\n",
        "\n",
        "\"\"\"                    Loading Train and Test Data:                         \"\"\"\n",
        "dataTrain = np.load('/content/drive/MyDrive/train_images.npy', encoding='latin1', allow_pickle=True)\n",
        "dataTest  = np.load('/content/drive/MyDrive/test_images.npy',  encoding = 'latin1', allow_pickle=True)\n",
        "\n",
        "\"\"\"  Extracting Train & Test Sets Sizes and Original Feature & Image Sizes  \"\"\"\n",
        "trainSize = dataTrain.shape[0]  # Number of Training Examples\n",
        "testSize  = dataTest.shape[0]                #Number of Testing Examples\n",
        "\n",
        "\"\"\"    Loading Labels of Train Data & Converting Words to Numeric Labels    \"\"\"\n",
        "myCSV = np.genfromtxt('/content/drive/MyDrive/train_labels.csv', delimiter=',', dtype='str')\n",
        "    # myCSV = np.genfromtxt('./all/train_labels.csv', delimiter=',', dtype = 'str')\n",
        "trainLabelWords = myCSV[1:, 1]  # Training labels: Words\n",
        "uniqueLabelWords = np.unique(trainLabelWords)  # Unique Labels\n",
        "trainLabel = np.zeros((trainSize, 1))  # Training Labels: Numerics\n",
        "NumberLabel = uniqueLabelWords.shape[0]\n",
        "refLabel = np.zeros((NumberLabel, 2))\n",
        "for i in range(NumberLabel):\n",
        "   trainLabel[trainLabelWords == uniqueLabelWords[i], 0] = i\n",
        "\n",
        "    "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hel3IElsbSw0"
      },
      "source": [
        "### In the following part, data are initialized and preprocessing process is performed to ehnace the model's accuracy."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w7EJwuteac8L",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "644351c8-8380-470f-85e5-eb344ca2b36a"
      },
      "source": [
        "\"\"\" Parameter Initialization \"\"\"\n",
        "filteredImgSize = 40\n",
        "binarizingTH = 8  # for first segmentation\n",
        "\n",
        "processedImgTrain1 = PreProcessing_1(dataTrain, filteredImgSize, binarizingTH)\n",
        "processedImgTest1 = PreProcessing_1(dataTest, filteredImgSize, binarizingTH)\n",
        "\n",
        "binarizingTH = 0.01  # range (0,1)\n",
        "featMatrixTrain = 1 * (processedImgTrain1 > binarizingTH)\n",
        "featMatrixTest  = 1 * (processedImgTest1>binarizingTH)\n",
        "featMatrixTrain = featMatrixTrain.astype('float32')\n",
        "featMatrixTest = featMatrixTest.astype('float32')\n",
        "\n",
        "\n",
        "# fix random seed for reproducibility\n",
        "seed = 7\n",
        "np.random.seed(seed)\n",
        "\n",
        "\n",
        "\"\"\"  Extracting Train & Test Sets Sizes and Original Feature & Image Sizes  \"\"\"\n",
        "trainSize = featMatrixTrain.shape[0]               #Number of Training Examples\n",
        "testSize  = featMatrixTest.shape[0]                #Number of Testing Examples\n",
        "featureSize = featMatrixTrain.shape[1]             #Number of Features\n",
        "\n",
        "\n",
        "nTrain = int(0.9*trainSize)\n",
        "idxTrainValid = np.random.choice(trainSize, [trainSize,1],replace = False)\n",
        "\n",
        "t_train = trainLabel[idxTrainValid[:nTrain],0]\n",
        "t_valid = trainLabel[idxTrainValid[nTrain:],0]\n",
        "X_train = featMatrixTrain[idxTrainValid[:nTrain,0],:]\n",
        "X_valid = featMatrixTrain[idxTrainValid[nTrain:,0],:]\n",
        "X_test = featMatrixTest\n",
        "\n",
        "X_train = (X_train.reshape(X_train.shape[0], 1,filteredImgSize, filteredImgSize))\n",
        "#X_train = np.expand_dims(X_train, axis=-1)\n",
        "X_valid = (X_valid.reshape(X_valid.shape[0], 1,filteredImgSize, filteredImgSize))\n",
        "#X_valid = np.expand_dims(X_valid, axis=-1)\n",
        "X_test = (X_test.reshape(X_test.shape[0], 1,filteredImgSize, filteredImgSize))\n",
        "#X_test = np.expand_dims(X_test, axis=-1)\n",
        "    #one-hot encode target column\n",
        "y_train = to_categorical(t_train)\n",
        "y_valid = to_categorical(t_valid)\n",
        "\n",
        "num_classes = y_valid.shape[1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "999  of data have been processed\n",
            "1999  of data have been processed\n",
            "2999  of data have been processed\n",
            "3999  of data have been processed\n",
            "4999  of data have been processed\n",
            "5999  of data have been processed\n",
            "6999  of data have been processed\n",
            "7999  of data have been processed\n",
            "8999  of data have been processed\n",
            "9999  of data have been processed\n",
            "999  of data have been processed\n",
            "1999  of data have been processed\n",
            "2999  of data have been processed\n",
            "3999  of data have been processed\n",
            "4999  of data have been processed\n",
            "5999  of data have been processed\n",
            "6999  of data have been processed\n",
            "7999  of data have been processed\n",
            "8999  of data have been processed\n",
            "9999  of data have been processed\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5Z9ix-Jbbw2E"
      },
      "source": [
        "## Convolutional Neural Network is defined in the following part."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "erq7x8YqfY-t"
      },
      "source": [
        "def CNN_model(filteredImgSize=40):\n",
        "    model = Sequential([\n",
        "        # First two convolutional layers\n",
        "        Conv2D(32, kernel_size=(3, 3), activation='relu', padding='same', input_shape=(1, filteredImgSize, filteredImgSize)),\n",
        "        Conv2D(32, kernel_size=(3, 3), activation='relu', padding='same'),\n",
        "        # normalization layer\n",
        "        BatchNormalization(),\n",
        "        # pooling layer\n",
        "        MaxPooling2D(pool_size=(2, 2)),\n",
        "        # add regularization\n",
        "        Dropout(0.25),\n",
        "        # Second two convolutional layers\n",
        "        Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same'),\n",
        "        Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same'),\n",
        "        # normalization layer\n",
        "        BatchNormalization(),\n",
        "        # pooling layer\n",
        "        MaxPooling2D(pool_size=(2, 2)),\n",
        "        # add regularization\n",
        "        Dropout(0.25),\n",
        "\n",
        "        Flatten(),\n",
        "\n",
        "        # FC layer\n",
        "        Dense(1024, activation='relu'),\n",
        "        BatchNormalization(),\n",
        "        Dropout(0.5),\n",
        "        Dense(1024, activation='relu'),\n",
        "        BatchNormalization(),\n",
        "        Dropout(0.5),\n",
        "        Dense(1024, activation='relu'),\n",
        "        BatchNormalization(),\n",
        "        Dropout(0.5),\n",
        "        Dense(31, activation='softmax')\n",
        "    ])\n",
        "    model.compile(loss=keras.losses.categorical_crossentropy,\n",
        "                  optimizer=keras.optimizers.Adam(lr=0.0004),\n",
        "                  metrics=['accuracy'])\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bVlqu8mIb8TS"
      },
      "source": [
        "# CNN model is run, and its accuracy for training and validation data is reported for each epoch."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "5cAzOrJW36Ci",
        "outputId": "6e720f18-f67e-4eec-b8ee-cd3d60d492f8"
      },
      "source": [
        "from keras import backend as K\n",
        "K.set_image_data_format('channels_first')\n",
        "\n",
        "modelBest=CNN_model()\n",
        "\n",
        "batch_size=35\n",
        "epoch_aug1=700\n",
        "\n",
        "#earlyStopping = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0.001, patience=30, verbose=0, mode='min')\n",
        "\n",
        "gen = ImageDataGenerator(rotation_range=15, width_shift_range=0.1, shear_range=0.3, \n",
        "                         height_shift_range=0.1, zoom_range=0.1)\n",
        "\n",
        "\n",
        "\n",
        "batches = gen.flow(X_train, y_train, batch_size=batch_size)\n",
        "val_batches = gen.flow(X_valid, y_valid, batch_size=batch_size)\n",
        "     \n",
        "\n",
        "results=modelBest.fit_generator(batches, steps_per_epoch=X_train.shape[0] // batch_size, epochs=epoch_aug1,\n",
        "                                validation_data=val_batches, validation_steps=X_valid.shape[0] // batch_size,\n",
        "                                use_multiprocessing=False)\n",
        "\n",
        "\n",
        "#results = modelBest.fit_generator(X, Y, validation_split=0.1, batch_size=55, epochs=100)\n",
        "\n",
        "plt.figure()\n",
        "plt.plot(results.history['accuracy'], label=\"Train\")\n",
        "plt.plot(results.history['val_accuracy'], label=\"Validation\")\n",
        "plt.legend(fontsize=25)\n",
        "plt.grid(True)\n",
        "plt.show()\n",
        "\n",
        "\n",
        "# The final accuracy of CNN for training and validation data is reported in the following part.\n",
        "scores = modelBest.evaluate(X_train, y_train, verbose=1)\n",
        "print(\"Large CNN Train Error: %.2f%%\" % (100-scores[1]*100))\n",
        "scores = modelBest.evaluate(X_valid, y_valid, verbose=1)\n",
        "print(\"Large CNN Valid Error: %.2f%%\" % (100-scores[1]*100))\n",
        "\n",
        "\n",
        "\n",
        "#predict images in the test set\n",
        "y_test_CNN = modelBest.predict(X_test)\n",
        "t_test_CNN = np.argmax(y_test_CNN,axis=1)\n",
        "\n",
        "testLabelCNN = np.zeros((testSize,2)).astype('str')\n",
        "\n",
        "# map the predict result to classes name\n",
        "for i in range(NumberLabel):\n",
        "  testLabelCNN[t_test_CNN == i,1] = uniqueLabelWords[i]\n",
        "\n",
        "testLabelCNN[:,0]=range(10000)\n",
        "\n",
        "# The testing data labels are predicted in the following part, and the predicted labels are saved in a csv file in order to be uploaded in Kaggle website.  \n",
        "test=pd.DataFrame(testLabelCNN,columns=['Id','Category']).set_index('Id')\n",
        "test.to_csv('7810.csv')\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 3.7987 - accuracy: 0.1499 - val_loss: 14.4877 - val_accuracy: 0.0323\n",
            "Epoch 2/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 3.2549 - accuracy: 0.2007 - val_loss: 5.5009 - val_accuracy: 0.0697\n",
            "Epoch 3/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 2.8838 - accuracy: 0.2493 - val_loss: 3.0986 - val_accuracy: 0.2566\n",
            "Epoch 4/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 2.5910 - accuracy: 0.3036 - val_loss: 2.2047 - val_accuracy: 0.3808\n",
            "Epoch 5/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 2.3286 - accuracy: 0.3596 - val_loss: 2.1884 - val_accuracy: 0.3657\n",
            "Epoch 6/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 2.1957 - accuracy: 0.3824 - val_loss: 2.1805 - val_accuracy: 0.4111\n",
            "Epoch 7/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 2.1040 - accuracy: 0.4047 - val_loss: 1.8079 - val_accuracy: 0.4909\n",
            "Epoch 8/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 2.0070 - accuracy: 0.4378 - val_loss: 2.2785 - val_accuracy: 0.3939\n",
            "Epoch 9/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.9432 - accuracy: 0.4519 - val_loss: 1.9808 - val_accuracy: 0.4434\n",
            "Epoch 10/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.8722 - accuracy: 0.4653 - val_loss: 1.6973 - val_accuracy: 0.5222\n",
            "Epoch 11/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.8112 - accuracy: 0.4863 - val_loss: 1.6207 - val_accuracy: 0.5192\n",
            "Epoch 12/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.7485 - accuracy: 0.4982 - val_loss: 1.5376 - val_accuracy: 0.5636\n",
            "Epoch 13/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 1.6902 - accuracy: 0.5123 - val_loss: 1.5135 - val_accuracy: 0.5697\n",
            "Epoch 14/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 1.6491 - accuracy: 0.5257 - val_loss: 1.5827 - val_accuracy: 0.5434\n",
            "Epoch 15/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.6284 - accuracy: 0.5303 - val_loss: 1.3429 - val_accuracy: 0.6121\n",
            "Epoch 16/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 1.5816 - accuracy: 0.5404 - val_loss: 1.6567 - val_accuracy: 0.5313\n",
            "Epoch 17/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.5225 - accuracy: 0.5608 - val_loss: 1.3622 - val_accuracy: 0.6162\n",
            "Epoch 18/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.5201 - accuracy: 0.5627 - val_loss: 1.4302 - val_accuracy: 0.5889\n",
            "Epoch 19/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.4717 - accuracy: 0.5743 - val_loss: 1.2861 - val_accuracy: 0.6293\n",
            "Epoch 20/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.4588 - accuracy: 0.5794 - val_loss: 1.3752 - val_accuracy: 0.6222\n",
            "Epoch 21/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.4413 - accuracy: 0.5828 - val_loss: 1.3430 - val_accuracy: 0.6333\n",
            "Epoch 22/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.4118 - accuracy: 0.5903 - val_loss: 1.4042 - val_accuracy: 0.5919\n",
            "Epoch 23/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 1.3808 - accuracy: 0.6040 - val_loss: 1.2136 - val_accuracy: 0.6606\n",
            "Epoch 24/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.3696 - accuracy: 0.6062 - val_loss: 1.3034 - val_accuracy: 0.6091\n",
            "Epoch 25/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.3485 - accuracy: 0.6080 - val_loss: 1.2128 - val_accuracy: 0.6434\n",
            "Epoch 26/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.3299 - accuracy: 0.6132 - val_loss: 1.1928 - val_accuracy: 0.6596\n",
            "Epoch 27/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 1.3166 - accuracy: 0.6148 - val_loss: 1.2091 - val_accuracy: 0.6535\n",
            "Epoch 28/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 1.2985 - accuracy: 0.6214 - val_loss: 1.1324 - val_accuracy: 0.6768\n",
            "Epoch 29/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.2789 - accuracy: 0.6273 - val_loss: 1.3018 - val_accuracy: 0.6212\n",
            "Epoch 30/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 1.2431 - accuracy: 0.6402 - val_loss: 1.1519 - val_accuracy: 0.6707\n",
            "Epoch 31/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 1.2261 - accuracy: 0.6426 - val_loss: 1.1785 - val_accuracy: 0.6646\n",
            "Epoch 32/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 1.2088 - accuracy: 0.6488 - val_loss: 1.1078 - val_accuracy: 0.6949\n",
            "Epoch 33/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 1.1717 - accuracy: 0.6606 - val_loss: 1.1753 - val_accuracy: 0.6535\n",
            "Epoch 34/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.1812 - accuracy: 0.6537 - val_loss: 1.1001 - val_accuracy: 0.6919\n",
            "Epoch 35/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.1705 - accuracy: 0.6591 - val_loss: 1.0506 - val_accuracy: 0.6919\n",
            "Epoch 36/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.1576 - accuracy: 0.6646 - val_loss: 1.1274 - val_accuracy: 0.6889\n",
            "Epoch 37/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.1514 - accuracy: 0.6654 - val_loss: 1.1753 - val_accuracy: 0.6727\n",
            "Epoch 38/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.1340 - accuracy: 0.6671 - val_loss: 1.0875 - val_accuracy: 0.6939\n",
            "Epoch 39/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 1.1153 - accuracy: 0.6703 - val_loss: 1.1268 - val_accuracy: 0.6828\n",
            "Epoch 40/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.1204 - accuracy: 0.6703 - val_loss: 1.0309 - val_accuracy: 0.7111\n",
            "Epoch 41/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.1128 - accuracy: 0.6729 - val_loss: 1.0414 - val_accuracy: 0.7101\n",
            "Epoch 42/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.1061 - accuracy: 0.6786 - val_loss: 1.1098 - val_accuracy: 0.6949\n",
            "Epoch 43/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.0844 - accuracy: 0.6803 - val_loss: 1.0257 - val_accuracy: 0.7040\n",
            "Epoch 44/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.0755 - accuracy: 0.6818 - val_loss: 1.0735 - val_accuracy: 0.7010\n",
            "Epoch 45/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 1.0371 - accuracy: 0.6887 - val_loss: 1.0249 - val_accuracy: 0.7111\n",
            "Epoch 46/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.0620 - accuracy: 0.6830 - val_loss: 1.0409 - val_accuracy: 0.7020\n",
            "Epoch 47/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.0263 - accuracy: 0.6874 - val_loss: 0.9765 - val_accuracy: 0.7202\n",
            "Epoch 48/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.0338 - accuracy: 0.6981 - val_loss: 1.0598 - val_accuracy: 0.7020\n",
            "Epoch 49/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 1.0070 - accuracy: 0.7046 - val_loss: 0.9971 - val_accuracy: 0.7354\n",
            "Epoch 50/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.0214 - accuracy: 0.7011 - val_loss: 1.0486 - val_accuracy: 0.7020\n",
            "Epoch 51/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 1.0086 - accuracy: 0.7033 - val_loss: 0.9857 - val_accuracy: 0.7182\n",
            "Epoch 52/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.9970 - accuracy: 0.7073 - val_loss: 1.0084 - val_accuracy: 0.7101\n",
            "Epoch 53/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.9844 - accuracy: 0.7091 - val_loss: 1.0088 - val_accuracy: 0.7333\n",
            "Epoch 54/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.9678 - accuracy: 0.7147 - val_loss: 1.0280 - val_accuracy: 0.7172\n",
            "Epoch 55/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.9722 - accuracy: 0.7099 - val_loss: 1.0069 - val_accuracy: 0.7293\n",
            "Epoch 56/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.9699 - accuracy: 0.7088 - val_loss: 1.0025 - val_accuracy: 0.7242\n",
            "Epoch 57/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.9464 - accuracy: 0.7156 - val_loss: 1.0364 - val_accuracy: 0.7232\n",
            "Epoch 58/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.9275 - accuracy: 0.7250 - val_loss: 1.0113 - val_accuracy: 0.7343\n",
            "Epoch 59/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.9435 - accuracy: 0.7234 - val_loss: 0.9983 - val_accuracy: 0.7202\n",
            "Epoch 60/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.9351 - accuracy: 0.7239 - val_loss: 0.9751 - val_accuracy: 0.7384\n",
            "Epoch 61/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.9378 - accuracy: 0.7272 - val_loss: 0.9972 - val_accuracy: 0.7242\n",
            "Epoch 62/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.9012 - accuracy: 0.7334 - val_loss: 0.9765 - val_accuracy: 0.7323\n",
            "Epoch 63/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.9199 - accuracy: 0.7262 - val_loss: 0.9170 - val_accuracy: 0.7343\n",
            "Epoch 64/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.9167 - accuracy: 0.7246 - val_loss: 0.9530 - val_accuracy: 0.7293\n",
            "Epoch 65/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.8964 - accuracy: 0.7342 - val_loss: 0.9475 - val_accuracy: 0.7384\n",
            "Epoch 66/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.8826 - accuracy: 0.7329 - val_loss: 1.2185 - val_accuracy: 0.6838\n",
            "Epoch 67/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.8920 - accuracy: 0.7386 - val_loss: 0.9799 - val_accuracy: 0.7323\n",
            "Epoch 68/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.8524 - accuracy: 0.7417 - val_loss: 0.9212 - val_accuracy: 0.7404\n",
            "Epoch 69/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.8691 - accuracy: 0.7382 - val_loss: 0.9661 - val_accuracy: 0.7323\n",
            "Epoch 70/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.8554 - accuracy: 0.7456 - val_loss: 1.0380 - val_accuracy: 0.7263\n",
            "Epoch 71/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.8580 - accuracy: 0.7414 - val_loss: 0.9517 - val_accuracy: 0.7434\n",
            "Epoch 72/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.8583 - accuracy: 0.7404 - val_loss: 0.9871 - val_accuracy: 0.7273\n",
            "Epoch 73/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.8518 - accuracy: 0.7476 - val_loss: 0.9535 - val_accuracy: 0.7465\n",
            "Epoch 74/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.8595 - accuracy: 0.7436 - val_loss: 1.0256 - val_accuracy: 0.7152\n",
            "Epoch 75/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.8447 - accuracy: 0.7484 - val_loss: 0.9340 - val_accuracy: 0.7455\n",
            "Epoch 76/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.8256 - accuracy: 0.7492 - val_loss: 0.9353 - val_accuracy: 0.7525\n",
            "Epoch 77/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.8196 - accuracy: 0.7511 - val_loss: 0.9257 - val_accuracy: 0.7465\n",
            "Epoch 78/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.8315 - accuracy: 0.7493 - val_loss: 0.9798 - val_accuracy: 0.7172\n",
            "Epoch 79/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.8129 - accuracy: 0.7552 - val_loss: 0.9744 - val_accuracy: 0.7263\n",
            "Epoch 80/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7995 - accuracy: 0.7622 - val_loss: 0.9753 - val_accuracy: 0.7293\n",
            "Epoch 81/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.8089 - accuracy: 0.7569 - val_loss: 1.0025 - val_accuracy: 0.7354\n",
            "Epoch 82/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.8228 - accuracy: 0.7538 - val_loss: 0.9715 - val_accuracy: 0.7303\n",
            "Epoch 83/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7934 - accuracy: 0.7602 - val_loss: 0.9648 - val_accuracy: 0.7333\n",
            "Epoch 84/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7908 - accuracy: 0.7583 - val_loss: 0.9835 - val_accuracy: 0.7394\n",
            "Epoch 85/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.7664 - accuracy: 0.7649 - val_loss: 0.8896 - val_accuracy: 0.7525\n",
            "Epoch 86/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7843 - accuracy: 0.7636 - val_loss: 0.9697 - val_accuracy: 0.7545\n",
            "Epoch 87/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.7811 - accuracy: 0.7611 - val_loss: 0.9246 - val_accuracy: 0.7606\n",
            "Epoch 88/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7706 - accuracy: 0.7677 - val_loss: 0.9038 - val_accuracy: 0.7606\n",
            "Epoch 89/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.7677 - accuracy: 0.7681 - val_loss: 0.9137 - val_accuracy: 0.7626\n",
            "Epoch 90/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7638 - accuracy: 0.7650 - val_loss: 0.9322 - val_accuracy: 0.7505\n",
            "Epoch 91/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7594 - accuracy: 0.7707 - val_loss: 0.9328 - val_accuracy: 0.7414\n",
            "Epoch 92/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7865 - accuracy: 0.7661 - val_loss: 1.0203 - val_accuracy: 0.7323\n",
            "Epoch 93/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7857 - accuracy: 0.7632 - val_loss: 0.9169 - val_accuracy: 0.7485\n",
            "Epoch 94/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.7562 - accuracy: 0.7677 - val_loss: 0.9632 - val_accuracy: 0.7424\n",
            "Epoch 95/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.7306 - accuracy: 0.7772 - val_loss: 0.9110 - val_accuracy: 0.7475\n",
            "Epoch 96/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.7129 - accuracy: 0.7839 - val_loss: 0.9861 - val_accuracy: 0.7485\n",
            "Epoch 97/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7369 - accuracy: 0.7764 - val_loss: 0.9273 - val_accuracy: 0.7566\n",
            "Epoch 98/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7343 - accuracy: 0.7778 - val_loss: 0.8914 - val_accuracy: 0.7626\n",
            "Epoch 99/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7177 - accuracy: 0.7803 - val_loss: 0.9632 - val_accuracy: 0.7495\n",
            "Epoch 100/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7239 - accuracy: 0.7798 - val_loss: 0.9755 - val_accuracy: 0.7404\n",
            "Epoch 101/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7111 - accuracy: 0.7822 - val_loss: 0.9291 - val_accuracy: 0.7636\n",
            "Epoch 102/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7071 - accuracy: 0.7776 - val_loss: 0.9084 - val_accuracy: 0.7606\n",
            "Epoch 103/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7065 - accuracy: 0.7823 - val_loss: 0.9532 - val_accuracy: 0.7434\n",
            "Epoch 104/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7191 - accuracy: 0.7797 - val_loss: 0.9689 - val_accuracy: 0.7505\n",
            "Epoch 105/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7174 - accuracy: 0.7770 - val_loss: 0.9413 - val_accuracy: 0.7545\n",
            "Epoch 106/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7055 - accuracy: 0.7822 - val_loss: 0.9049 - val_accuracy: 0.7707\n",
            "Epoch 107/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7141 - accuracy: 0.7787 - val_loss: 0.9324 - val_accuracy: 0.7657\n",
            "Epoch 108/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.7074 - accuracy: 0.7899 - val_loss: 0.8974 - val_accuracy: 0.7465\n",
            "Epoch 109/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.6829 - accuracy: 0.7929 - val_loss: 0.9281 - val_accuracy: 0.7576\n",
            "Epoch 110/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6854 - accuracy: 0.7884 - val_loss: 0.8954 - val_accuracy: 0.7667\n",
            "Epoch 111/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6899 - accuracy: 0.7909 - val_loss: 0.9696 - val_accuracy: 0.7475\n",
            "Epoch 112/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6805 - accuracy: 0.7931 - val_loss: 0.9424 - val_accuracy: 0.7646\n",
            "Epoch 113/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6725 - accuracy: 0.7917 - val_loss: 0.9464 - val_accuracy: 0.7455\n",
            "Epoch 114/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6696 - accuracy: 0.7897 - val_loss: 0.9428 - val_accuracy: 0.7525\n",
            "Epoch 115/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6728 - accuracy: 0.7914 - val_loss: 0.9352 - val_accuracy: 0.7525\n",
            "Epoch 116/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6771 - accuracy: 0.7941 - val_loss: 0.8947 - val_accuracy: 0.7707\n",
            "Epoch 117/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6640 - accuracy: 0.7916 - val_loss: 0.8662 - val_accuracy: 0.7697\n",
            "Epoch 118/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6571 - accuracy: 0.7978 - val_loss: 0.9326 - val_accuracy: 0.7646\n",
            "Epoch 119/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6500 - accuracy: 0.7977 - val_loss: 0.9496 - val_accuracy: 0.7737\n",
            "Epoch 120/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6635 - accuracy: 0.7938 - val_loss: 0.9761 - val_accuracy: 0.7596\n",
            "Epoch 121/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6550 - accuracy: 0.8011 - val_loss: 0.9305 - val_accuracy: 0.7556\n",
            "Epoch 122/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6430 - accuracy: 0.8026 - val_loss: 0.8657 - val_accuracy: 0.7747\n",
            "Epoch 123/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6467 - accuracy: 0.7964 - val_loss: 0.8819 - val_accuracy: 0.7616\n",
            "Epoch 124/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6434 - accuracy: 0.7999 - val_loss: 0.9226 - val_accuracy: 0.7465\n",
            "Epoch 125/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6540 - accuracy: 0.7953 - val_loss: 0.8825 - val_accuracy: 0.7707\n",
            "Epoch 126/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6348 - accuracy: 0.8026 - val_loss: 0.9228 - val_accuracy: 0.7747\n",
            "Epoch 127/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6474 - accuracy: 0.7966 - val_loss: 0.9236 - val_accuracy: 0.7657\n",
            "Epoch 128/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6329 - accuracy: 0.8019 - val_loss: 0.9452 - val_accuracy: 0.7525\n",
            "Epoch 129/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6292 - accuracy: 0.8014 - val_loss: 0.9384 - val_accuracy: 0.7616\n",
            "Epoch 130/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6495 - accuracy: 0.7991 - val_loss: 0.9616 - val_accuracy: 0.7576\n",
            "Epoch 131/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6263 - accuracy: 0.8104 - val_loss: 0.9409 - val_accuracy: 0.7525\n",
            "Epoch 132/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6198 - accuracy: 0.8058 - val_loss: 0.9706 - val_accuracy: 0.7455\n",
            "Epoch 133/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6113 - accuracy: 0.8104 - val_loss: 0.9790 - val_accuracy: 0.7414\n",
            "Epoch 134/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6254 - accuracy: 0.8028 - val_loss: 0.9157 - val_accuracy: 0.7545\n",
            "Epoch 135/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6105 - accuracy: 0.8072 - val_loss: 0.9301 - val_accuracy: 0.7636\n",
            "Epoch 136/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6024 - accuracy: 0.8146 - val_loss: 0.9706 - val_accuracy: 0.7616\n",
            "Epoch 137/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6119 - accuracy: 0.8087 - val_loss: 0.9394 - val_accuracy: 0.7626\n",
            "Epoch 138/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6292 - accuracy: 0.8041 - val_loss: 0.8793 - val_accuracy: 0.7616\n",
            "Epoch 139/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6196 - accuracy: 0.8092 - val_loss: 0.9247 - val_accuracy: 0.7545\n",
            "Epoch 140/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6029 - accuracy: 0.8146 - val_loss: 1.0906 - val_accuracy: 0.7242\n",
            "Epoch 141/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.6080 - accuracy: 0.8096 - val_loss: 0.9750 - val_accuracy: 0.7535\n",
            "Epoch 142/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5981 - accuracy: 0.8087 - val_loss: 0.9275 - val_accuracy: 0.7697\n",
            "Epoch 143/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5900 - accuracy: 0.8206 - val_loss: 0.8773 - val_accuracy: 0.7778\n",
            "Epoch 144/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5827 - accuracy: 0.8207 - val_loss: 0.9573 - val_accuracy: 0.7505\n",
            "Epoch 145/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5745 - accuracy: 0.8211 - val_loss: 0.8920 - val_accuracy: 0.7707\n",
            "Epoch 146/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5923 - accuracy: 0.8124 - val_loss: 0.9780 - val_accuracy: 0.7475\n",
            "Epoch 147/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5882 - accuracy: 0.8153 - val_loss: 0.9333 - val_accuracy: 0.7556\n",
            "Epoch 148/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5722 - accuracy: 0.8178 - val_loss: 0.9358 - val_accuracy: 0.7626\n",
            "Epoch 149/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5842 - accuracy: 0.8170 - val_loss: 0.9522 - val_accuracy: 0.7667\n",
            "Epoch 150/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5788 - accuracy: 0.8160 - val_loss: 0.9708 - val_accuracy: 0.7626\n",
            "Epoch 151/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5800 - accuracy: 0.8133 - val_loss: 0.9262 - val_accuracy: 0.7707\n",
            "Epoch 152/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5889 - accuracy: 0.8164 - val_loss: 0.9063 - val_accuracy: 0.7879\n",
            "Epoch 153/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5706 - accuracy: 0.8231 - val_loss: 0.8829 - val_accuracy: 0.7768\n",
            "Epoch 154/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.5672 - accuracy: 0.8260 - val_loss: 0.9089 - val_accuracy: 0.7717\n",
            "Epoch 155/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.5531 - accuracy: 0.8272 - val_loss: 0.9128 - val_accuracy: 0.7798\n",
            "Epoch 156/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.5646 - accuracy: 0.8171 - val_loss: 0.9648 - val_accuracy: 0.7545\n",
            "Epoch 157/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.5636 - accuracy: 0.8230 - val_loss: 0.8920 - val_accuracy: 0.7758\n",
            "Epoch 158/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.5549 - accuracy: 0.8231 - val_loss: 0.8940 - val_accuracy: 0.7616\n",
            "Epoch 159/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.5430 - accuracy: 0.8286 - val_loss: 0.9597 - val_accuracy: 0.7636\n",
            "Epoch 160/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5497 - accuracy: 0.8256 - val_loss: 0.9561 - val_accuracy: 0.7717\n",
            "Epoch 161/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5466 - accuracy: 0.8261 - val_loss: 0.9098 - val_accuracy: 0.7657\n",
            "Epoch 162/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.5298 - accuracy: 0.8310 - val_loss: 0.9325 - val_accuracy: 0.7556\n",
            "Epoch 163/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.5806 - accuracy: 0.8181 - val_loss: 0.9063 - val_accuracy: 0.7667\n",
            "Epoch 164/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5603 - accuracy: 0.8246 - val_loss: 0.9413 - val_accuracy: 0.7677\n",
            "Epoch 165/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.5576 - accuracy: 0.8270 - val_loss: 0.9332 - val_accuracy: 0.7576\n",
            "Epoch 166/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.5540 - accuracy: 0.8314 - val_loss: 0.8989 - val_accuracy: 0.7667\n",
            "Epoch 167/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.5306 - accuracy: 0.8367 - val_loss: 0.9705 - val_accuracy: 0.7616\n",
            "Epoch 168/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5309 - accuracy: 0.8321 - val_loss: 0.9223 - val_accuracy: 0.7586\n",
            "Epoch 169/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5251 - accuracy: 0.8342 - val_loss: 0.9395 - val_accuracy: 0.7707\n",
            "Epoch 170/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5399 - accuracy: 0.8262 - val_loss: 0.9328 - val_accuracy: 0.7586\n",
            "Epoch 171/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5519 - accuracy: 0.8291 - val_loss: 0.9336 - val_accuracy: 0.7818\n",
            "Epoch 172/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.5330 - accuracy: 0.8303 - val_loss: 0.9576 - val_accuracy: 0.7525\n",
            "Epoch 173/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.5298 - accuracy: 0.8336 - val_loss: 0.9517 - val_accuracy: 0.7545\n",
            "Epoch 174/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.5084 - accuracy: 0.8364 - val_loss: 0.9568 - val_accuracy: 0.7677\n",
            "Epoch 175/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5531 - accuracy: 0.8246 - val_loss: 0.8988 - val_accuracy: 0.7737\n",
            "Epoch 176/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5157 - accuracy: 0.8388 - val_loss: 0.8987 - val_accuracy: 0.7727\n",
            "Epoch 177/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5348 - accuracy: 0.8332 - val_loss: 0.9412 - val_accuracy: 0.7616\n",
            "Epoch 178/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.5106 - accuracy: 0.8402 - val_loss: 0.8913 - val_accuracy: 0.7838\n",
            "Epoch 179/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5223 - accuracy: 0.8373 - val_loss: 0.9647 - val_accuracy: 0.7636\n",
            "Epoch 180/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.5297 - accuracy: 0.8312 - val_loss: 0.9369 - val_accuracy: 0.7687\n",
            "Epoch 181/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5221 - accuracy: 0.8342 - val_loss: 0.9150 - val_accuracy: 0.7687\n",
            "Epoch 182/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5233 - accuracy: 0.8321 - val_loss: 0.9150 - val_accuracy: 0.7626\n",
            "Epoch 183/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5108 - accuracy: 0.8377 - val_loss: 0.9025 - val_accuracy: 0.7788\n",
            "Epoch 184/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5055 - accuracy: 0.8360 - val_loss: 0.9256 - val_accuracy: 0.7747\n",
            "Epoch 185/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5220 - accuracy: 0.8361 - val_loss: 0.9205 - val_accuracy: 0.7727\n",
            "Epoch 186/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5124 - accuracy: 0.8387 - val_loss: 0.9554 - val_accuracy: 0.7687\n",
            "Epoch 187/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5166 - accuracy: 0.8396 - val_loss: 0.9560 - val_accuracy: 0.7687\n",
            "Epoch 188/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5122 - accuracy: 0.8328 - val_loss: 0.9705 - val_accuracy: 0.7636\n",
            "Epoch 189/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5113 - accuracy: 0.8349 - val_loss: 0.9876 - val_accuracy: 0.7808\n",
            "Epoch 190/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5044 - accuracy: 0.8414 - val_loss: 0.9414 - val_accuracy: 0.7828\n",
            "Epoch 191/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4993 - accuracy: 0.8426 - val_loss: 0.9851 - val_accuracy: 0.7596\n",
            "Epoch 192/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5066 - accuracy: 0.8344 - val_loss: 0.9131 - val_accuracy: 0.7586\n",
            "Epoch 193/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5081 - accuracy: 0.8400 - val_loss: 0.9844 - val_accuracy: 0.7596\n",
            "Epoch 194/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4864 - accuracy: 0.8411 - val_loss: 0.9944 - val_accuracy: 0.7475\n",
            "Epoch 195/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.4935 - accuracy: 0.8410 - val_loss: 0.9877 - val_accuracy: 0.7636\n",
            "Epoch 196/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4983 - accuracy: 0.8432 - val_loss: 0.9016 - val_accuracy: 0.7737\n",
            "Epoch 197/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.4762 - accuracy: 0.8516 - val_loss: 0.9937 - val_accuracy: 0.7677\n",
            "Epoch 198/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.4887 - accuracy: 0.8447 - val_loss: 0.9726 - val_accuracy: 0.7657\n",
            "Epoch 199/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4900 - accuracy: 0.8449 - val_loss: 0.9268 - val_accuracy: 0.7576\n",
            "Epoch 200/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4788 - accuracy: 0.8507 - val_loss: 0.9345 - val_accuracy: 0.7808\n",
            "Epoch 201/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4846 - accuracy: 0.8472 - val_loss: 0.9177 - val_accuracy: 0.7657\n",
            "Epoch 202/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4897 - accuracy: 0.8442 - val_loss: 0.9430 - val_accuracy: 0.7697\n",
            "Epoch 203/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.4826 - accuracy: 0.8438 - val_loss: 0.9506 - val_accuracy: 0.7687\n",
            "Epoch 204/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.4788 - accuracy: 0.8491 - val_loss: 0.9246 - val_accuracy: 0.7747\n",
            "Epoch 205/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4882 - accuracy: 0.8488 - val_loss: 1.0028 - val_accuracy: 0.7636\n",
            "Epoch 206/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4677 - accuracy: 0.8529 - val_loss: 0.9231 - val_accuracy: 0.7778\n",
            "Epoch 207/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4860 - accuracy: 0.8454 - val_loss: 0.9389 - val_accuracy: 0.7677\n",
            "Epoch 208/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4717 - accuracy: 0.8513 - val_loss: 0.9501 - val_accuracy: 0.7768\n",
            "Epoch 209/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4713 - accuracy: 0.8504 - val_loss: 0.9736 - val_accuracy: 0.7616\n",
            "Epoch 210/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5049 - accuracy: 0.8389 - val_loss: 0.9607 - val_accuracy: 0.7667\n",
            "Epoch 211/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4836 - accuracy: 0.8459 - val_loss: 0.9439 - val_accuracy: 0.7626\n",
            "Epoch 212/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4785 - accuracy: 0.8489 - val_loss: 0.9573 - val_accuracy: 0.7636\n",
            "Epoch 213/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.5003 - accuracy: 0.8420 - val_loss: 0.9434 - val_accuracy: 0.7768\n",
            "Epoch 214/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.4653 - accuracy: 0.8487 - val_loss: 0.9638 - val_accuracy: 0.7566\n",
            "Epoch 215/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4905 - accuracy: 0.8468 - val_loss: 0.9092 - val_accuracy: 0.7798\n",
            "Epoch 216/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.4635 - accuracy: 0.8534 - val_loss: 0.9517 - val_accuracy: 0.7717\n",
            "Epoch 217/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4620 - accuracy: 0.8582 - val_loss: 0.9171 - val_accuracy: 0.7717\n",
            "Epoch 218/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.4453 - accuracy: 0.8617 - val_loss: 0.9104 - val_accuracy: 0.7727\n",
            "Epoch 219/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.4675 - accuracy: 0.8510 - val_loss: 0.9281 - val_accuracy: 0.7788\n",
            "Epoch 220/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4676 - accuracy: 0.8492 - val_loss: 0.9633 - val_accuracy: 0.7667\n",
            "Epoch 221/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4697 - accuracy: 0.8473 - val_loss: 0.9758 - val_accuracy: 0.7818\n",
            "Epoch 222/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4503 - accuracy: 0.8541 - val_loss: 0.9056 - val_accuracy: 0.7808\n",
            "Epoch 223/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4538 - accuracy: 0.8528 - val_loss: 0.9724 - val_accuracy: 0.7646\n",
            "Epoch 224/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4622 - accuracy: 0.8533 - val_loss: 0.9632 - val_accuracy: 0.7556\n",
            "Epoch 225/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4775 - accuracy: 0.8492 - val_loss: 0.9899 - val_accuracy: 0.7505\n",
            "Epoch 226/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4513 - accuracy: 0.8577 - val_loss: 0.9341 - val_accuracy: 0.7707\n",
            "Epoch 227/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4475 - accuracy: 0.8552 - val_loss: 1.0296 - val_accuracy: 0.7545\n",
            "Epoch 228/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4287 - accuracy: 0.8640 - val_loss: 0.8791 - val_accuracy: 0.7828\n",
            "Epoch 229/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4533 - accuracy: 0.8561 - val_loss: 0.9302 - val_accuracy: 0.7737\n",
            "Epoch 230/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4413 - accuracy: 0.8613 - val_loss: 0.9562 - val_accuracy: 0.7758\n",
            "Epoch 231/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4748 - accuracy: 0.8482 - val_loss: 0.9368 - val_accuracy: 0.7727\n",
            "Epoch 232/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4553 - accuracy: 0.8538 - val_loss: 0.9605 - val_accuracy: 0.7566\n",
            "Epoch 233/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.4447 - accuracy: 0.8544 - val_loss: 0.9759 - val_accuracy: 0.7778\n",
            "Epoch 234/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4573 - accuracy: 0.8543 - val_loss: 0.9718 - val_accuracy: 0.7636\n",
            "Epoch 235/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4418 - accuracy: 0.8550 - val_loss: 0.9542 - val_accuracy: 0.7717\n",
            "Epoch 236/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4360 - accuracy: 0.8579 - val_loss: 0.9528 - val_accuracy: 0.7707\n",
            "Epoch 237/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4394 - accuracy: 0.8614 - val_loss: 0.9691 - val_accuracy: 0.7687\n",
            "Epoch 238/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4392 - accuracy: 0.8588 - val_loss: 0.9704 - val_accuracy: 0.7596\n",
            "Epoch 239/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4400 - accuracy: 0.8577 - val_loss: 0.9300 - val_accuracy: 0.7677\n",
            "Epoch 240/700\n",
            "300/300 [==============================] - 4s 13ms/step - loss: 0.4368 - accuracy: 0.8586 - val_loss: 1.0040 - val_accuracy: 0.7566\n",
            "Epoch 241/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4236 - accuracy: 0.8611 - val_loss: 0.9486 - val_accuracy: 0.7707\n",
            "Epoch 242/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4348 - accuracy: 0.8612 - val_loss: 0.9657 - val_accuracy: 0.7707\n",
            "Epoch 243/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4427 - accuracy: 0.8577 - val_loss: 0.9453 - val_accuracy: 0.7697\n",
            "Epoch 244/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4197 - accuracy: 0.8624 - val_loss: 0.9107 - val_accuracy: 0.7768\n",
            "Epoch 245/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4141 - accuracy: 0.8674 - val_loss: 0.9370 - val_accuracy: 0.7687\n",
            "Epoch 246/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4431 - accuracy: 0.8567 - val_loss: 1.0114 - val_accuracy: 0.7636\n",
            "Epoch 247/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4263 - accuracy: 0.8651 - val_loss: 0.9938 - val_accuracy: 0.7646\n",
            "Epoch 248/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4314 - accuracy: 0.8603 - val_loss: 0.9751 - val_accuracy: 0.7808\n",
            "Epoch 249/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4274 - accuracy: 0.8623 - val_loss: 0.9676 - val_accuracy: 0.7758\n",
            "Epoch 250/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4204 - accuracy: 0.8653 - val_loss: 0.9449 - val_accuracy: 0.7737\n",
            "Epoch 251/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4172 - accuracy: 0.8681 - val_loss: 0.9939 - val_accuracy: 0.7687\n",
            "Epoch 252/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4170 - accuracy: 0.8638 - val_loss: 1.0286 - val_accuracy: 0.7646\n",
            "Epoch 253/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4129 - accuracy: 0.8640 - val_loss: 1.0351 - val_accuracy: 0.7606\n",
            "Epoch 254/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4328 - accuracy: 0.8632 - val_loss: 0.9762 - val_accuracy: 0.7646\n",
            "Epoch 255/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4308 - accuracy: 0.8641 - val_loss: 0.9607 - val_accuracy: 0.7727\n",
            "Epoch 256/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4177 - accuracy: 0.8619 - val_loss: 0.9821 - val_accuracy: 0.7616\n",
            "Epoch 257/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.4059 - accuracy: 0.8681 - val_loss: 0.9645 - val_accuracy: 0.7727\n",
            "Epoch 258/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4170 - accuracy: 0.8639 - val_loss: 0.9577 - val_accuracy: 0.7677\n",
            "Epoch 259/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4189 - accuracy: 0.8608 - val_loss: 0.9896 - val_accuracy: 0.7727\n",
            "Epoch 260/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4114 - accuracy: 0.8672 - val_loss: 0.9912 - val_accuracy: 0.7667\n",
            "Epoch 261/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4216 - accuracy: 0.8628 - val_loss: 1.0314 - val_accuracy: 0.7677\n",
            "Epoch 262/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4116 - accuracy: 0.8669 - val_loss: 0.9576 - val_accuracy: 0.7626\n",
            "Epoch 263/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4228 - accuracy: 0.8660 - val_loss: 0.9526 - val_accuracy: 0.7687\n",
            "Epoch 264/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4269 - accuracy: 0.8603 - val_loss: 0.9834 - val_accuracy: 0.7737\n",
            "Epoch 265/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4143 - accuracy: 0.8711 - val_loss: 1.0408 - val_accuracy: 0.7556\n",
            "Epoch 266/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4025 - accuracy: 0.8681 - val_loss: 1.0319 - val_accuracy: 0.7616\n",
            "Epoch 267/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4045 - accuracy: 0.8698 - val_loss: 1.0050 - val_accuracy: 0.7616\n",
            "Epoch 268/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4232 - accuracy: 0.8644 - val_loss: 0.9634 - val_accuracy: 0.7788\n",
            "Epoch 269/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4042 - accuracy: 0.8699 - val_loss: 0.9982 - val_accuracy: 0.7556\n",
            "Epoch 270/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3931 - accuracy: 0.8733 - val_loss: 0.9591 - val_accuracy: 0.7707\n",
            "Epoch 271/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3975 - accuracy: 0.8661 - val_loss: 0.9931 - val_accuracy: 0.7596\n",
            "Epoch 272/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3982 - accuracy: 0.8703 - val_loss: 1.0153 - val_accuracy: 0.7616\n",
            "Epoch 273/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3971 - accuracy: 0.8749 - val_loss: 0.9329 - val_accuracy: 0.7778\n",
            "Epoch 274/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4144 - accuracy: 0.8631 - val_loss: 1.0158 - val_accuracy: 0.7646\n",
            "Epoch 275/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3912 - accuracy: 0.8757 - val_loss: 0.9697 - val_accuracy: 0.7848\n",
            "Epoch 276/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3803 - accuracy: 0.8762 - val_loss: 1.0074 - val_accuracy: 0.7707\n",
            "Epoch 277/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3910 - accuracy: 0.8693 - val_loss: 0.9746 - val_accuracy: 0.7707\n",
            "Epoch 278/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4031 - accuracy: 0.8693 - val_loss: 1.0319 - val_accuracy: 0.7535\n",
            "Epoch 279/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3744 - accuracy: 0.8780 - val_loss: 0.9847 - val_accuracy: 0.7707\n",
            "Epoch 280/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4027 - accuracy: 0.8694 - val_loss: 0.9877 - val_accuracy: 0.7677\n",
            "Epoch 281/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3846 - accuracy: 0.8766 - val_loss: 1.0765 - val_accuracy: 0.7364\n",
            "Epoch 282/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3902 - accuracy: 0.8742 - val_loss: 1.0129 - val_accuracy: 0.7626\n",
            "Epoch 283/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4017 - accuracy: 0.8690 - val_loss: 1.0209 - val_accuracy: 0.7576\n",
            "Epoch 284/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3870 - accuracy: 0.8759 - val_loss: 0.9178 - val_accuracy: 0.7818\n",
            "Epoch 285/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3808 - accuracy: 0.8768 - val_loss: 1.0036 - val_accuracy: 0.7657\n",
            "Epoch 286/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3760 - accuracy: 0.8777 - val_loss: 1.0311 - val_accuracy: 0.7556\n",
            "Epoch 287/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3970 - accuracy: 0.8716 - val_loss: 1.0430 - val_accuracy: 0.7495\n",
            "Epoch 288/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4022 - accuracy: 0.8699 - val_loss: 1.0312 - val_accuracy: 0.7616\n",
            "Epoch 289/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3982 - accuracy: 0.8702 - val_loss: 1.0179 - val_accuracy: 0.7646\n",
            "Epoch 290/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3816 - accuracy: 0.8762 - val_loss: 0.9896 - val_accuracy: 0.7596\n",
            "Epoch 291/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3821 - accuracy: 0.8752 - val_loss: 1.0577 - val_accuracy: 0.7586\n",
            "Epoch 292/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3963 - accuracy: 0.8699 - val_loss: 0.9786 - val_accuracy: 0.7747\n",
            "Epoch 293/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.4008 - accuracy: 0.8703 - val_loss: 0.9941 - val_accuracy: 0.7586\n",
            "Epoch 294/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3914 - accuracy: 0.8711 - val_loss: 0.9818 - val_accuracy: 0.7727\n",
            "Epoch 295/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3773 - accuracy: 0.8793 - val_loss: 1.0045 - val_accuracy: 0.7707\n",
            "Epoch 296/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3891 - accuracy: 0.8720 - val_loss: 0.9485 - val_accuracy: 0.7818\n",
            "Epoch 297/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3759 - accuracy: 0.8801 - val_loss: 0.9879 - val_accuracy: 0.7646\n",
            "Epoch 298/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3744 - accuracy: 0.8787 - val_loss: 0.9786 - val_accuracy: 0.7737\n",
            "Epoch 299/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3713 - accuracy: 0.8834 - val_loss: 0.9910 - val_accuracy: 0.7848\n",
            "Epoch 300/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3815 - accuracy: 0.8782 - val_loss: 1.0827 - val_accuracy: 0.7485\n",
            "Epoch 301/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3713 - accuracy: 0.8821 - val_loss: 0.9879 - val_accuracy: 0.7677\n",
            "Epoch 302/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3625 - accuracy: 0.8794 - val_loss: 0.9977 - val_accuracy: 0.7687\n",
            "Epoch 303/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3741 - accuracy: 0.8801 - val_loss: 1.0406 - val_accuracy: 0.7444\n",
            "Epoch 304/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3840 - accuracy: 0.8752 - val_loss: 1.0420 - val_accuracy: 0.7677\n",
            "Epoch 305/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3871 - accuracy: 0.8747 - val_loss: 0.9938 - val_accuracy: 0.7576\n",
            "Epoch 306/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3568 - accuracy: 0.8839 - val_loss: 0.9636 - val_accuracy: 0.7788\n",
            "Epoch 307/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3700 - accuracy: 0.8788 - val_loss: 0.9491 - val_accuracy: 0.7828\n",
            "Epoch 308/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3744 - accuracy: 0.8747 - val_loss: 1.0378 - val_accuracy: 0.7606\n",
            "Epoch 309/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.3846 - accuracy: 0.8802 - val_loss: 0.9672 - val_accuracy: 0.7778\n",
            "Epoch 310/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3436 - accuracy: 0.8831 - val_loss: 0.9427 - val_accuracy: 0.7758\n",
            "Epoch 311/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3563 - accuracy: 0.8842 - val_loss: 1.0268 - val_accuracy: 0.7778\n",
            "Epoch 312/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3644 - accuracy: 0.8806 - val_loss: 1.0377 - val_accuracy: 0.7525\n",
            "Epoch 313/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3862 - accuracy: 0.8762 - val_loss: 1.0642 - val_accuracy: 0.7566\n",
            "Epoch 314/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3796 - accuracy: 0.8778 - val_loss: 1.0634 - val_accuracy: 0.7626\n",
            "Epoch 315/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3550 - accuracy: 0.8836 - val_loss: 0.9946 - val_accuracy: 0.7727\n",
            "Epoch 316/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3612 - accuracy: 0.8831 - val_loss: 1.0610 - val_accuracy: 0.7707\n",
            "Epoch 317/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3613 - accuracy: 0.8843 - val_loss: 0.9608 - val_accuracy: 0.7788\n",
            "Epoch 318/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3569 - accuracy: 0.8843 - val_loss: 0.9739 - val_accuracy: 0.7798\n",
            "Epoch 319/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3624 - accuracy: 0.8849 - val_loss: 0.9704 - val_accuracy: 0.7818\n",
            "Epoch 320/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3553 - accuracy: 0.8831 - val_loss: 1.0021 - val_accuracy: 0.7758\n",
            "Epoch 321/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3602 - accuracy: 0.8869 - val_loss: 1.0270 - val_accuracy: 0.7596\n",
            "Epoch 322/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3475 - accuracy: 0.8872 - val_loss: 0.9992 - val_accuracy: 0.7677\n",
            "Epoch 323/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3563 - accuracy: 0.8869 - val_loss: 1.0099 - val_accuracy: 0.7697\n",
            "Epoch 324/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3494 - accuracy: 0.8867 - val_loss: 1.0334 - val_accuracy: 0.7505\n",
            "Epoch 325/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3674 - accuracy: 0.8836 - val_loss: 0.9931 - val_accuracy: 0.7636\n",
            "Epoch 326/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3633 - accuracy: 0.8828 - val_loss: 0.9701 - val_accuracy: 0.7727\n",
            "Epoch 327/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3502 - accuracy: 0.8863 - val_loss: 1.0090 - val_accuracy: 0.7727\n",
            "Epoch 328/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3530 - accuracy: 0.8827 - val_loss: 1.0743 - val_accuracy: 0.7535\n",
            "Epoch 329/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3544 - accuracy: 0.8854 - val_loss: 1.0221 - val_accuracy: 0.7667\n",
            "Epoch 330/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3718 - accuracy: 0.8801 - val_loss: 1.0582 - val_accuracy: 0.7697\n",
            "Epoch 331/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3623 - accuracy: 0.8813 - val_loss: 1.0640 - val_accuracy: 0.7586\n",
            "Epoch 332/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.3467 - accuracy: 0.8822 - val_loss: 1.0212 - val_accuracy: 0.7545\n",
            "Epoch 333/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3492 - accuracy: 0.8839 - val_loss: 1.0231 - val_accuracy: 0.7646\n",
            "Epoch 334/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3604 - accuracy: 0.8797 - val_loss: 0.9535 - val_accuracy: 0.7848\n",
            "Epoch 335/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3379 - accuracy: 0.8892 - val_loss: 1.0429 - val_accuracy: 0.7727\n",
            "Epoch 336/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3453 - accuracy: 0.8853 - val_loss: 0.9878 - val_accuracy: 0.7606\n",
            "Epoch 337/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3511 - accuracy: 0.8851 - val_loss: 1.0549 - val_accuracy: 0.7616\n",
            "Epoch 338/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3511 - accuracy: 0.8890 - val_loss: 0.9994 - val_accuracy: 0.7616\n",
            "Epoch 339/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3580 - accuracy: 0.8840 - val_loss: 0.9807 - val_accuracy: 0.7778\n",
            "Epoch 340/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3489 - accuracy: 0.8872 - val_loss: 0.9894 - val_accuracy: 0.7596\n",
            "Epoch 341/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3283 - accuracy: 0.8908 - val_loss: 0.9324 - val_accuracy: 0.7960\n",
            "Epoch 342/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3340 - accuracy: 0.8890 - val_loss: 1.0179 - val_accuracy: 0.7717\n",
            "Epoch 343/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3464 - accuracy: 0.8901 - val_loss: 0.9857 - val_accuracy: 0.7778\n",
            "Epoch 344/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3370 - accuracy: 0.8873 - val_loss: 1.0102 - val_accuracy: 0.7677\n",
            "Epoch 345/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3369 - accuracy: 0.8884 - val_loss: 1.0605 - val_accuracy: 0.7616\n",
            "Epoch 346/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3561 - accuracy: 0.8867 - val_loss: 0.9851 - val_accuracy: 0.7657\n",
            "Epoch 347/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3339 - accuracy: 0.8880 - val_loss: 1.0057 - val_accuracy: 0.7798\n",
            "Epoch 348/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3202 - accuracy: 0.8918 - val_loss: 1.0318 - val_accuracy: 0.7687\n",
            "Epoch 349/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3422 - accuracy: 0.8892 - val_loss: 1.0653 - val_accuracy: 0.7687\n",
            "Epoch 350/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3362 - accuracy: 0.8926 - val_loss: 1.0676 - val_accuracy: 0.7697\n",
            "Epoch 351/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3400 - accuracy: 0.8863 - val_loss: 1.0160 - val_accuracy: 0.7768\n",
            "Epoch 352/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3358 - accuracy: 0.8927 - val_loss: 1.0291 - val_accuracy: 0.7667\n",
            "Epoch 353/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3252 - accuracy: 0.8951 - val_loss: 1.0076 - val_accuracy: 0.7778\n",
            "Epoch 354/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3359 - accuracy: 0.8928 - val_loss: 1.0052 - val_accuracy: 0.7747\n",
            "Epoch 355/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3300 - accuracy: 0.8898 - val_loss: 1.0688 - val_accuracy: 0.7677\n",
            "Epoch 356/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3374 - accuracy: 0.8888 - val_loss: 1.0827 - val_accuracy: 0.7667\n",
            "Epoch 357/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3164 - accuracy: 0.8923 - val_loss: 0.9936 - val_accuracy: 0.7707\n",
            "Epoch 358/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3379 - accuracy: 0.8897 - val_loss: 1.1172 - val_accuracy: 0.7515\n",
            "Epoch 359/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3288 - accuracy: 0.8936 - val_loss: 1.0509 - val_accuracy: 0.7596\n",
            "Epoch 360/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3187 - accuracy: 0.8931 - val_loss: 1.0106 - val_accuracy: 0.7859\n",
            "Epoch 361/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3097 - accuracy: 0.9014 - val_loss: 1.0169 - val_accuracy: 0.7677\n",
            "Epoch 362/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3328 - accuracy: 0.8939 - val_loss: 1.0410 - val_accuracy: 0.7788\n",
            "Epoch 363/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3158 - accuracy: 0.8973 - val_loss: 1.0341 - val_accuracy: 0.7687\n",
            "Epoch 364/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3425 - accuracy: 0.8894 - val_loss: 1.0218 - val_accuracy: 0.7646\n",
            "Epoch 365/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3203 - accuracy: 0.8951 - val_loss: 1.0039 - val_accuracy: 0.7768\n",
            "Epoch 366/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3463 - accuracy: 0.8888 - val_loss: 1.0280 - val_accuracy: 0.7636\n",
            "Epoch 367/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3648 - accuracy: 0.8833 - val_loss: 1.0827 - val_accuracy: 0.7657\n",
            "Epoch 368/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3587 - accuracy: 0.8817 - val_loss: 1.0470 - val_accuracy: 0.7515\n",
            "Epoch 369/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3408 - accuracy: 0.8878 - val_loss: 1.0037 - val_accuracy: 0.7606\n",
            "Epoch 370/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3163 - accuracy: 0.8983 - val_loss: 1.0130 - val_accuracy: 0.7667\n",
            "Epoch 371/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3221 - accuracy: 0.8960 - val_loss: 0.9923 - val_accuracy: 0.7747\n",
            "Epoch 372/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3168 - accuracy: 0.8967 - val_loss: 1.0877 - val_accuracy: 0.7667\n",
            "Epoch 373/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3330 - accuracy: 0.8897 - val_loss: 0.9934 - val_accuracy: 0.7737\n",
            "Epoch 374/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3410 - accuracy: 0.8878 - val_loss: 0.9841 - val_accuracy: 0.7758\n",
            "Epoch 375/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3110 - accuracy: 0.8986 - val_loss: 0.9831 - val_accuracy: 0.7596\n",
            "Epoch 376/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3079 - accuracy: 0.9013 - val_loss: 0.9887 - val_accuracy: 0.7707\n",
            "Epoch 377/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3000 - accuracy: 0.8998 - val_loss: 1.0257 - val_accuracy: 0.7798\n",
            "Epoch 378/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3186 - accuracy: 0.8959 - val_loss: 1.0566 - val_accuracy: 0.7646\n",
            "Epoch 379/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3204 - accuracy: 0.8960 - val_loss: 1.0092 - val_accuracy: 0.7606\n",
            "Epoch 380/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3154 - accuracy: 0.8996 - val_loss: 1.0214 - val_accuracy: 0.7677\n",
            "Epoch 381/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3203 - accuracy: 0.9001 - val_loss: 0.9899 - val_accuracy: 0.7737\n",
            "Epoch 382/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3190 - accuracy: 0.8951 - val_loss: 1.0071 - val_accuracy: 0.7737\n",
            "Epoch 383/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3132 - accuracy: 0.8968 - val_loss: 1.0505 - val_accuracy: 0.7626\n",
            "Epoch 384/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3158 - accuracy: 0.8984 - val_loss: 0.9993 - val_accuracy: 0.7667\n",
            "Epoch 385/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3327 - accuracy: 0.8923 - val_loss: 1.0592 - val_accuracy: 0.7586\n",
            "Epoch 386/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3172 - accuracy: 0.8983 - val_loss: 0.9949 - val_accuracy: 0.7747\n",
            "Epoch 387/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3062 - accuracy: 0.9000 - val_loss: 1.0503 - val_accuracy: 0.7606\n",
            "Epoch 388/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3157 - accuracy: 0.8963 - val_loss: 1.0440 - val_accuracy: 0.7535\n",
            "Epoch 389/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3022 - accuracy: 0.9016 - val_loss: 1.0296 - val_accuracy: 0.7707\n",
            "Epoch 390/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3103 - accuracy: 0.8991 - val_loss: 1.0548 - val_accuracy: 0.7525\n",
            "Epoch 391/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3061 - accuracy: 0.8980 - val_loss: 1.0099 - val_accuracy: 0.7646\n",
            "Epoch 392/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3139 - accuracy: 0.8972 - val_loss: 1.1064 - val_accuracy: 0.7747\n",
            "Epoch 393/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3124 - accuracy: 0.8959 - val_loss: 1.0217 - val_accuracy: 0.7667\n",
            "Epoch 394/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.3073 - accuracy: 0.9013 - val_loss: 1.0382 - val_accuracy: 0.7747\n",
            "Epoch 395/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3133 - accuracy: 0.8994 - val_loss: 1.0436 - val_accuracy: 0.7657\n",
            "Epoch 396/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3313 - accuracy: 0.8878 - val_loss: 0.9878 - val_accuracy: 0.7677\n",
            "Epoch 397/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3112 - accuracy: 0.8991 - val_loss: 1.0603 - val_accuracy: 0.7606\n",
            "Epoch 398/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3198 - accuracy: 0.8950 - val_loss: 0.9135 - val_accuracy: 0.7768\n",
            "Epoch 399/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3149 - accuracy: 0.8968 - val_loss: 1.0139 - val_accuracy: 0.7758\n",
            "Epoch 400/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3060 - accuracy: 0.8996 - val_loss: 1.0191 - val_accuracy: 0.7778\n",
            "Epoch 401/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3087 - accuracy: 0.8987 - val_loss: 0.9986 - val_accuracy: 0.7717\n",
            "Epoch 402/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2889 - accuracy: 0.9006 - val_loss: 1.0530 - val_accuracy: 0.7657\n",
            "Epoch 403/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2988 - accuracy: 0.9011 - val_loss: 1.0381 - val_accuracy: 0.7566\n",
            "Epoch 404/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.3036 - accuracy: 0.8976 - val_loss: 1.0436 - val_accuracy: 0.7505\n",
            "Epoch 405/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2960 - accuracy: 0.9010 - val_loss: 1.0022 - val_accuracy: 0.7636\n",
            "Epoch 406/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3282 - accuracy: 0.8928 - val_loss: 1.0755 - val_accuracy: 0.7687\n",
            "Epoch 407/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3099 - accuracy: 0.8981 - val_loss: 1.0637 - val_accuracy: 0.7586\n",
            "Epoch 408/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3046 - accuracy: 0.8994 - val_loss: 1.0894 - val_accuracy: 0.7556\n",
            "Epoch 409/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2946 - accuracy: 0.9011 - val_loss: 1.0749 - val_accuracy: 0.7677\n",
            "Epoch 410/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3202 - accuracy: 0.8983 - val_loss: 0.9913 - val_accuracy: 0.7636\n",
            "Epoch 411/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2995 - accuracy: 0.9012 - val_loss: 1.0308 - val_accuracy: 0.7667\n",
            "Epoch 412/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2938 - accuracy: 0.9052 - val_loss: 1.1271 - val_accuracy: 0.7556\n",
            "Epoch 413/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3030 - accuracy: 0.9006 - val_loss: 1.0281 - val_accuracy: 0.7646\n",
            "Epoch 414/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2973 - accuracy: 0.9063 - val_loss: 1.0847 - val_accuracy: 0.7616\n",
            "Epoch 415/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3063 - accuracy: 0.8989 - val_loss: 1.0383 - val_accuracy: 0.7566\n",
            "Epoch 416/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3139 - accuracy: 0.8979 - val_loss: 1.0438 - val_accuracy: 0.7808\n",
            "Epoch 417/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3213 - accuracy: 0.8952 - val_loss: 0.9950 - val_accuracy: 0.7717\n",
            "Epoch 418/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2909 - accuracy: 0.9032 - val_loss: 1.0334 - val_accuracy: 0.7586\n",
            "Epoch 419/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3036 - accuracy: 0.9029 - val_loss: 1.0263 - val_accuracy: 0.7828\n",
            "Epoch 420/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2860 - accuracy: 0.9043 - val_loss: 0.9874 - val_accuracy: 0.7828\n",
            "Epoch 421/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3055 - accuracy: 0.9001 - val_loss: 1.0575 - val_accuracy: 0.7707\n",
            "Epoch 422/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2945 - accuracy: 0.9007 - val_loss: 1.0684 - val_accuracy: 0.7636\n",
            "Epoch 423/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3214 - accuracy: 0.8961 - val_loss: 1.0315 - val_accuracy: 0.7697\n",
            "Epoch 424/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3033 - accuracy: 0.8999 - val_loss: 1.0632 - val_accuracy: 0.7889\n",
            "Epoch 425/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2971 - accuracy: 0.9023 - val_loss: 1.0978 - val_accuracy: 0.7717\n",
            "Epoch 426/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3134 - accuracy: 0.8988 - val_loss: 1.0561 - val_accuracy: 0.7768\n",
            "Epoch 427/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2866 - accuracy: 0.9062 - val_loss: 1.0342 - val_accuracy: 0.7687\n",
            "Epoch 428/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2923 - accuracy: 0.9076 - val_loss: 0.9846 - val_accuracy: 0.7778\n",
            "Epoch 429/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2802 - accuracy: 0.9073 - val_loss: 1.1020 - val_accuracy: 0.7758\n",
            "Epoch 430/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2967 - accuracy: 0.9023 - val_loss: 1.0259 - val_accuracy: 0.7596\n",
            "Epoch 431/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3040 - accuracy: 0.8987 - val_loss: 1.0701 - val_accuracy: 0.7545\n",
            "Epoch 432/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2970 - accuracy: 0.9024 - val_loss: 1.0458 - val_accuracy: 0.7697\n",
            "Epoch 433/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3100 - accuracy: 0.9042 - val_loss: 1.0686 - val_accuracy: 0.7727\n",
            "Epoch 434/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2900 - accuracy: 0.9029 - val_loss: 1.0365 - val_accuracy: 0.7747\n",
            "Epoch 435/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3011 - accuracy: 0.9000 - val_loss: 1.0766 - val_accuracy: 0.7636\n",
            "Epoch 436/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2856 - accuracy: 0.9074 - val_loss: 1.0419 - val_accuracy: 0.7848\n",
            "Epoch 437/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2806 - accuracy: 0.9089 - val_loss: 1.0396 - val_accuracy: 0.7616\n",
            "Epoch 438/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2935 - accuracy: 0.9039 - val_loss: 1.0363 - val_accuracy: 0.7697\n",
            "Epoch 439/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2744 - accuracy: 0.9101 - val_loss: 1.0349 - val_accuracy: 0.7758\n",
            "Epoch 440/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2921 - accuracy: 0.9004 - val_loss: 1.0707 - val_accuracy: 0.7717\n",
            "Epoch 441/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2956 - accuracy: 0.9034 - val_loss: 0.9992 - val_accuracy: 0.7747\n",
            "Epoch 442/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3016 - accuracy: 0.8984 - val_loss: 1.0542 - val_accuracy: 0.7616\n",
            "Epoch 443/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2817 - accuracy: 0.9071 - val_loss: 1.0513 - val_accuracy: 0.7768\n",
            "Epoch 444/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3078 - accuracy: 0.8990 - val_loss: 1.0012 - val_accuracy: 0.7687\n",
            "Epoch 445/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2719 - accuracy: 0.9127 - val_loss: 1.0019 - val_accuracy: 0.7657\n",
            "Epoch 446/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2868 - accuracy: 0.9071 - val_loss: 1.0437 - val_accuracy: 0.7677\n",
            "Epoch 447/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2794 - accuracy: 0.9102 - val_loss: 1.0869 - val_accuracy: 0.7657\n",
            "Epoch 448/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2771 - accuracy: 0.9064 - val_loss: 1.0681 - val_accuracy: 0.7687\n",
            "Epoch 449/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.3052 - accuracy: 0.8968 - val_loss: 1.0321 - val_accuracy: 0.7707\n",
            "Epoch 450/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2833 - accuracy: 0.9063 - val_loss: 1.0449 - val_accuracy: 0.7586\n",
            "Epoch 451/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2734 - accuracy: 0.9107 - val_loss: 1.0185 - val_accuracy: 0.7747\n",
            "Epoch 452/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2680 - accuracy: 0.9137 - val_loss: 1.0562 - val_accuracy: 0.7576\n",
            "Epoch 453/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2789 - accuracy: 0.9070 - val_loss: 1.0535 - val_accuracy: 0.7717\n",
            "Epoch 454/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2807 - accuracy: 0.9107 - val_loss: 0.9901 - val_accuracy: 0.7727\n",
            "Epoch 455/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2926 - accuracy: 0.9064 - val_loss: 1.0725 - val_accuracy: 0.7677\n",
            "Epoch 456/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2771 - accuracy: 0.9100 - val_loss: 1.0132 - val_accuracy: 0.7768\n",
            "Epoch 457/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2803 - accuracy: 0.9052 - val_loss: 1.0457 - val_accuracy: 0.7717\n",
            "Epoch 458/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2719 - accuracy: 0.9082 - val_loss: 1.0572 - val_accuracy: 0.7606\n",
            "Epoch 459/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2707 - accuracy: 0.9134 - val_loss: 1.0529 - val_accuracy: 0.7606\n",
            "Epoch 460/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2832 - accuracy: 0.9071 - val_loss: 1.0754 - val_accuracy: 0.7616\n",
            "Epoch 461/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2844 - accuracy: 0.9077 - val_loss: 1.0779 - val_accuracy: 0.7606\n",
            "Epoch 462/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2701 - accuracy: 0.9121 - val_loss: 1.0016 - val_accuracy: 0.7808\n",
            "Epoch 463/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2732 - accuracy: 0.9089 - val_loss: 1.0672 - val_accuracy: 0.7778\n",
            "Epoch 464/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2698 - accuracy: 0.9099 - val_loss: 1.1208 - val_accuracy: 0.7576\n",
            "Epoch 465/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2811 - accuracy: 0.9068 - val_loss: 1.0614 - val_accuracy: 0.7616\n",
            "Epoch 466/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2921 - accuracy: 0.9087 - val_loss: 1.0038 - val_accuracy: 0.7697\n",
            "Epoch 467/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2774 - accuracy: 0.9083 - val_loss: 1.0380 - val_accuracy: 0.7768\n",
            "Epoch 468/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2746 - accuracy: 0.9097 - val_loss: 1.0887 - val_accuracy: 0.7737\n",
            "Epoch 469/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2753 - accuracy: 0.9108 - val_loss: 1.0584 - val_accuracy: 0.7677\n",
            "Epoch 470/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2775 - accuracy: 0.9077 - val_loss: 1.0458 - val_accuracy: 0.7677\n",
            "Epoch 471/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2884 - accuracy: 0.9068 - val_loss: 1.0655 - val_accuracy: 0.7707\n",
            "Epoch 472/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2826 - accuracy: 0.9074 - val_loss: 1.1108 - val_accuracy: 0.7616\n",
            "Epoch 473/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2700 - accuracy: 0.9114 - val_loss: 1.0473 - val_accuracy: 0.7758\n",
            "Epoch 474/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2659 - accuracy: 0.9099 - val_loss: 1.0737 - val_accuracy: 0.7616\n",
            "Epoch 475/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2585 - accuracy: 0.9146 - val_loss: 1.0195 - val_accuracy: 0.7505\n",
            "Epoch 476/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2527 - accuracy: 0.9150 - val_loss: 1.0936 - val_accuracy: 0.7737\n",
            "Epoch 477/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2836 - accuracy: 0.9081 - val_loss: 1.0968 - val_accuracy: 0.7636\n",
            "Epoch 478/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2753 - accuracy: 0.9127 - val_loss: 0.9897 - val_accuracy: 0.7677\n",
            "Epoch 479/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2770 - accuracy: 0.9097 - val_loss: 1.0107 - val_accuracy: 0.7717\n",
            "Epoch 480/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2670 - accuracy: 0.9126 - val_loss: 1.0206 - val_accuracy: 0.7778\n",
            "Epoch 481/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2714 - accuracy: 0.9099 - val_loss: 1.0109 - val_accuracy: 0.7788\n",
            "Epoch 482/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2625 - accuracy: 0.9138 - val_loss: 1.0478 - val_accuracy: 0.7808\n",
            "Epoch 483/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2913 - accuracy: 0.9077 - val_loss: 1.0608 - val_accuracy: 0.7677\n",
            "Epoch 484/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2648 - accuracy: 0.9106 - val_loss: 1.0457 - val_accuracy: 0.7838\n",
            "Epoch 485/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2840 - accuracy: 0.9056 - val_loss: 1.0567 - val_accuracy: 0.7737\n",
            "Epoch 486/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2581 - accuracy: 0.9147 - val_loss: 1.1059 - val_accuracy: 0.7626\n",
            "Epoch 487/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2781 - accuracy: 0.9089 - val_loss: 1.0450 - val_accuracy: 0.7727\n",
            "Epoch 488/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2566 - accuracy: 0.9123 - val_loss: 1.0202 - val_accuracy: 0.7798\n",
            "Epoch 489/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2735 - accuracy: 0.9084 - val_loss: 1.0117 - val_accuracy: 0.7778\n",
            "Epoch 490/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2730 - accuracy: 0.9117 - val_loss: 0.9830 - val_accuracy: 0.7838\n",
            "Epoch 491/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2657 - accuracy: 0.9131 - val_loss: 1.0570 - val_accuracy: 0.7646\n",
            "Epoch 492/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2635 - accuracy: 0.9146 - val_loss: 0.9345 - val_accuracy: 0.7838\n",
            "Epoch 493/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2610 - accuracy: 0.9128 - val_loss: 1.0896 - val_accuracy: 0.7636\n",
            "Epoch 494/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2672 - accuracy: 0.9116 - val_loss: 1.0306 - val_accuracy: 0.7768\n",
            "Epoch 495/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2705 - accuracy: 0.9122 - val_loss: 1.0657 - val_accuracy: 0.7768\n",
            "Epoch 496/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2680 - accuracy: 0.9088 - val_loss: 1.0529 - val_accuracy: 0.7737\n",
            "Epoch 497/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2513 - accuracy: 0.9143 - val_loss: 1.0453 - val_accuracy: 0.7747\n",
            "Epoch 498/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2597 - accuracy: 0.9144 - val_loss: 1.0771 - val_accuracy: 0.7596\n",
            "Epoch 499/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2740 - accuracy: 0.9103 - val_loss: 1.0500 - val_accuracy: 0.7646\n",
            "Epoch 500/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2570 - accuracy: 0.9169 - val_loss: 1.0549 - val_accuracy: 0.7768\n",
            "Epoch 501/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2524 - accuracy: 0.9151 - val_loss: 1.0780 - val_accuracy: 0.7697\n",
            "Epoch 502/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2609 - accuracy: 0.9157 - val_loss: 1.0321 - val_accuracy: 0.7636\n",
            "Epoch 503/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2533 - accuracy: 0.9187 - val_loss: 1.0921 - val_accuracy: 0.7646\n",
            "Epoch 504/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2690 - accuracy: 0.9136 - val_loss: 1.0461 - val_accuracy: 0.7657\n",
            "Epoch 505/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2665 - accuracy: 0.9151 - val_loss: 1.1723 - val_accuracy: 0.7596\n",
            "Epoch 506/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2704 - accuracy: 0.9107 - val_loss: 1.0657 - val_accuracy: 0.7677\n",
            "Epoch 507/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2588 - accuracy: 0.9172 - val_loss: 1.0890 - val_accuracy: 0.7717\n",
            "Epoch 508/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2643 - accuracy: 0.9104 - val_loss: 1.0958 - val_accuracy: 0.7616\n",
            "Epoch 509/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2669 - accuracy: 0.9124 - val_loss: 1.0858 - val_accuracy: 0.7697\n",
            "Epoch 510/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2600 - accuracy: 0.9138 - val_loss: 1.0625 - val_accuracy: 0.7636\n",
            "Epoch 511/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2618 - accuracy: 0.9129 - val_loss: 1.0037 - val_accuracy: 0.7879\n",
            "Epoch 512/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2739 - accuracy: 0.9104 - val_loss: 1.0611 - val_accuracy: 0.7758\n",
            "Epoch 513/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2458 - accuracy: 0.9197 - val_loss: 1.0832 - val_accuracy: 0.7596\n",
            "Epoch 514/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2524 - accuracy: 0.9176 - val_loss: 1.0982 - val_accuracy: 0.7727\n",
            "Epoch 515/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2695 - accuracy: 0.9124 - val_loss: 1.0228 - val_accuracy: 0.7707\n",
            "Epoch 516/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2460 - accuracy: 0.9202 - val_loss: 1.0916 - val_accuracy: 0.7697\n",
            "Epoch 517/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2609 - accuracy: 0.9114 - val_loss: 1.0965 - val_accuracy: 0.7707\n",
            "Epoch 518/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2553 - accuracy: 0.9177 - val_loss: 1.0865 - val_accuracy: 0.7717\n",
            "Epoch 519/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2501 - accuracy: 0.9168 - val_loss: 1.0662 - val_accuracy: 0.7737\n",
            "Epoch 520/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2515 - accuracy: 0.9162 - val_loss: 1.0869 - val_accuracy: 0.7636\n",
            "Epoch 521/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2586 - accuracy: 0.9138 - val_loss: 1.1122 - val_accuracy: 0.7657\n",
            "Epoch 522/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2700 - accuracy: 0.9106 - val_loss: 1.0827 - val_accuracy: 0.7747\n",
            "Epoch 523/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2750 - accuracy: 0.9116 - val_loss: 1.0784 - val_accuracy: 0.7747\n",
            "Epoch 524/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2696 - accuracy: 0.9129 - val_loss: 1.0548 - val_accuracy: 0.7808\n",
            "Epoch 525/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2581 - accuracy: 0.9173 - val_loss: 1.0802 - val_accuracy: 0.7747\n",
            "Epoch 526/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2582 - accuracy: 0.9180 - val_loss: 1.0298 - val_accuracy: 0.7707\n",
            "Epoch 527/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2530 - accuracy: 0.9151 - val_loss: 1.0958 - val_accuracy: 0.7677\n",
            "Epoch 528/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2472 - accuracy: 0.9186 - val_loss: 1.0649 - val_accuracy: 0.7687\n",
            "Epoch 529/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2638 - accuracy: 0.9183 - val_loss: 1.1143 - val_accuracy: 0.7798\n",
            "Epoch 530/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2626 - accuracy: 0.9168 - val_loss: 1.0673 - val_accuracy: 0.7636\n",
            "Epoch 531/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2608 - accuracy: 0.9149 - val_loss: 1.0615 - val_accuracy: 0.7737\n",
            "Epoch 532/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2631 - accuracy: 0.9147 - val_loss: 1.0902 - val_accuracy: 0.7828\n",
            "Epoch 533/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2479 - accuracy: 0.9188 - val_loss: 1.0263 - val_accuracy: 0.7768\n",
            "Epoch 534/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2496 - accuracy: 0.9196 - val_loss: 1.0248 - val_accuracy: 0.7737\n",
            "Epoch 535/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2473 - accuracy: 0.9138 - val_loss: 1.0832 - val_accuracy: 0.7545\n",
            "Epoch 536/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2546 - accuracy: 0.9156 - val_loss: 1.1349 - val_accuracy: 0.7586\n",
            "Epoch 537/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2710 - accuracy: 0.9124 - val_loss: 1.0326 - val_accuracy: 0.7717\n",
            "Epoch 538/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2533 - accuracy: 0.9132 - val_loss: 1.0719 - val_accuracy: 0.7737\n",
            "Epoch 539/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2536 - accuracy: 0.9178 - val_loss: 1.0826 - val_accuracy: 0.7768\n",
            "Epoch 540/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2590 - accuracy: 0.9164 - val_loss: 1.1021 - val_accuracy: 0.7798\n",
            "Epoch 541/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2424 - accuracy: 0.9210 - val_loss: 0.9682 - val_accuracy: 0.7778\n",
            "Epoch 542/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2486 - accuracy: 0.9194 - val_loss: 1.0158 - val_accuracy: 0.7727\n",
            "Epoch 543/700\n",
            "300/300 [==============================] - 4s 14ms/step - loss: 0.2595 - accuracy: 0.9174 - val_loss: 1.0695 - val_accuracy: 0.7616\n",
            "Epoch 544/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2575 - accuracy: 0.9146 - val_loss: 1.0736 - val_accuracy: 0.7586\n",
            "Epoch 545/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2586 - accuracy: 0.9171 - val_loss: 1.0450 - val_accuracy: 0.7737\n",
            "Epoch 546/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2510 - accuracy: 0.9219 - val_loss: 1.1081 - val_accuracy: 0.7768\n",
            "Epoch 547/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2475 - accuracy: 0.9181 - val_loss: 1.1150 - val_accuracy: 0.7616\n",
            "Epoch 548/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2398 - accuracy: 0.9224 - val_loss: 1.1674 - val_accuracy: 0.7616\n",
            "Epoch 549/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2413 - accuracy: 0.9203 - val_loss: 1.1203 - val_accuracy: 0.7596\n",
            "Epoch 550/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2438 - accuracy: 0.9187 - val_loss: 1.0129 - val_accuracy: 0.7899\n",
            "Epoch 551/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2413 - accuracy: 0.9219 - val_loss: 1.1088 - val_accuracy: 0.7717\n",
            "Epoch 552/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2559 - accuracy: 0.9144 - val_loss: 1.0686 - val_accuracy: 0.7707\n",
            "Epoch 553/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2363 - accuracy: 0.9216 - val_loss: 1.0448 - val_accuracy: 0.7636\n",
            "Epoch 554/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2360 - accuracy: 0.9202 - val_loss: 1.0517 - val_accuracy: 0.7848\n",
            "Epoch 555/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2685 - accuracy: 0.9116 - val_loss: 1.0609 - val_accuracy: 0.7687\n",
            "Epoch 556/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2423 - accuracy: 0.9204 - val_loss: 1.0356 - val_accuracy: 0.7899\n",
            "Epoch 557/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2530 - accuracy: 0.9179 - val_loss: 1.1050 - val_accuracy: 0.7717\n",
            "Epoch 558/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2409 - accuracy: 0.9201 - val_loss: 1.0503 - val_accuracy: 0.7808\n",
            "Epoch 559/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2614 - accuracy: 0.9162 - val_loss: 1.0925 - val_accuracy: 0.7576\n",
            "Epoch 560/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2476 - accuracy: 0.9178 - val_loss: 1.0934 - val_accuracy: 0.7697\n",
            "Epoch 561/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2352 - accuracy: 0.9217 - val_loss: 1.0972 - val_accuracy: 0.7687\n",
            "Epoch 562/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2473 - accuracy: 0.9199 - val_loss: 1.1148 - val_accuracy: 0.7677\n",
            "Epoch 563/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2535 - accuracy: 0.9158 - val_loss: 1.0777 - val_accuracy: 0.7717\n",
            "Epoch 564/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2404 - accuracy: 0.9196 - val_loss: 1.0921 - val_accuracy: 0.7707\n",
            "Epoch 565/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2509 - accuracy: 0.9167 - val_loss: 1.0726 - val_accuracy: 0.7707\n",
            "Epoch 566/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2400 - accuracy: 0.9236 - val_loss: 1.0185 - val_accuracy: 0.7758\n",
            "Epoch 567/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2440 - accuracy: 0.9206 - val_loss: 1.0266 - val_accuracy: 0.7889\n",
            "Epoch 568/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2278 - accuracy: 0.9232 - val_loss: 1.0652 - val_accuracy: 0.7808\n",
            "Epoch 569/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2390 - accuracy: 0.9230 - val_loss: 1.0220 - val_accuracy: 0.7778\n",
            "Epoch 570/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2371 - accuracy: 0.9224 - val_loss: 1.1044 - val_accuracy: 0.7667\n",
            "Epoch 571/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2359 - accuracy: 0.9226 - val_loss: 1.0316 - val_accuracy: 0.7929\n",
            "Epoch 572/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2380 - accuracy: 0.9218 - val_loss: 1.0600 - val_accuracy: 0.7657\n",
            "Epoch 573/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2454 - accuracy: 0.9203 - val_loss: 1.1022 - val_accuracy: 0.7677\n",
            "Epoch 574/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2416 - accuracy: 0.9229 - val_loss: 1.0692 - val_accuracy: 0.7707\n",
            "Epoch 575/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2470 - accuracy: 0.9182 - val_loss: 1.0867 - val_accuracy: 0.7778\n",
            "Epoch 576/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2373 - accuracy: 0.9228 - val_loss: 1.1257 - val_accuracy: 0.7697\n",
            "Epoch 577/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2456 - accuracy: 0.9207 - val_loss: 1.0977 - val_accuracy: 0.7788\n",
            "Epoch 578/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2392 - accuracy: 0.9199 - val_loss: 1.0727 - val_accuracy: 0.7657\n",
            "Epoch 579/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2330 - accuracy: 0.9229 - val_loss: 1.0828 - val_accuracy: 0.7626\n",
            "Epoch 580/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2576 - accuracy: 0.9158 - val_loss: 1.1137 - val_accuracy: 0.7576\n",
            "Epoch 581/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2396 - accuracy: 0.9191 - val_loss: 1.0720 - val_accuracy: 0.7586\n",
            "Epoch 582/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2505 - accuracy: 0.9186 - val_loss: 1.0837 - val_accuracy: 0.7697\n",
            "Epoch 583/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2387 - accuracy: 0.9238 - val_loss: 1.0137 - val_accuracy: 0.7879\n",
            "Epoch 584/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2325 - accuracy: 0.9203 - val_loss: 1.0759 - val_accuracy: 0.7707\n",
            "Epoch 585/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2502 - accuracy: 0.9159 - val_loss: 1.1823 - val_accuracy: 0.7505\n",
            "Epoch 586/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2424 - accuracy: 0.9209 - val_loss: 1.0298 - val_accuracy: 0.7768\n",
            "Epoch 587/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2317 - accuracy: 0.9239 - val_loss: 1.0935 - val_accuracy: 0.7677\n",
            "Epoch 588/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2439 - accuracy: 0.9189 - val_loss: 1.0500 - val_accuracy: 0.7768\n",
            "Epoch 589/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2279 - accuracy: 0.9229 - val_loss: 1.0578 - val_accuracy: 0.7747\n",
            "Epoch 590/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2498 - accuracy: 0.9160 - val_loss: 1.0530 - val_accuracy: 0.7727\n",
            "Epoch 591/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2382 - accuracy: 0.9229 - val_loss: 1.0809 - val_accuracy: 0.7667\n",
            "Epoch 592/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2386 - accuracy: 0.9240 - val_loss: 1.0242 - val_accuracy: 0.7929\n",
            "Epoch 593/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2343 - accuracy: 0.9234 - val_loss: 1.0807 - val_accuracy: 0.7687\n",
            "Epoch 594/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2324 - accuracy: 0.9232 - val_loss: 1.1177 - val_accuracy: 0.7707\n",
            "Epoch 595/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2435 - accuracy: 0.9210 - val_loss: 1.1161 - val_accuracy: 0.7768\n",
            "Epoch 596/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2328 - accuracy: 0.9219 - val_loss: 1.0906 - val_accuracy: 0.7808\n",
            "Epoch 597/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2631 - accuracy: 0.9106 - val_loss: 1.0497 - val_accuracy: 0.7838\n",
            "Epoch 598/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2469 - accuracy: 0.9193 - val_loss: 1.0623 - val_accuracy: 0.7788\n",
            "Epoch 599/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2434 - accuracy: 0.9211 - val_loss: 1.0784 - val_accuracy: 0.7697\n",
            "Epoch 600/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2459 - accuracy: 0.9208 - val_loss: 1.0747 - val_accuracy: 0.7848\n",
            "Epoch 601/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2377 - accuracy: 0.9228 - val_loss: 1.0675 - val_accuracy: 0.7697\n",
            "Epoch 602/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2430 - accuracy: 0.9212 - val_loss: 1.0524 - val_accuracy: 0.7717\n",
            "Epoch 603/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2322 - accuracy: 0.9246 - val_loss: 1.0541 - val_accuracy: 0.7869\n",
            "Epoch 604/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2228 - accuracy: 0.9288 - val_loss: 1.0274 - val_accuracy: 0.7808\n",
            "Epoch 605/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2382 - accuracy: 0.9242 - val_loss: 1.0552 - val_accuracy: 0.7808\n",
            "Epoch 606/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2328 - accuracy: 0.9244 - val_loss: 1.0989 - val_accuracy: 0.7758\n",
            "Epoch 607/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2312 - accuracy: 0.9254 - val_loss: 1.0316 - val_accuracy: 0.7768\n",
            "Epoch 608/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2236 - accuracy: 0.9281 - val_loss: 1.1136 - val_accuracy: 0.7778\n",
            "Epoch 609/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2323 - accuracy: 0.9240 - val_loss: 1.0998 - val_accuracy: 0.7606\n",
            "Epoch 610/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2298 - accuracy: 0.9232 - val_loss: 1.1229 - val_accuracy: 0.7636\n",
            "Epoch 611/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2315 - accuracy: 0.9226 - val_loss: 1.1777 - val_accuracy: 0.7384\n",
            "Epoch 612/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2299 - accuracy: 0.9230 - val_loss: 1.0582 - val_accuracy: 0.7758\n",
            "Epoch 613/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2351 - accuracy: 0.9236 - val_loss: 1.0912 - val_accuracy: 0.7717\n",
            "Epoch 614/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2296 - accuracy: 0.9256 - val_loss: 1.1119 - val_accuracy: 0.7616\n",
            "Epoch 615/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2299 - accuracy: 0.9223 - val_loss: 1.1236 - val_accuracy: 0.7626\n",
            "Epoch 616/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2179 - accuracy: 0.9284 - val_loss: 1.0956 - val_accuracy: 0.7687\n",
            "Epoch 617/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2224 - accuracy: 0.9222 - val_loss: 1.1159 - val_accuracy: 0.7697\n",
            "Epoch 618/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2304 - accuracy: 0.9252 - val_loss: 1.0706 - val_accuracy: 0.7747\n",
            "Epoch 619/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2080 - accuracy: 0.9334 - val_loss: 1.0484 - val_accuracy: 0.7758\n",
            "Epoch 620/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2248 - accuracy: 0.9257 - val_loss: 1.0405 - val_accuracy: 0.7788\n",
            "Epoch 621/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2260 - accuracy: 0.9259 - val_loss: 1.0964 - val_accuracy: 0.7677\n",
            "Epoch 622/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2191 - accuracy: 0.9260 - val_loss: 1.0501 - val_accuracy: 0.7667\n",
            "Epoch 623/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2298 - accuracy: 0.9273 - val_loss: 1.0864 - val_accuracy: 0.7677\n",
            "Epoch 624/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2270 - accuracy: 0.9272 - val_loss: 1.0497 - val_accuracy: 0.7828\n",
            "Epoch 625/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2361 - accuracy: 0.9211 - val_loss: 1.0741 - val_accuracy: 0.7657\n",
            "Epoch 626/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2225 - accuracy: 0.9267 - val_loss: 1.0628 - val_accuracy: 0.7828\n",
            "Epoch 627/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2148 - accuracy: 0.9321 - val_loss: 1.0115 - val_accuracy: 0.7768\n",
            "Epoch 628/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2188 - accuracy: 0.9277 - val_loss: 1.1291 - val_accuracy: 0.7646\n",
            "Epoch 629/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2188 - accuracy: 0.9276 - val_loss: 1.0258 - val_accuracy: 0.7879\n",
            "Epoch 630/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2242 - accuracy: 0.9252 - val_loss: 1.1092 - val_accuracy: 0.7687\n",
            "Epoch 631/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2261 - accuracy: 0.9303 - val_loss: 1.0874 - val_accuracy: 0.7838\n",
            "Epoch 632/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2197 - accuracy: 0.9253 - val_loss: 1.0372 - val_accuracy: 0.7828\n",
            "Epoch 633/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2156 - accuracy: 0.9318 - val_loss: 1.1221 - val_accuracy: 0.7646\n",
            "Epoch 634/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2268 - accuracy: 0.9266 - val_loss: 1.0572 - val_accuracy: 0.7717\n",
            "Epoch 635/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2179 - accuracy: 0.9286 - val_loss: 1.0680 - val_accuracy: 0.7667\n",
            "Epoch 636/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2230 - accuracy: 0.9276 - val_loss: 1.0602 - val_accuracy: 0.7919\n",
            "Epoch 637/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2306 - accuracy: 0.9243 - val_loss: 1.0295 - val_accuracy: 0.7778\n",
            "Epoch 638/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2309 - accuracy: 0.9269 - val_loss: 1.1105 - val_accuracy: 0.7657\n",
            "Epoch 639/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2257 - accuracy: 0.9258 - val_loss: 1.1480 - val_accuracy: 0.7636\n",
            "Epoch 640/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2238 - accuracy: 0.9270 - val_loss: 1.0987 - val_accuracy: 0.7707\n",
            "Epoch 641/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2142 - accuracy: 0.9299 - val_loss: 1.0699 - val_accuracy: 0.7859\n",
            "Epoch 642/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2194 - accuracy: 0.9280 - val_loss: 1.1110 - val_accuracy: 0.7707\n",
            "Epoch 643/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2187 - accuracy: 0.9272 - val_loss: 1.1050 - val_accuracy: 0.7737\n",
            "Epoch 644/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2063 - accuracy: 0.9346 - val_loss: 1.0836 - val_accuracy: 0.7687\n",
            "Epoch 645/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2405 - accuracy: 0.9222 - val_loss: 1.0909 - val_accuracy: 0.7677\n",
            "Epoch 646/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2239 - accuracy: 0.9257 - val_loss: 1.1415 - val_accuracy: 0.7657\n",
            "Epoch 647/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2257 - accuracy: 0.9259 - val_loss: 1.1255 - val_accuracy: 0.7677\n",
            "Epoch 648/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2118 - accuracy: 0.9302 - val_loss: 1.1241 - val_accuracy: 0.7848\n",
            "Epoch 649/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2215 - accuracy: 0.9276 - val_loss: 1.1092 - val_accuracy: 0.7747\n",
            "Epoch 650/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2064 - accuracy: 0.9289 - val_loss: 1.1141 - val_accuracy: 0.7646\n",
            "Epoch 651/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2120 - accuracy: 0.9318 - val_loss: 1.0794 - val_accuracy: 0.7758\n",
            "Epoch 652/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2384 - accuracy: 0.9238 - val_loss: 1.1221 - val_accuracy: 0.7727\n",
            "Epoch 653/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2219 - accuracy: 0.9279 - val_loss: 1.1044 - val_accuracy: 0.7818\n",
            "Epoch 654/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2217 - accuracy: 0.9277 - val_loss: 1.1249 - val_accuracy: 0.7606\n",
            "Epoch 655/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2146 - accuracy: 0.9302 - val_loss: 1.1116 - val_accuracy: 0.7616\n",
            "Epoch 656/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2263 - accuracy: 0.9249 - val_loss: 1.1017 - val_accuracy: 0.7818\n",
            "Epoch 657/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2151 - accuracy: 0.9330 - val_loss: 1.1307 - val_accuracy: 0.7818\n",
            "Epoch 658/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2175 - accuracy: 0.9297 - val_loss: 1.0698 - val_accuracy: 0.7747\n",
            "Epoch 659/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2157 - accuracy: 0.9303 - val_loss: 1.0958 - val_accuracy: 0.7747\n",
            "Epoch 660/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2130 - accuracy: 0.9316 - val_loss: 1.0972 - val_accuracy: 0.7778\n",
            "Epoch 661/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2326 - accuracy: 0.9248 - val_loss: 1.0775 - val_accuracy: 0.7727\n",
            "Epoch 662/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2035 - accuracy: 0.9344 - val_loss: 1.0949 - val_accuracy: 0.7929\n",
            "Epoch 663/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2129 - accuracy: 0.9270 - val_loss: 1.1249 - val_accuracy: 0.7576\n",
            "Epoch 664/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2013 - accuracy: 0.9326 - val_loss: 1.1305 - val_accuracy: 0.7657\n",
            "Epoch 665/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2210 - accuracy: 0.9270 - val_loss: 1.1248 - val_accuracy: 0.7717\n",
            "Epoch 666/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2103 - accuracy: 0.9291 - val_loss: 1.0684 - val_accuracy: 0.7727\n",
            "Epoch 667/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2128 - accuracy: 0.9311 - val_loss: 1.0704 - val_accuracy: 0.7687\n",
            "Epoch 668/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2140 - accuracy: 0.9301 - val_loss: 1.0786 - val_accuracy: 0.7808\n",
            "Epoch 669/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.1953 - accuracy: 0.9339 - val_loss: 1.0625 - val_accuracy: 0.7667\n",
            "Epoch 670/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2205 - accuracy: 0.9282 - val_loss: 1.1501 - val_accuracy: 0.7687\n",
            "Epoch 671/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2188 - accuracy: 0.9286 - val_loss: 1.1298 - val_accuracy: 0.7657\n",
            "Epoch 672/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2221 - accuracy: 0.9250 - val_loss: 1.1438 - val_accuracy: 0.7657\n",
            "Epoch 673/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.1995 - accuracy: 0.9348 - val_loss: 1.1307 - val_accuracy: 0.7687\n",
            "Epoch 674/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2091 - accuracy: 0.9314 - val_loss: 1.1037 - val_accuracy: 0.7677\n",
            "Epoch 675/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2132 - accuracy: 0.9316 - val_loss: 1.0993 - val_accuracy: 0.7747\n",
            "Epoch 676/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2127 - accuracy: 0.9300 - val_loss: 1.0745 - val_accuracy: 0.7747\n",
            "Epoch 677/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.1975 - accuracy: 0.9357 - val_loss: 1.0921 - val_accuracy: 0.7677\n",
            "Epoch 678/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2183 - accuracy: 0.9273 - val_loss: 1.0438 - val_accuracy: 0.7859\n",
            "Epoch 679/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2203 - accuracy: 0.9280 - val_loss: 1.1001 - val_accuracy: 0.7707\n",
            "Epoch 680/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2220 - accuracy: 0.9301 - val_loss: 1.1052 - val_accuracy: 0.7747\n",
            "Epoch 681/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2050 - accuracy: 0.9310 - val_loss: 1.1535 - val_accuracy: 0.7808\n",
            "Epoch 682/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2154 - accuracy: 0.9243 - val_loss: 1.1412 - val_accuracy: 0.7687\n",
            "Epoch 683/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2106 - accuracy: 0.9288 - val_loss: 1.0829 - val_accuracy: 0.7697\n",
            "Epoch 684/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2075 - accuracy: 0.9291 - val_loss: 1.1581 - val_accuracy: 0.7758\n",
            "Epoch 685/700\n",
            "300/300 [==============================] - 5s 16ms/step - loss: 0.2167 - accuracy: 0.9291 - val_loss: 1.0738 - val_accuracy: 0.7859\n",
            "Epoch 686/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2114 - accuracy: 0.9300 - val_loss: 1.0647 - val_accuracy: 0.7737\n",
            "Epoch 687/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2180 - accuracy: 0.9307 - val_loss: 1.1138 - val_accuracy: 0.7657\n",
            "Epoch 688/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2202 - accuracy: 0.9250 - val_loss: 1.1867 - val_accuracy: 0.7576\n",
            "Epoch 689/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.1946 - accuracy: 0.9354 - val_loss: 1.1284 - val_accuracy: 0.7596\n",
            "Epoch 690/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2106 - accuracy: 0.9332 - val_loss: 1.1055 - val_accuracy: 0.7657\n",
            "Epoch 691/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2244 - accuracy: 0.9269 - val_loss: 1.1352 - val_accuracy: 0.7626\n",
            "Epoch 692/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2053 - accuracy: 0.9307 - val_loss: 1.0799 - val_accuracy: 0.7687\n",
            "Epoch 693/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2009 - accuracy: 0.9323 - val_loss: 1.1427 - val_accuracy: 0.7697\n",
            "Epoch 694/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2030 - accuracy: 0.9331 - val_loss: 1.1312 - val_accuracy: 0.7707\n",
            "Epoch 695/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2035 - accuracy: 0.9321 - val_loss: 1.1145 - val_accuracy: 0.7808\n",
            "Epoch 696/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2045 - accuracy: 0.9347 - val_loss: 1.1627 - val_accuracy: 0.7576\n",
            "Epoch 697/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2130 - accuracy: 0.9300 - val_loss: 1.1059 - val_accuracy: 0.7768\n",
            "Epoch 698/700\n",
            "300/300 [==============================] - 4s 15ms/step - loss: 0.2160 - accuracy: 0.9301 - val_loss: 1.1347 - val_accuracy: 0.7616\n",
            "Epoch 699/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2125 - accuracy: 0.9313 - val_loss: 1.1624 - val_accuracy: 0.7727\n",
            "Epoch 700/700\n",
            "300/300 [==============================] - 5s 15ms/step - loss: 0.2108 - accuracy: 0.9328 - val_loss: 1.0900 - val_accuracy: 0.7707\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2dd3gVVfrHPyc3vRMSQgoQepEmBJCiBFBBUNCVRVlFwYJlLbu2tf0U1G22XbGurt0Vsa0igrgqEZUiHaQaeoAkkBDSSLs5vz/OrclNIaTd8H6e5z53ypmZ701mvnPmPe85o7TWCIIgCN6PT3MLEARBEBoGMXRBEIRWghi6IAhCK0EMXRAEoZUghi4IgtBK8G2uA0dHR+ukpKR6bVtYWEhISEjDCmpEvEmvN2kF79LrTVpB9DYmp6N13bp1x7TWMR5Xaq2b5TN48GBdX5YtW1bvbZsDb9LrTVq19i693qRVa9HbmJyOVmCtrsZXJeQiCILQShBDFwRBaCWIoQuCILQSxNAFQRBaCWLogiAIrQQxdEEQhFaCGLogCEIrQQxdEATBRml5BQdziqpdX1BSzrGCEgCsFZriMiv7swvJLijho7UHOVFU5lZ+Rdox0rIKAMjKLyavuIziMmuj6W+2nqKCIAhNwfe7jjLrzZ9Z9cA42oUH8vy3v9KjfRj+Fh++33WUuy/sQVigH8cLS/njhxtJ3XmUH+4bw4GcIn5KO8YN53bhrg83kl1QypZDJwCYcU4n3l21n5iwAI7mlziO9VFSOlcM6cDYXu3YeDCXWW+tAaB3XDjbj+QBkBAZxOWdK0hphN8qhi4IgtdTbq3g8ZUn+SxjA8eLyrhvQk+S2oYwZ+FWPlqXDsCM138mMtiP1Xtz3LZ9a8U+BnSIZHdWAQUl5QCc++Qyx/qXUndXOd67q/YDuJk5wM/7cvh5X06V8nYzB/D39SE8oHFeLCSGLghCndFaU1xWQZC/pU5ld2Tk0yM2jDd/2ktmXjGd2obg66MY0TWaNiF+5BaVcfB4EQrF9NdW0T8xgq4xoaQfL0IpRXxEIOd0acsn69MZ2CGS137YS7C/hfBAPyKD/Zg8MJ7+CZFc99YaSq0V7N54GICth/MoKbOSbzNogJ2Z+dVq3XQwt9bfM7pHDN/vOgpAm2A//vqb/jzx5TbSj5/k3euHsvFALit2Z7NyTzYAQztH8fPeHPwtPjw9bQAT+7Zn+5F8useGsuqnH2o9Xn0QQxcEwQ2tNT+lZfP6lhKOR6TjoxTn947li02Huf/TLQAMSWrDo5ecRdeYUOZ99ysfrU3nor7tAUjdlcW71w3jqn+v5lDuSdqG+JNdWFqnY29OP8Heo4VuRvyZzaTX7DsOQFGplaJSKxl5xez4aqejXEyQ4oaUnoQE+PLl5iNkF5aQn2ni12/OHEL32FCu/vdqyis039w1mn3ZhUSHBlBu1Ww8eJwxvdrhoxRTX1nJbwcncvU5nXh7xT5GdG1LdmEpwzpHsXb/cfrGRzhuaBf2iWXDwVwGdYzk3O4x3D6uO9/vOkrntiEktgliU3ouZ3ds49DYLzGivv+WOiGGLghehtaahZsOc37vWCw+ir8t2cHV53Ska0woz3+XRr/ECJLahtA5OsQ2aBMcyStm9jtrycov4ZbRXVmxO5tLBsQx79tfmTWyM5vTczlyopj24YGOEAXADws2edSwZt9xLn7+R87pEsWqPSbEYA9DAKQ8neqYtpt5cqc2rN1/vNrfNbRzFO9cN5TyCs2Ly9LYdjiPYwUlbD3sDFdEBPlx4mQZF/aJ5bpRnbny1VUA/PinMWxbv5oLR3cF4OpzOnk8xuI7zwUg0M9Cr/bhjuUTIuIc05//fqRj+toRSQB0t80PSYpy25+Pj2JwpzZuy0b3cA6E6GrmTYEYuiA0IxUVGg1YfJTb8qP5JWw5lMvoHu2w+CiWbDnCnz7ZzPRhHRmaFMWdH2wE4JIB8Xyx6TBvrdhXZd9RIf60CfajvEKzP9uZufHYom0AfLM9E4CHP/vFo7ZzE3z54VC5x3UA7cICWLUnh2Gdo5gxvBORQf78kHaUHUfy+X7XUUZ1i+blqwcx/+cDjO7Rjp7tw/jl0An+/tUOfvj1GM9dOZBBNsMrLC2na0wofhaTePenCb0AKLNWsOFALkOS2rDxYC79EyPZl11IXEQgvj6mbGSwH4ltgkmzKM9CXQj2b92W17p/nSA0E1prlKrZYP63LZMb31lLu7AAXr56MF9uPsK0IYm0Dw/koud+4FhBCZcMiMfXR/HfDYcA+Nf3e/jX93sc+/hi0+Fq959TWEqOS6jDXkP29/WhtLzCrewTl/blwj6x3PTeOgYkRjK8a1sCju7gL1edy5dbjtA7LpzcolL6J0YS7G/BRylyi0rZnpHPpH5xjhvSqO7RgDFiXx+FUorZ53V1HKdvQgTvXj+MtKx8urULq/Xv6GfxYWhnUyu213a7xoQ61r81a4jb/JmOGLog1MLdH27i4PEi5t94DhYfxf7sQiw+ihMny7jx7bVM6h9HZl4JY3u1Q6P508dbiAj24w/nd+fl1N2M7hFDr7hwXvvpJAe++pK2If50bBvMhgOmIS4rv4TLX14BwBs/7XU7dk2GHRrgS0FJOQ9N7E3vuHCW7czi3O7RdGobQkJkELsy8+mbEMGvmflYfBRdXIyvokKTuisLgLG9Yh3L/3urM9yQmrqDDlHB3DzaaciuxIQF0D3Wsynba9rVURczrwspPds1yH5aC2LoguCC1poTJ8t47Yc9TB/akbBAPz5Zb2LKXR9czKLbRzHzzTWOziUAr/1gTHihi/kezS/hof+aUMZ/Vh9wO0Z2YakjrhwdGuDYlz0rws6bM4dw4ztrmTG8EzmFpTw4sTfD/vItAOGBvozt1Y7PNh5mWJco+idGOmrHdvommAY4T6br46PcjFxoHYihC2ckmw7m0ibYnwM5Rby3aj9DO0fROTrE0REE4MVlVfOPL33xJ8ornDnEZ8WH0zk6hCA/i6MxsWtMCI9N6cvqPdnM+y4NAH+LD38/N4CgxD7c/N46Lh+UyJzJfQgL9GPd/hyeXrqLl64axOq9ORSUlPObsxPw8VFse2wC/r7O2u7ev06ksNRKaIAvecVlnNs9hn4JjZs5IXgPYuiCV1LhYqrFZVbKKzQWpcguLKF9eCC7jxayOT2Xfy3fg7VCc3aHSPKKy7l2RCc6R4cw5cWf3Pb31daMao/1m7MT6JsQwWOLtjnMfFjnKFbvzWFU92geuKg3AH/5TT8+33iYKQPj8bP4MLJbND/vy2HVnhx+un8sW9etJKVve96/cRiDOrYh0M+kvg3uFMX82ecAMMGW+mfH1cwBlFKEBpjLNjzQj8sHJ9bnzye0UsTQBa+iokJzIKeIq19fTfrxk4xO9OX57atZt/848RGBHC0ooWtMKDsynJ1IEiKD+NTWqGjP7HDljnHdKSwpZ9+xQq4f1ZkFaw8yc0QSCZFBhAT4EhLgi9aaAR0iuPzllQC8eNUg1uzNYUQ3Z5jDz+LD1EoG+9JVg1m3/zgxYQGOZSO6uodGBKGhEEMXWgxH80v44dejZOaV8N6q/dxwbmeKSq2s33+c28d1Z392IXuOFvLct786tvk+vRwwuc2HTxQDsCMjn05tg7l3fE8CfS2M6NaWt1bsY9GmI2xz6YK97uHzOZxbTN+EcLeMFFeTtqOUYlDHNtx9QQ9KrRVEhwZwUb+4KuUqExXizwV9JFYtNA1i6EKzkVtUSlpWARYfxZ0fbORApVHu5n6xzTH97Y6sKtvfktKVRev2cjDfpOD5W3y4OaUrS3/J4K3rhhAXEeQoe2tKN25N6UZ+cRlBfhaOnCimbWgAbUMDquy3OpRS3D6ue+0FBaGZEEMXGoXiMitv/LSXa4YnsXpPNjsy8rk1pSt//nI77cIDKCyx8vnGQ+zLrn6oUjtJbYOrlHvl6kFM6BvHYP8jBHboR2x4ACEBvsRHBnHXBT2q3VdYoB8AHaKCT+8HCkILRAxdqDNpWfnEhgc6TPHzjYf44ddjPDypN5HB/uQVl1FWXoEG/r5kBx+tS+dJl7E2nlq6s5o9u3PHuO74+ig6tQ0mwNeHIUkmna9j22Byi8oY1jkKX1ues6+PqpKuJwhnKmLoQp3Ye6yQ859dztTBiTz92wEcKyhxdD9ftPkwW+aM5+J5P3Igpwg/i6LM6j486OBObVh/4Di60qihSW2D+W1yByb1i2P13mzW7T/usYZdl3i1IJzpiKELgHlTiz1FTmtN6q6jnNst2lETfnW56W6+cnc2j32xza1HY3FZBZNf+MkRA/f18aHM6nwry7d3j3Z0zy4us7Ju/3EGd2rDqj3ZdI0JdYQ/kqJDuGJIx8b/sYLQShFDF8jKK2boX77F39eH8ztYeH33z/zw6zEAbjqvCwF+Fub/bHo7Hso96Wbm9u7nOzLy+OP5PbjzfNNoWFGhyS4sxeKjiArxd5QP9LMw0pZFIt22BaFhEUM/gymzVqA1rLJ1Ny8tr2Dx3grgmKPMv5Y7B4K6d3xPnlq6k0A/HwZ3asPkAfHERwYx4/Wf+eSWEY6R88B0LXfNvRYEofERQ2/laK35ZnsWW9Jz+f3YbvhbfNiZmc+GA7n8bckOTpwsI8Q2WH/78ECyC4oJD/LnlpSuPPHldsd+YsMDuGV0V64Y0oHiMiuJbZxZItsfm1CnN9gIgtC4iKG3Mo4VlPD7/6xnxvBOjOwazbC/fEup1eRpf70tkz1HCx3zdkICfPnXjGRGdY/mu2XLGDHqPAL9LBSVWnn2f7t4eFJvpg/tiI+PItpD3raYuSC0DMTQWwFaa47ml7BgzUFKrRWs3pvD6r05RIcGuJm3vTt85VeCvXTVIJJtb2LxUcoxxshtY7px7YgkIoL8mvDXCIJQX8TQvZgvNh0mdedRfBRurw2z4zrE68OTevPXJTuYM/ksZpzTia9+OULfhAgigvwceeWV8fFRYuaC4EWIoXsJxwpK+G5HFn3jI+gTH05aVgG3z9/gsez7Nwzj3VX76dQ2hNd/3EOZVXPDuV244dwujjIT+kpetyC0NsTQvYQ/f7nd8RqyqYMTyXapfQN8eccoJs37ETCDS9kHmLrx3M5VYuaCILROxNBbIOXWCsorNM98vZNtR/LILy5nc/oJx/qPbeGVAR0iCQvwZfKAeM6Kj+De8T0ZkBjptq9TGXxKEATvpk6GrpSaADwHWIB/a63/Vml9R+BtINJW5n6t9eIG1nrGMPxv33E0v6TK8pHd2nLo+Ekev7Qv/91wiJvO60rP9s7Xi/1+TLemlCkIQgujVkNXSlmAF4ELgHRgjVJqodZ6m0uxh4EPtdYvK6X6AIuBpEbQ2yr5NTOfQD8LmXlmPG9PZj6mZwx3XdCTfonmdWPndo9pUo2CILR86lJDHwqkaa33ACilPgCmAK6GroFw23QEUP2rygUHecVlfLw2nccWbfO4PsjPwo3ndqZ9RBC/GyZjnAiCUDNKVx7+rnIBpaYCE7TWN9jmZwDDtNa3uZSJA74G2gAhwPla63Ue9jUbmA0QGxs7+IMPPqiX6IKCAkJDQ+u1bXNg11uhNceLNfmlmjB/xZ+Wn6Tcw58/PkRx39BAwv0VPi5v0mlKrd7C6eqNyl5Hrx3PseqcV6mwBDagsqqcaX/b+hJUlM7J4JrflRpUdJiTQbGgnJ3amkKvpbwQi7WE0oCo09rP6WgdM2bMOq11sqd1DdUoOh14S2v9jFJqOPCuUqqv1totvUJr/SrwKkBycrJOSUmp18FSU1Op77bNgV3vHz7YwGcbqz689E+M4JwubTmQXcSskUkM69K29p2WFYOuAP+GfVGDt/5t683LD0HZCc7r3R7iBzaYLk/USWtFBexNhS5joIlv5pVx06s1FJ+AoMgat6mR/AzY/CGMuB12fQX+IdD5POf6gz9D1jZIvRN+9yH0GO9cV5wH/qHg4wPZu+H5KTD6T5DyoGe9tVGSD37B4GOBOREw4g648PHat3u6JxRkwJwTtZetgca6znxqL8IhoIPLfKJtmSvXAx8CaK1XAoHAGf3WgV2Z+ZS5pAtqrR1m3r1dKF2iQ7hkQDzf3DWahbeN4sGJvXllxuC6mTnAv86Dv9UxDFNhhWO/el7ektAaNvwHSgtrL3tkMxxY5ZyvOIXUzIIs2PqZmbbYRoIsO+m5rLUMFt0FJ6p23GpwSotg/dvw7mWw9VP3dVs/g8JjnrerL1k7zN+ttAgOb4C8I9WX/fk1+HsnyD1Q8z5reuL/7Fb43/8ZU59/Jbx9iXNdaSG8fgF8caeZz/zFZV0R/K0DfDvXzOdnmO89qZCzBw6tg5UvMWzVTXX7P1nL4a+J8M4UOJlrlq2YB+vfgS0fm/lD6yF9bdVtCzKq3+/2RebvBLBrKfw0r3YtDUxdauhrgO5Kqc4YI78S+F2lMgeAccBbSqneGEM/2pBCvYm9xwq58B/L6REbSlGplfTjJ+Erk/Rzz4U9uDWlGz4+p1n7OmZ7+8/u76Dr2KrrD6yGH56GK+dD6l/N9B0bICACQtrCzq9g/hUwawl0GmG22b6IyON7gBT3fVVUwO5voUsKrH4F0tfApS/D1w8bA/6/qu/7dLD+XYjpBR2GVF+mKAeCo4xBf34rHFwNk+dBYTZ8PBMufQUiEkxZreGTG+AX24WX8jns/QHevhhu/hHa9zPLTxyCr/4Eg2dBt3Hux/vketi7HDruBF9bWufJHGNwfoHQJslZdv8KWPu6MbzZy6pqz95tnpSiT+Ndo7u+hl8+gc0fQICtKSrHOcolhcfgo2uh4wi4bknt+/v1f1CSB30v97y+vBQ+u9kc87z7YPmTznXV1Tx3fWW+M7ZApEtFYudXYC0159COL+GLO+C6pdDxHDhqO0djeppv+436v7Od2x/eaJ6MsndXOqAyT6EV5VCQaRatfxsumAvltptv7kGYd7ZjiyAwJn3b2qpPNyePmxvBS+c4l+37AV4a7pxfeLv57jcVXhtjpq94z5zDhVnQb5qzrNaw6iXoMQHadjXLFlxlvouyzTUHMPw281RRUmBuGufe7TznGoFaDV1rXa6Uug1YiklJfENrvVUp9RiwVmu9ELgbeE0p9UdMA+lMXVtwvhWzzPZC412ZBVXWDe8a7W7mWpsao/IxJ11kB/ALqrId/3sUsrbDVR+6L3/3Mpj4NAy90X35x7Mg7xDkpRvzAkj7FhbfA5Oeha3/NcsyfjEXo9aw4CoGAgzoby6M2H4w8wvzmGmtlHnT93JY+4aZ/vQmSPsG/viLU/u2zyFnL3zzqJl/5Lg5sV2pqIAjG+C1sTD1TaMZ4Ngu873pfaN9xTy46O/m75R7wGnmdr6zPSq/MgrCE+APW+C9y+HodnPRVzZ0ey1u63/hwEoz/YFLHaX3JbD9C5jxX/N/ATi8Ht6YAOMegQ7nmBrkkY1OE3g4C1BQdAx8A80NKmOLWVaYBQvvoHvIWeDpMfv93zqnS/Jsv+kJo+F3H5obKMCBFbDxfXOTGXStuUmWFplao48vpP7N/J3+M9WULzsJmVvhwj/Dr19DTA9o0xne+40xM3A3c4D3rzDhh43/ISBwjHN5iO2B215Dt5YbrfOvMPMRHeGEbd3KFyCqC7w41Mzf/JMxv0IPdbw3xsPDmZBd6Qlyw3vOGvm1X5jvk8fhxWEQHm/m8z3kXmSnwdxIuPpTY+ADf2fOpxeHgl9I1fKe9uFqXQuudk4fdumZnbsflj4I696G2342f2c7djMHOHEQ2nQyy1a+ACtegPtreco5DeoUQ7fllC+utOwRl+ltwMiGleZ9WCs0y3Zk8bNtfPEpA+O5IrkD85etp1tYGf3iQxnU0RaDLDsJv3xqaqSunHMrTPgrVfjpn+b7WBqEVnoxxOJ74Pg+OO9eE+O0lhszB/holjEjMEYA8OVdzlrWyRxY9hf43uXCfmeK+c7cAl89WNXMwWZWNjbbGrc/vRGmvGRO/A+vcS+/+hVzcq96ycxf/w28fj4ERji3tXNgpam12m8O9prd09XUgl3DJXmH4KOZxszBXGj5GfDV/VBeAhf/EwJt/4Ov7ve8v+02A3n3Mhg8013XmxfBuEedZmPn2d7mt9hr1uMegW8fM9PhCZB3iIQTB51PI3XhyCb48m7Ysci57LNbzPeGd40xVA672WvDAJ//3nwrH2MmdWHXV47aeI+oHXD+ZFj+lLnxg/mb5R024YniXOd2J1xMaucSOL7fOf9KDdZQXmyeFBbfCyhMfRDIcamxu+7r6A7zsRMa66zBu85/8Duzbx+LOb8ByuoQygNY9Mfayzw3wHwXZJra98sjPJfL2mYM/cgmp4ZneuJ79j/qpuUUqTXLpbFITk7Wa9d6iFHVgZbWcFdcZmXNvhxmvP6zY9mk/nG8+LtBgE1vqs0kp74BiUNNqOSLO6rurH1/uPmHSgfIMzFEO0NugDX/rrptUJQJOZQXm7BFXahuXw1NUBtTwzoV4gaaWnDfqXDZK/D4KTbLDL+t7kZ2KiQOhfSfay/nSkC4qdGeP8fU6mLPMjei8X92DwOcLr0udr8B1IX4Qc6bvgvlliB8B0wzoY7qaHeWuWFkbqm6bsxDRovdzGoj+XoT3qrMgOmwab7nbX77lrmBA1kxI2l365fw5zhnWMaOfxiU5tdNR3XM/t5UVHL3117WzrCbjeFvfM9tcVrX6+g2o36mrpSqNsulLo2igge01ry3aj8Hc4q4+PkfeeSNz3nQ9z+0JxuAsZ1D4Js5sGAG8Ydc4p4fXwf/7As/PON5xxmbTW2otMiEDd6/0jSwuGI34GsrXbgnc2Dv97Wb+Xn3OqcLskxrf124/4AzxmsntL3nsn2nOqfH/t+pmzkYMwcTcvhLwqltO+4RE/M/VZLOhU4jIbJT9WVO1cwBIjtRHBBjzonje43R7f7Ws5n3utgYla+H0FttVDbzHhdVLZN0Llz9iXO+XR/ndCdnbdrXetKzmV/8T+hvC7UMnG7aO6JsA78FumTBdD4PYvtWrzXclproGwizU03ocOBVzvUXPWW+K5v5yDtNuuKV78NZl8EtK2HWErb3vtvEziubeddxMONT97YR8Hx+3LrKaK58noOJ9U9w6SRvb6+xTw+Ybra/4j8w80sTmlv9ijHzcPc0zPTEyVX33wDIWC71ZNuRPB7+zDyGhlPI5sC7ARh9dm/m+w1mcslC+NHcgau+wx5zl/cNqnrygXlE/d1HJi4NsMtDQ1ifS90vxLow6BqT+jXmIXMRrfk3bF/oVmRfp2kkjZ1lQjjfzHF/nA2MgAcOwn9vdl5kAaFgbyoYdK3TAAZfC70mmhBD8nWw/Gnnb/3TfvMY76n2fPWnJsbrSuUa0bhHTIPb5gXV/9aOIyDQw0VZHQOvNjfCaxY6Y/0HVsMbF5rpBw+bcNF3T5j5qW/AJzeCtmUKRfd0NlQ7UHD7Onh+ECQmc0LvITDr+5p1nHuPaQ8Jaw8//xv2/whnX21iygAXPQlL7oM7N5lY7zxbqmVkJxMeqhwKuvQlcx65hrSmvgmhMfCHX0wI5/xHTbvB3u/NE9vzg8A/jKPhZxFzbBVV6H8FdBxujjl0tmnku20dfP936DUJ/nWuKde2OyQmw8b/OLeNGwg9J5rfZG+QvHWl84Zw6Usw/i9mn76B8OtS53UA5hy74DHzsRNrrgO9N7WqVh9fSHnAtDfcucmE70oKTNgmaZRpW+g+3nyX5EO73nDjMigtMO0F1lKTfRPT232/Cclw47em4bYgw/1m0c5WttNwOGj7+138LLxva1QdflujpaSKodeHI5v4cZWzMWWaJdUx3VOlM2fyWbDgbx42rET7fsb0vplTdd3XD5vvITfCmtfc13VJgWlvu6cdjrgdVjxvpjuNNCdskXla4OGj4Ovvvo/z7jEn9Bu2XN8xD8GKF8hoP5akTiNMQ+mA6cY0/tEH8l1S2hKTXQzdNpZMh3NMDWtPqjHgtt3cc4zv3Gj02rNVxv/ZNJru/NJZZnYqxDuzFqrQebS5QIfdDGteB1wMfcqL5u/5L9sxOww1oYDJzzsbLl2Z9o7RWJBp4q7t+phsFdeG2zCXpw//EPNkYzf0TqOMEeUeMBkuAeHmbzX/CmcjZlAbkwHx+zUQ2YETHz5KbHWG3mmUqXn2uNC5LG6AMfSEZPP7Kmz6ht3kLGMPK43/i8l2at/PpGKue9P8ruAo6D/NfL93uQmvhNqGjYjsADNtNfqeE8wHTFZR17FsXbedlLbHoMMwiEg0bSCHN5q+D+16wdiHnDp8fGDMA+7po8FR5kZ5ZLOpedvPC3u+/8QnTWN/5ach11z3c+8xjZpxA03lo9xDe05lrv3CmRL5SLb7Ov8Q8wmLtf3ui5y/346vP/hGOds6bllp4uAACYPNU4i9natyVpQrfS837QmXPGeyfrqkmBvDhU/A97Xc2OuJGPopsuLXDEb85zxuAl4O/ojkTm14IGMpRA8zNdg9y0w4xbVWUR1RXWDEnSZs8dnN7uuO7TQnetLIqoZuf4z1sZiMlY7DbQaaAt3PN+u0Ng11Hc+pauZ2Op5jjl2QAd0vgNH3UZya6lyvlPncscGYnZ1eF5vGOoDfvm1OfIu/Oc6dm0ytPKRSPn2Yh9DMlBfg2B0Q3NaYrz39a8QdJrMF4PLXTZqh8oFrXZ4mSitlEJ19tUnH8/GFUX80fxswTyV2Q+91scl6OLTehCJ8/U0s2/F7K71KL8w2Znx3F5ON7GhMPCzWfFwbIQGu/x98OMM0rNoNL8Y8o2W3HQK//suY0+g/GaM8tsvcEILbVq21jXnQNAz3s4WvKmcJgbkxjv+zc777Bea7y2j3cvYacOUGdU8MnG6b2O48tv23R3b0uIkDHx+Y9Iy5MStl/saX2Br0K6euDrjSfGqi03CTtbRrqTH0kDq0o5zqk2ttxLrsLywW7q9jDL19P/i9S/jzms8bVpcHxNDryIrdx/js3Xk8yXOOZS9NjmdEj3h48hj0/qOJYf/6tTO7oTamvGgugIHTjaGHxcHdO2DpQ6bWFRrrHpPsMcFkIPS51LlsyPXOabuZg7mY/vALWGr5Fw+5Hpb9ueaYceU0yrD2MP0DiO7hrLm4HreymVdHcJS5qRVScH0AACAASURBVFTmwsedhh5/tjG/jsPdy/SdasvMcWnU9/U3TyOVjW/WEvMkYY959ppUN32+/nD7epOlYmf299V3QgLz+694z+OqksAY82QQf7bTGEOGeywLmHDWuP+rm9baiOpibo5dUhpmfzUx5IaG32f3C2HyC9Xn1rsS1Kbhj+8liKFXR+5BUyv0C2LDmh/I/2YeT1rcs3JGtCs3ea9gasiu8fAuKaaX4f6fzPyFT5ju3AFhJpXOWuputjf9YAwcoN9vjaEHhLk/fk7/wDQu1jXtrTYzBxNGGHrjqV8EPT00tjUGwVGmplqZmB4wJ5cjr1xG3BiXjiqearGdqkkpqwv2pwZXPadDnymnt/3p4Frb9jaUgkEz6lbWx1J7mVaKGDqYrsPpa42RWvxMBwiXHmhng+lSVZmtnzq7Crfv6x7TvvJ9Y77Ln+b7kEmMHnGBc13lWi1AXH/ndPxAW01ukOkpZ0ep0zeUyijVMms0voEm/TIgosZiO3vdSVzPlKbRJHgXcY07Nk9LRAwdTE9FMNkDp8JPtvBLREfTaOSa029vfLnkn2jXuHRdsdfk6pPu1xqY/T0cWuu5xi0ItfGQrffsGcaZ94sbg8teMd9KmXhtQ9Z4PeXDngm062U+glAfPA2fcQYghl5TAxewuqIXw3x2GJMOjXXvdgzwUKZJXbJzOvFaT9jjgR2GNex+BUFodcjz7KK7qizag8lqyD1vLv3H2QZtij/bPQXJjquZNxZ37YAZnzX+cQRB8GrOTEMvyYd9P5pBrDa9X2X1RcV/5v1xK4kc+weCgu1vFbHlCIfYOmUMv80MTdsUhMc1+IssBEFofZyZIZc1r5thXWP7VVl1Y+ldlODPyD62TBR7LM7e6eP29SYdsa651oIgCE3EmWno9jeeeBghLnrwpTwYE0qntraxk33tIRWboZ/K+CCCIAhNyJlp6K7jOLvwXvQf+OvlA9wX2g29md/vKAiCUBtnjqF/epPJQOk2zm04zvxZ3zP45X1c1Lc9T105tOp2DiMXQxcEoWVzZhh6RYV5q479zTo2yiY8xV/XWyjFj8uGdMHf10Mbsb2zkDoz248FQfAezgyXKvH84ttXT47l/dXm1Vl94quLjdsNXWrogiC0bM4MQz9ZNWZeHtufRZvNGN8do4JpF1ZNPrl9eFVvHthIEIQzgtYfcvnhWedb7114JOIv7N5awEMTezMtuYOHDW20SYJHc6WGLghCi6f1G3rlV3LZ+PCXfK4Y0oEbz+tS+z7EzAVB8AJav6FXYueQJ/jTCvD39eGm87rWvoEgCIKX0HoN/fh+5/suXbjmh0gC2yay/JYRRIcGNIMwQRCExqH1Gvpz/T0uLiSQr24dSZuQat6zKQiC4KWcGVkuLiS0ixYzFwShVdJ6a+iVWKHOZk1ZZ168ekjthQVBELyQM6aGvqq0K5ETH6Vbu9DaCwuCIHghrdPQXd/tacNHwYVnxTaDGEEQhKahdRp6zh7HpLa9k7NXfARxEWfmewYFQTgzaH2GnrMHnh/kmK3wM+Oax1TXtV8QBKGV0PoM/fUL3WZLLMbQoyTnXBCEVk7rM/TCo26zOeWmZh4fGdIcagRBEJqM1mXoOXurLHqm9HI2BQ0jYNh1zSBIEASh6Wg9hp53BOYNrLJ4Q0E4v57/BgRHNYMoQRCEpqNOhq6UmqCU2qmUSlNK3V9NmWlKqW1Kqa1KqfcbVmYd+OJOj4t7dIjjN2cnNLEYQRCEpqfWnqJKKQvwInABkA6sUUot1FpvcynTHXgAGKm1Pq6UatdYgqvFWlJl0YKKsdw4cTg+PjL8rSAIrZ+61NCHAmla6z1a61LgA2BKpTI3Ai9qrY8DaK2zGlZmHfDQmci/+1iGJEmoRRCEMwOlPRihWwGlpgITtNY32OZnAMO01re5lPkM2AWMBCzAHK31Vx72NRuYDRAbGzv4gw8+qFykThQUFBAa6t6Ff8DGh2mTu8Vt2aexdxDVe1y9jtGQeNLbUvEmreBder1JK4jexuR0tI4ZM2ad1jrZ07qGGpzLF+gOpACJwHKlVD+ttdvLPLXWrwKvAiQnJ+uUlJR6HSw1NZUq2+6LhEqvDu2ePI5+Q+p3jIbEo94WijdpBe/S601aQfQ2Jo2ltS4hl0OA60s3E23LXEkHFmqty7TWezG19e4NI7GOuDxpLOj9POeUvkTn/qOaVIIgCEJzUhdDXwN0V0p1Vkr5A1cCCyuV+QxTO0cpFQ30APbQpDgN/YX9iXTs2IXQgDNmdGBBEITaDV1rXQ7cBiwFtgMfaq23KqUeU0pNthVbCmQrpbYBy4B7tdbZjSXas9AKx+TBnJPcOkbeFyoIwplFnaqwWuvFwOJKyx5xmdbAXbZP01NRAQdWOmbjIwIZ3SOmWaQIgiA0F62jp+jKF9xmz0qIQCnJPRcE4cyidRj6jkVusz1ivSN1SRAEoSFpHYZeYXWb/c2gxGYSIgiC0Hy0DkO3+LnNdo2RGrogCGcercPQy4qaW4EgCEKz0zoMvVQMXRAEoZUYemFzKxAEQWh2WoehlxVSNPD65lYhCILQrLQOQy8tIrPYAoDVEtjMYgRBEJoH7zf08lKoKOOXo+VczxzKbvm5uRUJgiA0C95v6LZeolm5BYT3HkNgdKdmFiQIgtA8eL+hp30LwM8nE+gaE9LMYgRBEJoP7zf0mJ5YfYNZWjGEpGgxdEEQzly839CtJZT6hgHQJVp6iAqCcObSCgy9jBLbKMCdpYYuCMIZjPcbenkJxRUWEiKDCPK3NLcaQRCEZsO7Db20EEoLOGm10EUaRAVBOMPx7pdu/iUegALdVUZYFAThjMe7a+g2irXU0AVBEFqFoZdqP9qHS5d/QRDObFqFoZfhS0xYQHPLEARBaFZahaGXiqELgiC0HkOPDhVDFwThzMZ7DV1rx6Sy+BPoJznogiCc2XivoVeUOyYDAoOaUYggCELLwHsN3VrqmAwKlAwXQRAE7zX08hLHZHCw5KALgiB4r6FbyxyTYSESchEEQfBiQ3eGXMJCw5pRiCAIQsugdRh6WHgzChEEQWgZtApDDwiSgbkEQRBahaFbAqRRVBAEwXsNvdxp6PhJo6ggCIL3GrrV1dCDm0+HIAhCC6FOhq6UmqCU2qmUSlNK3V9DucuVUlopldxwEqvBKjV0QRAEV2p9Y5FSygK8CFwApANrlFILtdbbKpULA+4EVjeG0CqUFzunLX5NckgBtNbk5+eTl5dHUVERVqu1WfVERESwffv2ZtVQV7xJK4jexsBisRAcHIzFYkFrjVKqQfdfl1fQDQXStNZ7AJRSHwBTgG2Vyj0O/B24t0EVVkdpoXPa4t8khzzT0VqTlZVFYWEhUVFRtG/fHovF0uAn5amQn59PWJh39EPwJq0gehsarTVWq5WCggKioqLIysqiXbt2DXr91MXQE4CDLvPpwDDXAkqpQUAHrfWXSqlqDV0pNRuYDRAbG0tqauopCwYoKChg5+H19LTNr9uwifzdRfXaV1NQUFBQ79/a1NSk1WKxEBUVRUJCAhaLhZMnTzatOA9YrVby8/ObW0ad8CatIHobC4vFQlxcHEeOHGHXrl0N+pR72i+JVkr5AM8CM2srq7V+FXgVIDk5WaekpNTrmKmpqXRpkwC7zPzg4aMhpke99tUUpKamUt/f2tTUpDU9PZ3Q0FAiIyObVlQNtPRamSvepBVEb2OSn59PQkICERERJCYmNth+69Ioegjo4DKfaFtmJwzoC6QqpfYB5wALG7thtKjgBABr+j3aos28NVFUVERoqHTiEoSGIDQ0lKKiho0s1MXQ1wDdlVKdlVL+wJXAQvtKrfUJrXW01jpJa50ErAIma63XNqjSShQX5lOmLZzofVVjHkZwwWq1YrHIi0QEoSGwWCwNnlRQq6FrrcuB24ClwHbgQ631VqXUY0qpyQ2qpo4EFB8jdvNL+Ckr0fIu0SalORtABaE10RjXUp1i6FrrxcDiSsseqaZsyunLqpmAkmzHdNsQyXARBEEAL+0panVJU5SXQwuCIBi80tCVdsadgvwlpisIggBebujLLCOaWYkgCELLwUsNvQKAZaETm1mJILQOUlNTUUpJo7eX46WGbmrofr4yhovgPYSHhztM81Q/b731VnPLF7yA0+4p2hz4VJQD4OsnGS6C91DduB0FBQUUFpqxiWJjYz1uGxTUuCOKBgcH07Nnz9oLCi0arzR0e8hFauiCN5GWluaxa/qcOXOYO3cuABkZGU0tC4ChQ4eyY8eOZjm20HB4dcjFIjV0QRAEB15t6H5+UkMXWj/2OHpqaipZWVncdddd9OjRg+DgYLcQTlFREfPnz+eaa65h4MCBxMTEEBAQQHx8PJdeeilLliyp9hg1NYq+9dZbKKVISkoCYN26dUybNo24uDgCAgLo0qULd911F8ePH2/w3y6cGl4achFDF8480tLSuPLKK8nMzCQwMLDK+f/hhx8ya9YswNwEwsPD8fX15ciRI3z++ed8/vnn3H333Tz99NP11vD+++8zc+ZMysrKiIiIoLy8nL179/KPf/yDr7/+mlWrVskAbs2IV9bQsRm6v4RchDOIP/7xj0RGRvLtt99SWFhIXl4eO3fudKxv06YN99xzDz/++CMFBQXk5uZSWFjI4cOHmTt3Ln5+fjzzzDMsXLiwhqNUz9GjR7nuuuu49tprOXDgALm5ueTn5/PCCy/g5+fH1q1befLJJxvq5wr1wCtr6BUVthq6vxh6S2PuF1vZdjivyY7XmCNA9okP59FLzmqUfdcHHx8fvvnmG7fxs3v0cA4dPWXKFKZMmVJlu7i4OB555BGCg4O59957mTdvHpMnn/q4ekVFRVx77bW89tprjmXBwcH8/ve/Z8+ePTz77LPMnz+fxx577JT3LTQMXllDr7BKyEU485gxY8ZpvQxh0qRJAKxcubLew7Y+/PDDHpfbbyRpaWkNPsa3UHe8soZurZCQS0ulqWu03vSWmtNl5MiRtZbJzMzkpZde4uuvv2bXrl2cOHGiinkXFRVx/PhxoqOjT+n4UVFRdOvWzeO6+Ph4x/Tx48cJDg4+pX0LDYN3GrrtBA0IkBq6cObQrl27GtevXLmSiRMnkpub61gWGhrqyIaxWq0cO3YMgMLCwlM29JpunL6+TispKys7pf0KDYdXhlx0helY5O8nQ+cKZw41tRWUl5czffp0cnNzGThwIIsXLyYvL4/8/HwyMzPJyMhg1apVjvJa66aQLDQxXlpDN13/AwIk5CIIYGrn+/fvx2KxsGjRIhISEqqUaa5eqELT4aU1dFvIRbJcBAGAgwcPAhATE+PRzAG++eabppQkNANebeiBYuiCAEBERARgGkUzMzOrrE9PT2fevHlNLUtoYrzS0O156AEBgc2sRBBaBqNGjSIkJAStNdOmTWPXrl2ASSBYunQpKSkpMtb5GYBXGrq9hh7kL1kuggCmhm7v0r98+XJ69uxJWFgYoaGhTJgwgRMnTvDmm282s0qhsfFaQy/TFgID5H2igmDn5ptv5ssvvyQlJYXQ0FDKy8tJSEjg9ttvZ9OmTfTr16+5JQqNjFdmuegKK1Z8CPQTQxe8nzlz5jBnzpxq159KiuHEiROZOLH6VzNWt6+UlJRq182cOZOZM2fWeNykpCRJhWwBeGUNnQor5VgIEkMXBEFw4JWGrrUVKxb8LF4pXxAEoVHwTkessFLhpdIFQRAaC+90RV2BVUm4RRAEwRWvNHStK9DKK6ULgiA0Gl7pimZwLq+ULgiC0Gh4pSsqXUGF1NAFQRDc8EpXNCEXiaELgiC44pWGjq4AqaELgiC44Z2uKIYuCIJQBe90RQm5CIIgVMFLDV2Dj3dKFwRBaCzq5IpKqQlKqZ1KqTSl1P0e1t+llNqmlNqslPpWKdWp4aW6oCtAauiCIAhu1GroSikL8CJwEdAHmK6U6lOp2AYgWWvdH/gYeLKhhbpp0hUoqaELgiC4URdXHAqkaa33aK1LgQ+AKa4FtNbLtNZFttlVQGLDynRSZq3AB6mhC4IgVKYu46EnAAdd5tOBYTWUvx5Y4mmFUmo2MBsgNjaW1NTUuql04WS5JoQKSkrL67V9c1BQUNAqtEZERJCfn9+0gmrBarW2OE3V4U1aQfQ2JnatxcXFDesNWusaP8BU4N8u8zOAF6opezWmhh5Q234HDx6s60Nm3kn9w8PDdeY/zqvX9s3BsmXLmltCnalJ67Zt25pOSB3Jy8trbgl1pjm1Llu2TAPaXPJ1W1dXvTXtuyl48803NaA7duzYLMevD/a/bX2uKWCtrsZX6xJyOQR0cJlPtC1zQyl1PvAQMFlrXVKfm0tdKCmrwIJG+UjIRfAubr/9dpRStG3blpKSul8i3bt3RynF5MmTG1Fdy2Pfvn21vs1JcKcuhr4G6K6U6qyU8geuBBa6FlBKnQ38C2PmWQ0v08nJMis+qkIMXfA6ZsyYAUBOTg6ff/55nbb5/vvvSUtLA+D6669vFF3BwcH07NmTnj17Nsr+68u+ffuYO3cuc+fOrbFcREQEPXv2pHPnzk2krOVSq6FrrcuB24ClwHbgQ631VqXUY0ope5XhKSAU+EgptVEptbCa3Z02J0ut+FCBj0UMXfAuhg4dSp8+JkHszTffrNM29nKxsbFMmjSp0XTt2LGDHTt2NMr+G5vLLruMHTt28MUXXzS3lGanTrl/WuvFWuseWuuuWus/25Y9orVeaJs+X2sdq7UeaPs02rNhcZkVC1JDF7wTey3766+/5tChKpFLN/Lz8/n4448BuOaaa/D19cp3ugtNiNclc58ss9XQxdAFL2TGjBn4+flRUVHBW2+9VWPZBQsWUFhYCMB1111HUVER8+fP55prrmHgwIHExMQQEBBAfHw8l156KUuWeEwuq5XU1FSUUiilqi2zY8cOrrrqKtq3b09gYCBdunTh9ttvJzMzs8Z9l5WVsXDhQmbPnk1ycjJxcXH4+/vTrl07xo8fz/z58+0JFW4kJSUxZswYx7xdn/0zc+ZMx7q33noLpRR9+/atVsfu3bu55ZZb6N69O0FBQYSHhzNo0CAee+wx8vLy6vR3SUtL47rrrqNDhw4EBASQmJjIjTfeWOuNuUmprrW0sT/1zXJZsuWw3vR/A3Te65fVa/vmQLJcGg9vzHK5/PLLNaC7detWY/kRI0ZoQI8YMUJr7czmALRSSkdEROjg4GDHMkDffffdHvd1OlkuS5Ys0QEBAY4yoaGhOjAwUAM6Li5Ov/HGG3XaN6DDw8N1WFiY27Lf/va32mq1um2XnJys27Rp4ygTGxvr9rnjjjscZWvLclmwYIGb/rCwMLf5Dh06eDy3XbV/9913OjQ01LG9r6+vY118fLxOT0/3eOzqaM4slxbFSVvIRWLogrdiD7ukpaWxfPlyj2V27tzJihUr3Mq3adOGe+65hx9//JGCggJyc3MpLCzk8OHDzJ07Fz8/P5555hkWLmy4Jqz09HSuuOIKSkpK6N+/P6tXryY/P5/CwkKWLFmCxWLhrrvuqnb74OBgbrrpJv73v/9x4sQJTpw4QV5eHtnZ2Tz33HOEh4fz0Ucf8cILL7htt2bNGj799FPHfEZGhtvnueeeq5P+9evXc/XVV1NSUsLIkSPZvHkzeXl5FBUVsXDhQuLi4jh48CCXXHIJBQUF1e7n8ssvZ+zYsWzfvp28vDwKCwtZsGABYWFhHD58mAceeKBOehqd6py+sT/1raG/v3q/3vZ/Z+mT71xRr+2bA6mhNx7eWEO3Wq06MTFRA/raa6/1WPa+++5z1Ibz8/PrtP+nnnpKA3rcuHFV1tW3hn7LLbdoQLdt21ZnZmZW2XbLli3az8+v3nnoH330kQZ0165dT0mzKzXV0CdMmOB4GiosLKyyfv369Y7a9lNPPVXt8ceMGVPlKUJrrefNm6cBHRQUpMvKymr7uQ4aq4buda0sJstFSw29pbLkfsjY0mSHC7KWg6WRTuP2/eCivzX4bn18fJg5cyZPPPEEH3/8MS+88AKhoaGO9VarlXfffReAadOmua2riUmTJnHvvfeycuVKrFYrltO8RrTWLFiwAICbb76Zdu3aVSnTt29fpk6dyvz58+t1DHvmzu7du8nIyKB9+/b1F1yJ3Nxcli5dCsC9995LcHBwlTJnn302v/nNb/jwww+ZP38+99xzj8d9Pfjgg/h4GD9qypQp3HHHHZw8eZJff/2V3r17N5j++uB1IRcAX1WBpbEuYkFoAmbNmoVSyvHo7sqSJUs4cuQIUDX3PDMzk0cffZThw4fTtm1bfH19HQ139pTIoqIijh8/ftoa9+3bR05ODgBjx46ttlxN68Bk6zz11FOMHj2adu3a4e/v79DsarLp6emnrdmV9evXOxpczz///GrLXXDBBQBs3ryZsrIyj2WGDfM82kl8fLxj2v63ak68zhWvG9WZop8QQ2+pNEKNtiZO5ucTFhbWpMdsCLp06UJKSgrLli3jjTfecDPuN954A4BevXoxYsQIx/KVK1cyceJEcnNzHctCQ0MJDg5GKYXVauXYsWMAFBYWEh0dfVoajx496phOSEiotlxiYvVj8e3atYtx48a5mXVwcDCRkZGOGq89U8ae0dNQZGU5+zjWRX95eTk5OTnExsZWKVPdOeaaSlrdzaAp8coaupJX0AmtALuJr1ixgl27dgHGRBctWgSYVEU75eXlTJ8+ndzcXAYOHMjixYvJy8sjPz+fzMxMMjIyWLVqlaO8vWba3MyaNYv09HSSkpL46KOPyM7OprCwkKysLDIyMtxS/lqKZm/GS12xAiQPXfByLr/8ciIjIwFnrfy9996jrKwMX19frrnmGkfZlStXsn//fiwWC4sWLeKiiy6qUmvMyMhoUH0xMTGO6Zpyratbd/DgQUemzvz585k6dSpRUVFuZRpasyuuMf+awjn2db6+vlX0eRteaehK3lgktAICAwP53e9+B8A777yD1Wp1dPW/+OKL3R79Dx40I1jHxMRUGz745ptvGlRfUlKSw+CWLVtWbbnvvvvO43K7ZjCNj56oSbNrI2R9au+DBg1y7OPbb7+ttpxdw4ABA/Dz8zvl47QkvNfQ5Y1FQivAHnY5cuQIjz/+OFu2mAwh13ALmAGowMSbPfXOTE9PZ968eQ2qTSnFtGnTAHjllVcc8XlXtm3b5hieoDJ2zQCbNm2qsj4/P58nnnii2uOHh4c7pl3bDepKZGQk48ePB+Cpp56iqKioSplNmzbxySefADB9+vRTPkZLw0tdUUsMXWgVDBo0iIEDBwLw+OOPAxAXF8fEiRPdyo0aNYqQkBC01kybNs0Rc7darSxdupSUlJQau+7XlwceeICwsDCOHTvGBRdcwNq1awFTY/7666+56KKLPKYDAvTu3ZuOHTsC5ga1bt06x7qVK1eSkpJSYzZOjx498Pf3B+Df//53vWrpTzzxBH5+fqSlpTF+/HjHDbOiooLFixczceJEysvL6dq1KzfddNMp77+l4ZWuKCEXoTVhr6VXVFQAcO2111bJIY+IiODpp58GYPny5fTs2ZOwsDBCQ0OZMGECJ06cqPMIjqdCx44dmT9/PgEBAWzcuJEhQ4YQHh5OSEgI48ePp6ysjGeffdbjtj4+Prz44ov4+vqydetWkpOTCQkJISQkhBEjRrBz584qKZuuBAcHO4Ycvu+++wgNDaVTp04kJSVVmy9emUGDBvHuu+/i7+/Pjz/+SP/+/YmIiCAkJIRJkyZx+PBhOnTowBdffFHnfP+WjPcaujSKCq2Eq666isDAQMd85XCLnZtvvpkvv/ySlJQUQkNDKS8vJyEhgdtvv51NmzbRr1+/RtE3adIk1q9fz5VXXkm7du0oLS0lNjaW2267jQ0bNtQ4DvnFF1/M8uXLmTRpEpGRkZSXlxMdHc2sWbNYt24d48aNq/HYL774InPmzHH8tgMHDrB//36P4Z/quOKKK9i6dSs33XQTXbt2paSkBF9fXwYOHMjcuXP55Zdfmr1DUEOhmitVKDk5Wdsf306V8sfj8U2+pslznutLamoqKSkpzS2jTtSkdfv27S3uxM/3ojx0b9IKorcxsWutzzWllFqntU72tM4ra+ggeeiCIAiV8UpXlCwXQRCEqnilK0qjqCAIQlW80tAlbVEQBKEqXumKkuUiCIJQFe8zdK1RSMhFEAShMl5o6KbzhdTQBUEQ3PE+Q6+wmu9G6OYsCILgzXifoZfZBtjxC2leHWcoMma1IDQMjXEteZ+hl+Sb7wDv6BHWmrBYLFit1uaWIQitgoZ472tlvNjQvX8gHW8jODiYgoKC5pYhCK2CgoKCakeqrC/eZ+ilNkORGnqTEx4eTk5OjtTSBeE0sVqt5OTkuI353hB4n6GX5JlvfzH0piYsLIyQkBD2799Pbm4u5eXlElMXhDqitaa8vJzc3FwOHTpESEhIgw8m5lt7kRaGxNCbDaUU7dq1Iz8/n7y8PLKyspq9tl5cXOw29GxLxpu0guhtDCwWC8HBweTk5NCnT58GfymJFxu6xNCbA6UU4eHhDf6oWF9SU1OrfV9lS8ObtILobUzS0tIa5Q1TXhhykRi6IAiCJ7zP0Nt04mj0cImhC4IgVML7Qi69JrE1I4QUi/dJFwRBaEy8r4YuCIIgeEQMXRAEoZVQJ0NXSk1QSu1USqUppe73sD5AKbXAtn61UiqpoYUKgiAINVOroSulLMCLwEVAH2C6UqpPpWLXA8e11t2AfwB/b2ihgiAIQs3UpYY+FEjTWu/RWpcCHwBTKpWZArxtm/4YGKcaI8lSEARBqBZVW9dtpdRUYILW+gbb/AxgmNb6Npcyv9jKpNvmd9vKHKu0r9nAbIDY2NjBH3zwQb1EFxQUEBrqPR2LvEmvN2kF79LrTVpB9DYmp6N1zJgx67TWyZ7WNWnun9b6VeBVgOTkZJ2SklKv/aSmplLfbZsDb9LrTVrBu/R63sZ9twAABR1JREFUk1YQvY1JY2mtS8jlENDBZT7RtsxjGaWULxABZDeEQEEQBKFu1KWGvgborpTqjDHuK4HfVSqzELgWWAlMBb7TtcRy1q1bd0wptf/UJQMQDRyrtVTLwZv0epNW8C693qQVRG9jcjpaO1W3olZD11qXK6VuA5YCFuANrfVWpdRjwFqt9ULgdeBdpVQakIMx/dr2G1NX9ZVRSq2tLobUEvEmvd6kFbxLrzdpBdHbmDSW1jrF0LXWi4HFlZY94jJdDPy2YaUJgiAIp4L0FBUEQWgleKuhv9rcAk4Rb9LrTVrBu/R6k1YQvY1Jo2itNQ9dEARB8A68tYYuCIIgVEIMXRAEoZXgdYZe28iPzYFS6g2lVJZtCAT7siil1P+UUr/avtvYliul1Dyb/s1KqUFNrLWDUmqZUmqbUmqrUurOlqpXKRWolPpZKbXJpnWubXln26ieabZRPv1ty1vEqJ9KKYtSaoNSalFL1quU2qeU2qKU2qiUWmtb1uLOAxe9kUqpj5VSO5RS25VSw1uiXqVUT9vf1P7JU0r9oUm0aq295oPJg98NdAH8gU1Anxag6zxgEPCLy7Ingftt0/cDf7dNTwSWAAo4B1jdxFrjgEG26TBgF2YUzRan13bMUNu0H7DapuFD4Erb8leAW2zTtwKv2KavBBY00/lwF/A+sMg23yL1AvuA6ErLWtx54KLtbeAG27Q/ENmS9dp0WIAMTGegRtfa5D/wNP84w4GlLvMPAA80ty6blqRKhr4TiLNNxwE7bdP/AqZ7KtdMuj8HLmjpeoFgYD0wDNPDzrfyOYHp/DbcNu1rK6eaWGci8C0wFlhku0hbpN5qDL1FngeY4UT2Vv77tFS9Lse9EPipqbR6W8glATjoMp9uW9YSidVaH7FNZwCxtukW8xtsj/hnY2q+LVKvLXyxEcgC/od5QsvVWpd70OPQalt/AmjbVFpt/BO4D6iwzbel5erVwNdKqXXKjIQKLfQ8ADoDR4E3beGsfyulQmi5eu1cCcy3TTe6Vm8zdK9Em9tui8oPVUqFAp8Af9Ba57mua0l6tdZWrfVATM13KNCrmSVVi1LqYiBLa72uubXUkVFa60GYl9f8Xil1nuvKlnQeYJ5gBgEva63PBgoxYQsHLUwvtraSycBHldc1llZvM/S6jPzYUshUSsUB2L6zbMub/TcopfwwZv4frfWntsUtVi+A1joXWIYJWUQqM6pnZT3NPernSGCyUmof5kUwY4HnWqperfUh23cW8F/MDbOlngfpQLrWerVt/mOMwbdUvWBulOu11pm2+UbX6m2G7hj50Xb3uxIz0mNLxD4CJbbvz12WX2Nr2T4HOOHyGNboKKUUZjC17VrrZ1uyXqVUjFIq0jYdhIn1b8cY+9RqtNp/Q51G/WxItNYPaK0TtdZJmHPzO631VS1Rr1IqRCkVZp/GxHp/oQWeBwBa6wzgoFKqp23ROGBbS9VrYzrOcItdU+NqbepGggZoZJiIyczYDTzU3HpsmuYDR4AyTE3iekws9FvgV+AbIMpWVmHe0bob2AIkN7HWUZhHvc3ARttnYkvUC/QHNti0/gI8YlveBfgZSMM8zgbYlgfa5tNs67s04zmRgjPLpcXptWnaZPtstV9LLfE8cNE8EFhrOx8+A9q0VL1ACOZpK8JlWaNrla7/giAIrQRvC7kIgiAI1SCGLgiC0EoQQxcEQWgliKELgiC0EsTQBUEQWgli6IIgCK0EMXRBEIRWwv8DgmZwdapVuXkAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Iq-fF96-dE3v"
      },
      "source": [
        "## In the following part, the training data divided into two groups, including training data and validation data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fXmTx6mCMCkh"
      },
      "source": [
        "from scipy.sparse import csr_matrix\r\n",
        "from sklearn.model_selection import train_test_split\r\n",
        "X_train = featMatrixTrain\r\n",
        "X2=X_train[:][:]\r\n",
        "Y1=trainLabel[:][:]\r\n",
        "X1=csr_matrix(X2)\r\n",
        "X_train, X_test, Y_train, Y_test = train_test_split(X1, Y1, test_size=0.1, random_state=41)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WV07RXFLdMzP"
      },
      "source": [
        "## It the following part, Coyote Optimization Algorithm (COA) is modeled. COA is applied for regularization process, and finding the optimal values of hyperparameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TJVWcMb1huCr"
      },
      "source": [
        "## Coyote Optimization algorithm for tuning hyperparameters ##\r\n",
        "\r\n",
        "# We extract this optimization code from:\r\n",
        "# Coyote Optimization Algorithm (COA) for Global Optimization.\r\n",
        "# A nature-inspired metaheuristic proposed by Juliano Pierezan and\r\n",
        "# Leandro dos Santos Coelho (2018).\r\n",
        "\r\n",
        "\r\n",
        "\r\n",
        "def COA(FOBJ, lu, nfevalMAX, n_packs=20, n_coy=5):\r\n",
        "\r\n",
        "    # Optimization problem variables\r\n",
        "    D = lu.shape[1]\r\n",
        "    VarMin = lu[0]\r\n",
        "    VarMax = lu[1]\r\n",
        "\r\n",
        "    # Algorithm parameters\r\n",
        "    if n_coy < 3:\r\n",
        "        raise Exception(\"At least 3 coyotes per pack must be used\")\r\n",
        "\r\n",
        "    # Probability of leaving a pack\r\n",
        "    p_leave = 0.005*(n_coy**2)\r\n",
        "    Ps = 1/D\r\n",
        "\r\n",
        "    # Packs initialization (Eq. 2)\r\n",
        "    pop_total = n_packs*n_coy\r\n",
        "    costs = np.zeros((1, pop_total))\r\n",
        "    coyotes = np.tile(VarMin, [pop_total, 1]) + np.random.rand(pop_total, D) * np.tile(VarMax, [pop_total, 1]) - \\\r\n",
        "              np.tile(VarMin, [pop_total, 1])\r\n",
        "    ages = np.zeros((1, pop_total))\r\n",
        "    packs = np.random.permutation(pop_total).reshape(n_packs, n_coy)\r\n",
        "\r\n",
        "    # Evaluate coyotes adaptation (Eq. 3)\r\n",
        "    for c in range(pop_total):\r\n",
        "        costs[0, c] = FOBJ(coyotes[c, :])\r\n",
        "\r\n",
        "    nfeval = pop_total\r\n",
        "\r\n",
        "    # Output variables\r\n",
        "    globalMin = np.min(costs[0, :])\r\n",
        "    ibest = np.argmin(costs[0, :])\r\n",
        "    globalParams = coyotes[ibest, :]\r\n",
        "\r\n",
        "    # Main loop\r\n",
        "    year = 1\r\n",
        "    while nfeval < nfevalMAX:  # Stopping criteria\r\n",
        "        # Update the years counter\r\n",
        "        year += 1\r\n",
        "\r\n",
        "        # Execute the operations inside each pack\r\n",
        "        for p in range(n_packs):\r\n",
        "            # Get the coyotes that belong to each pack\r\n",
        "            coyotes_aux = coyotes[packs[p, :], :]\r\n",
        "            costs_aux = costs[0, packs[p, :]]\r\n",
        "            ages_aux = ages[0, packs[p, :]]\r\n",
        "\r\n",
        "            # Detect alphas according to the costs (Eq. 5)\r\n",
        "            ind = np.argsort(costs_aux)\r\n",
        "            costs_aux = costs_aux[ind]\r\n",
        "            coyotes_aux = coyotes_aux[ind, :]\r\n",
        "            ages_aux = ages_aux[ind]\r\n",
        "            c_alpha = coyotes_aux[0, :]\r\n",
        "\r\n",
        "            # Compute the social tendency of the pack (Eq. 6)\r\n",
        "            tendency = np.median(coyotes_aux, 0)\r\n",
        "\r\n",
        "            #  Update coyotes' social condition\r\n",
        "            new_coyotes = np.zeros((n_coy, D))\r\n",
        "            for c in range(n_coy):\r\n",
        "                rc1 = c\r\n",
        "                while rc1 == c:\r\n",
        "                    rc1 = np.random.randint(n_coy)\r\n",
        "                rc2 = c\r\n",
        "                while rc2 == c or rc2 == rc1:\r\n",
        "                    rc2 = np.random.randint(n_coy)\r\n",
        "\r\n",
        "                # Try to update the social condition according\r\n",
        "                # to the alpha and the pack tendency(Eq. 12)\r\n",
        "                new_coyotes[c, :] = coyotes_aux[c, :] + np.random.rand()*(c_alpha - coyotes_aux[rc1, :]) + \\\r\n",
        "                                    np.random.rand()*(tendency - coyotes_aux[rc2, :])\r\n",
        "\r\n",
        "                # Keep the coyotes in the search space (optimization problem constraint)\r\n",
        "                new_coyotes[c, :] = Limita(new_coyotes[c, :], D, VarMin, VarMax)\r\n",
        "\r\n",
        "                # Evaluate the new social condition (Eq. 13)\r\n",
        "                new_cost = FOBJ(new_coyotes[c, :])\r\n",
        "                nfeval += 1\r\n",
        "\r\n",
        "                # Adaptation (Eq. 14)\r\n",
        "                if new_cost < costs_aux[c]:\r\n",
        "                    costs_aux[c] = new_cost\r\n",
        "                    coyotes_aux[c, :] = new_coyotes[c, :]\r\n",
        "\r\n",
        "            # Birth of a new coyote from random parents (Eq. 7 and Alg. 1)\r\n",
        "            parents = np.random.permutation(n_coy)[:2]\r\n",
        "            prob1 = (1-Ps)/2\r\n",
        "            prob2 = prob1\r\n",
        "            pdr = np.random.permutation(D)\r\n",
        "            p1 = np.zeros((1, D))\r\n",
        "            p2 = np.zeros((1, D))\r\n",
        "            p1[0, pdr[0]] = 1  # Guarantee 1 charac. per individual\r\n",
        "            p2[0, pdr[1]] = 1  # Guarantee 1 charac. per individual\r\n",
        "            r = np.random.rand(1, D-2)\r\n",
        "            p1[0, pdr[2:]] = r < prob1\r\n",
        "            p2[0, pdr[2:]] = r > 1-prob2\r\n",
        "\r\n",
        "            # Eventual noise\r\n",
        "            n = np.logical_not(np.logical_or(p1, p2))\r\n",
        "\r\n",
        "            # Generate the pup considering intrinsic and extrinsic influence\r\n",
        "            pup = p1*coyotes_aux[parents[0], :] + \\\r\n",
        "                  p2*coyotes_aux[parents[1], :] + \\\r\n",
        "                  n*(VarMin + np.random.rand(1, D) * (VarMax - VarMin))\r\n",
        "\r\n",
        "            # Verify if the pup will survive\r\n",
        "            pup_cost = FOBJ(pup[0, :])\r\n",
        "            nfeval += 1\r\n",
        "            worst = np.flatnonzero(costs_aux > pup_cost)\r\n",
        "            if len(worst) > 0:\r\n",
        "                older = np.argsort(ages_aux[worst])\r\n",
        "                which = worst[older[::-1]]\r\n",
        "                coyotes_aux[which[0], :] = pup\r\n",
        "                costs_aux[which[0]] = pup_cost\r\n",
        "                ages_aux[which[0]] = 0\r\n",
        "\r\n",
        "            # Update the pack information\r\n",
        "            coyotes[packs[p], :] = coyotes_aux\r\n",
        "            costs[0, packs[p]] = costs_aux\r\n",
        "            ages[0, packs[p]] = ages_aux\r\n",
        "\r\n",
        "        # A coyote can leave a pack and enter in another pack (Eq. 4)\r\n",
        "        if n_packs > 1:\r\n",
        "            if np.random.rand() < p_leave:\r\n",
        "                rp = np.random.permutation(n_packs)[:2]\r\n",
        "                rc = [np.random.randint(0, n_coy), np.random.randint(0, n_coy)]\r\n",
        "                aux = packs[rp[0], rc[0]]\r\n",
        "                packs[rp[0], rc[0]] = packs[rp[1], rc[1]]\r\n",
        "                packs[rp[1], rc[1]] = aux\r\n",
        "\r\n",
        "        # Update coyotes ages\r\n",
        "        ages += 1\r\n",
        "\r\n",
        "        # Output variables (best alpha coyote among all alphas)\r\n",
        "        globalMin = np.min(costs[0, :])\r\n",
        "        ibest = np.argmin(costs)\r\n",
        "        globalParams = coyotes[ibest, :]\r\n",
        "\r\n",
        "    return globalMin, globalParams\r\n",
        "\r\n",
        "def Limita(X, D, VarMin, VarMax):\r\n",
        "    # Keep the coyotes in the search space (optimization problem constraint)\r\n",
        "    for abc in range(D):\r\n",
        "        X[abc] = max([min([X[abc], VarMax[abc]]), VarMin[abc]])\r\n",
        "\r\n",
        "    return X\r\n",
        "\r\n",
        "\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZLKIrlHHdjlT"
      },
      "source": [
        "## In the following part, Multi-layer Perceptron (MLP) is modeled, and its hyper-parameters are tuned by COA."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 722
        },
        "id": "JW8OKyD70ASe",
        "outputId": "95530494-9df1-42f7-b687-42cff034e65b"
      },
      "source": [
        "## MLP Classifier\r\n",
        "\r\n",
        "from sklearn.neural_network import MLPClassifier\r\n",
        "\r\n",
        "# Hyper parameters range for regularization\r\n",
        "\r\n",
        "def Sphere(X):\r\n",
        "#hidden_layer_sizes\r\n",
        "\r\n",
        "    h1 = math.floor((X[0]*300)) + 50\r\n",
        "\r\n",
        "#activation\r\n",
        "    if X[1]<0.25:\r\n",
        "        h2 = 'identity'\r\n",
        "    elif X[1] >= 0.25 and X[1] < 0.5:\r\n",
        "        h2 = 'logistic'\r\n",
        "    elif X[1] >= 0.5 and X[1] < 0.75:\r\n",
        "        h2 = 'tanh'\r\n",
        "    else:\r\n",
        "        h2 = 'relu'\r\n",
        "\r\n",
        "#solver\r\n",
        "    if X[2]<0.33:\r\n",
        "        h3 = 'lbfgs'\r\n",
        "    elif X[2] >= 0.33 and X[2] < 0.66:\r\n",
        "        h3 = 'sgd'\r\n",
        "    else:\r\n",
        "        h3 = 'adam'\r\n",
        "\r\n",
        "#batch_size\r\n",
        "    h4 = math.floor(X[3]*1000) + 200\r\n",
        "\r\n",
        "\r\n",
        "\r\n",
        "    MLP = MLPClassifier(hidden_layer_sizes = h1, activation = h2, solver = h3, batch_size = h4)\r\n",
        "    MLP.fit(X_train, Y_train)\r\n",
        "    y_pred_test_MLP = MLP.predict(X_test)\r\n",
        "    accuracy_score_MLP = accuracy_score(y_pred_test_MLP, Y_test)\r\n",
        "    y= -accuracy_score_MLP        \r\n",
        "\r\n",
        "\r\n",
        "    return y\r\n",
        "\r\n",
        "if __name__==\"__main__\":\r\n",
        "\r\n",
        "    import time\r\n",
        "    # Objective function definition\r\n",
        "\r\n",
        "\r\n",
        "    fobj = Sphere           # Function\r\n",
        "    d = 4                  # Problem dimension\r\n",
        "    lu = np.zeros((2, d))   # Boundaires\r\n",
        "    lu[0, :] = 0          # Lower boundaires\r\n",
        "    lu[1, :] = 1           # Upper boundaries\r\n",
        "\r\n",
        "    # COA parameters\r\n",
        "    n_packs = 8            # Number of Packs\r\n",
        "    n_coy = 3               # Number of coyotes\r\n",
        "    nfevalmax = 2       # Stopping criteria: maximum number of function evaluations\r\n",
        "\r\n",
        "    # Experimanetal variables\r\n",
        "    n_exper = 1             # Number of experiments\r\n",
        "    t = time.time()         # Time counter (and initial value)\r\n",
        "    y = np.zeros(n_exper)   # Experiments costs (for stats.)\r\n",
        "    for i in range(n_exper):\r\n",
        "        # Apply the COA to the problem with the defined parameters\r\n",
        "        gbest, par = COA(fobj, lu, nfevalmax, n_packs, n_coy)\r\n",
        "        # Keep the global best\r\n",
        "        y[i] = gbest\r\n",
        "        # Show the result (objective cost and time)\r\n",
        "        print(\"Experiment \", i+1, \", Best: \", gbest, \", time (s): \", time.time()-t)\r\n",
        "        t = time.time()\r\n",
        "\r\n",
        "    # Show the statistics\r\n",
        "    print(\"Statistics (min., avg., median, max., std.)\")\r\n",
        "    print([np.min(y), np.mean(y), np.median(y), np.max(y), np.std(y)])\r\n",
        "    print('parameter:')\r\n",
        "    print(par)\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:934: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:573: UserWarning: Training interrupted by user.\n",
            "  warnings.warn(\"Training interrupted by user.\")\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:934: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:573: UserWarning: Training interrupted by user.\n",
            "  warnings.warn(\"Training interrupted by user.\")\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:934: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:573: UserWarning: Training interrupted by user.\n",
            "  warnings.warn(\"Training interrupted by user.\")\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:934: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:573: UserWarning: Training interrupted by user.\n",
            "  warnings.warn(\"Training interrupted by user.\")\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:934: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-9-514201415d93>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     69\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_exper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m         \u001b[0;31m# Apply the COA to the problem with the defined parameters\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 71\u001b[0;31m         \u001b[0mgbest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpar\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCOA\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlu\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnfevalmax\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_packs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_coy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     72\u001b[0m         \u001b[0;31m# Keep the global best\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m         \u001b[0my\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgbest\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-8-330d353a91ed>\u001b[0m in \u001b[0;36mCOA\u001b[0;34m(FOBJ, lu, nfevalMAX, n_packs, n_coy)\u001b[0m\n\u001b[1;32m     32\u001b[0m     \u001b[0;31m# Evaluate coyotes adaptation (Eq. 3)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpop_total\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m         \u001b[0mcosts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFOBJ\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcoyotes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m     \u001b[0mnfeval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpop_total\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-9-514201415d93>\u001b[0m in \u001b[0;36mSphere\u001b[0;34m(X)\u001b[0m\n\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0mMLP\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMLPClassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden_layer_sizes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mactivation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msolver\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 36\u001b[0;31m     \u001b[0mMLP\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     37\u001b[0m     \u001b[0my_pred_test_MLP\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMLP\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m     \u001b[0maccuracy_score_MLP\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred_test_MLP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y)\u001b[0m\n\u001b[1;32m    993\u001b[0m         \"\"\"\n\u001b[1;32m    994\u001b[0m         return self._fit(X, y, incremental=(self.warm_start and\n\u001b[0;32m--> 995\u001b[0;31m                                             hasattr(self, \"classes_\")))\n\u001b[0m\u001b[1;32m    996\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    997\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py\u001b[0m in \u001b[0;36m_fit\u001b[0;34m(self, X, y, incremental)\u001b[0m\n\u001b[1;32m    373\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msolver\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'lbfgs'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    374\u001b[0m             self._fit_lbfgs(X, y, activations, deltas, coef_grads,\n\u001b[0;32m--> 375\u001b[0;31m                             intercept_grads, layer_units)\n\u001b[0m\u001b[1;32m    376\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    377\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py\u001b[0m in \u001b[0;36m_fit_lbfgs\u001b[0;34m(self, X, y, activations, deltas, coef_grads, intercept_grads, layer_units)\u001b[0m\n\u001b[1;32m    467\u001b[0m                     \u001b[0;34m\"gtol\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtol\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    468\u001b[0m                 },\n\u001b[0;32m--> 469\u001b[0;31m                 args=(X, y, activations, deltas, coef_grads, intercept_grads))\n\u001b[0m\u001b[1;32m    470\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_iter_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_check_optimize_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"lbfgs\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_res\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_iter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    471\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopt_res\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfun\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/scipy/optimize/_minimize.py\u001b[0m in \u001b[0;36mminimize\u001b[0;34m(fun, x0, args, method, jac, hess, hessp, bounds, constraints, tol, callback, options)\u001b[0m\n\u001b[1;32m    608\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mmeth\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'l-bfgs-b'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    609\u001b[0m         return _minimize_lbfgsb(fun, x0, args, jac, bounds,\n\u001b[0;32m--> 610\u001b[0;31m                                 callback=callback, **options)\n\u001b[0m\u001b[1;32m    611\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mmeth\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'tnc'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    612\u001b[0m         return _minimize_tnc(fun, x0, args, jac, bounds, callback=callback,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/scipy/optimize/lbfgsb.py\u001b[0m in \u001b[0;36m_minimize_lbfgsb\u001b[0;34m(fun, x0, args, jac, bounds, disp, maxcor, ftol, gtol, eps, maxfun, maxiter, iprint, callback, maxls, **unknown_options)\u001b[0m\n\u001b[1;32m    343\u001b[0m             \u001b[0;31m# until the completion of the current minimization iteration.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    344\u001b[0m             \u001b[0;31m# Overwrite f and g:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 345\u001b[0;31m             \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc_and_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    346\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mtask_str\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mb'NEW_X'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    347\u001b[0m             \u001b[0;31m# new iteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/scipy/optimize/lbfgsb.py\u001b[0m in \u001b[0;36mfunc_and_grad\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m    293\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mfunc_and_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 295\u001b[0;31m             \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    296\u001b[0m             \u001b[0mg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjac\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    297\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/scipy/optimize/optimize.py\u001b[0m in \u001b[0;36mfunction_wrapper\u001b[0;34m(*wrapper_args)\u001b[0m\n\u001b[1;32m    325\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfunction_wrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mwrapper_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    326\u001b[0m         \u001b[0mncalls\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 327\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwrapper_args\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    328\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    329\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mncalls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunction_wrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/scipy/optimize/optimize.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, x, *args)\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m         \u001b[0mfg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjac\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py\u001b[0m in \u001b[0;36m_loss_grad_lbfgs\u001b[0;34m(self, packed_coef_inter, X, y, activations, deltas, coef_grads, intercept_grads)\u001b[0m\n\u001b[1;32m    175\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpacked_coef_inter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m         loss, coef_grads, intercept_grads = self._backprop(\n\u001b[0;32m--> 177\u001b[0;31m             X, y, activations, deltas, coef_grads, intercept_grads)\n\u001b[0m\u001b[1;32m    178\u001b[0m         \u001b[0mgrad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_pack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcoef_grads\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mintercept_grads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    179\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py\u001b[0m in \u001b[0;36m_backprop\u001b[0;34m(self, X, y, activations, deltas, coef_grads, intercept_grads)\u001b[0m\n\u001b[1;32m    219\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    220\u001b[0m         \u001b[0;31m# Forward propagate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 221\u001b[0;31m         \u001b[0mactivations\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_pass\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mactivations\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    222\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m         \u001b[0;31m# Get loss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py\u001b[0m in \u001b[0;36m_forward_pass\u001b[0;34m(self, activations)\u001b[0m\n\u001b[1;32m    102\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_layers_\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m             activations[i + 1] = safe_sparse_dot(activations[i],\n\u001b[0;32m--> 104\u001b[0;31m                                                  self.coefs_[i])\n\u001b[0m\u001b[1;32m    105\u001b[0m             \u001b[0mactivations\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mintercepts_\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/utils/extmath.py\u001b[0m in \u001b[0;36msafe_sparse_dot\u001b[0;34m(a, b, dense_output)\u001b[0m\n\u001b[1;32m    149\u001b[0m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    150\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 151\u001b[0;31m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0ma\u001b[0m \u001b[0;34m@\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    152\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    153\u001b[0m     if (sparse.issparse(a) and sparse.issparse(b)\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/scipy/sparse/base.py\u001b[0m in \u001b[0;36m__matmul__\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m    562\u001b[0m             raise ValueError(\"Scalar operands are not allowed, \"\n\u001b[1;32m    563\u001b[0m                              \"use '*' instead\")\n\u001b[0;32m--> 564\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__mul__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mother\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    565\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    566\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__rmatmul__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/scipy/sparse/base.py\u001b[0m in \u001b[0;36m__mul__\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m    473\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mul_vector\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mother\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mM\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    474\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mN\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 475\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mul_multivector\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mother\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    476\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    477\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misscalarlike\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mother\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/scipy/sparse/compressed.py\u001b[0m in \u001b[0;36m_mul_multivector\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m    485\u001b[0m         \u001b[0mfn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_sparsetools\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'_matvecs'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    486\u001b[0m         fn(M, N, n_vecs, self.indptr, self.indices, self.data,\n\u001b[0;32m--> 487\u001b[0;31m            other.ravel(), result.ravel())\n\u001b[0m\u001b[1;32m    488\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    489\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7oKxclwhdrR4"
      },
      "source": [
        "### In the following part, the accuracy of MLP for training and validation data is assessed, and printed. Moreover, the optimal value of MLP's hyper-parameters is presented after running the following code. Needless to say, initially, the previous part should be run, and afterward, you can assess MLP in this part."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7xHx4QIV00qC",
        "outputId": "36ba71fc-dd6d-4ae1-cc5b-2c25d292da28"
      },
      "source": [
        "#Check the accuracy of MLP Classifier\r\n",
        "X = par\r\n",
        "\r\n",
        "#hidden_layer_sizes\r\n",
        "\r\n",
        "h1 = math.floor((X[0]*300)) + 50\r\n",
        "\r\n",
        "#activation\r\n",
        "if X[1]<0.25:\r\n",
        "    h2 = 'identity'\r\n",
        "elif X[1] >= 0.25 and X[1] < 0.5:\r\n",
        "    h2 = 'logistic'\r\n",
        "elif X[1] >= 0.5 and X[1] < 0.75:\r\n",
        "    h2 = 'tanh'\r\n",
        "else:\r\n",
        "    h2 = 'relu'\r\n",
        "\r\n",
        "#solver\r\n",
        "if X[2]<0.33:\r\n",
        "    h3 = 'lbfgs'\r\n",
        "elif X[2] >= 0.33 and X[2] < 0.66:\r\n",
        "    h3 = 'sgd'\r\n",
        "else:\r\n",
        "    h3 = 'adam'\r\n",
        "\r\n",
        "#batch_size\r\n",
        "h4 = math.floor(X[3]*1000) + 200\r\n",
        "\r\n",
        "\r\n",
        "MLP = MLPClassifier(hidden_layer_sizes = h1, activation = h2, solver = h3, batch_size = h4)\r\n",
        "MLP.fit(X_train, Y_train)\r\n",
        "y_pred_train_MLP = MLP.predict(X_train)\r\n",
        "y_pred_val_MLP = MLP.predict(X_test)\r\n",
        "\r\n",
        "accuracy_score_train = accuracy_score(y_pred_train_MLP, Y_train)\r\n",
        "accuracy_score_val = accuracy_score(y_pred_val_MLP, Y_test)\r\n",
        "print('Training data accuracy-MLP', accuracy_score_train)\r\n",
        "print('Validation data accuracy-MLP', accuracy_score_val)\r\n",
        "print('hidden_layer_sizes: ', h1, ' activation: ', h2, ' solver: ', h3, ' batch_size: ', h4)\r\n",
        "\r\n",
        "\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:934: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Training data accuracy-MLP 1.0\n",
            "Validation data accuracy-MLP 0.4493939393939394\n",
            "hidden_layer_sizes:  268  activation:  relu  solver:  adam  batch_size:  815\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uq7b07NJFf7D"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2m5QlJw_eKjj"
      },
      "source": [
        "# In the following part, K-nearest neighborhood (KNN) is modeled, and its hyper-parameters are tuned by COA."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dtl5VZpG4KV5"
      },
      "source": [
        "## KNN Classifier ## (Optimized by COA)\r\n",
        "\r\n",
        "from sklearn.neighbors import KNeighborsClassifier\r\n",
        "\r\n",
        "\r\n",
        "def Sphere(X):\r\n",
        "\r\n",
        "## Defining the range of hyperparameters\r\n",
        "\r\n",
        "#leaf_size\r\n",
        "\r\n",
        "    h1 = math.floor((X[0]*100)) + 1\r\n",
        "\r\n",
        "#algorithm\r\n",
        "    if X[1]<0.25:\r\n",
        "        h2 = 'auto'\r\n",
        "    elif X[1] >= 0.25 and X[1] < 0.5:\r\n",
        "        h2 = 'ball_tree'\r\n",
        "    elif X[1] >= 0.5 and X[1] < 0.75:\r\n",
        "        h2 = 'kd_tree'\r\n",
        "    else:\r\n",
        "        h2 = 'brute'\r\n",
        "\r\n",
        "\r\n",
        "#p\r\n",
        "    h3 = math.floor((X[2])*2) + 1\r\n",
        "\r\n",
        "# n_neighbors\r\n",
        "    h4 = math.floor(X[3]*100) + 4\r\n",
        "\r\n",
        "    KNN = KNeighborsClassifier(leaf_size = h1, algorithm = h2, p = h3, n_neighbors = h4)\r\n",
        "    KNN.fit(X_train, Y_train)\r\n",
        "    y_pred_test_KNN = KNN.predict(X_test)\r\n",
        "    accuracy_score_KNN = accuracy_score(y_pred_test_KNN, Y_test)\r\n",
        "    y= -accuracy_score_KNN        \r\n",
        "\r\n",
        "\r\n",
        "\r\n",
        "    return y\r\n",
        "\r\n",
        "if __name__==\"__main__\":\r\n",
        "\r\n",
        "    import time\r\n",
        "    # Objective function definition\r\n",
        "\r\n",
        "\r\n",
        "    fobj = Sphere           # Function\r\n",
        "    d = 4                  # Problem dimension\r\n",
        "    lu = np.zeros((2, d))   # Boundaires\r\n",
        "    lu[0, :] = 0          # Lower boundaires\r\n",
        "    lu[1, :] = 1           # Upper boundaries\r\n",
        "\r\n",
        "    # COA parameters\r\n",
        "    n_packs = 8            # Number of Packs\r\n",
        "    n_coy = 3               # Number of coyotes\r\n",
        "    nfevalmax = 3       # Stopping criteria: maximum number of function evaluations\r\n",
        "\r\n",
        "    # Experimanetal variables\r\n",
        "    n_exper = 1             # Number of experiments\r\n",
        "    t = time.time()         # Time counter (and initial value)\r\n",
        "    y = np.zeros(n_exper)   # Experiments costs (for stats.)\r\n",
        "    for i in range(n_exper):\r\n",
        "        # Apply the COA to the problem with the defined parameters\r\n",
        "        gbest, par = COA(fobj, lu, nfevalmax, n_packs, n_coy)\r\n",
        "        # Keep the global best\r\n",
        "        y[i] = gbest\r\n",
        "        # Show the result (objective cost and time)\r\n",
        "        print(\"Experiment \", i+1, \", Best: \", gbest, \", time (s): \", time.time()-t)\r\n",
        "        t = time.time()\r\n",
        "\r\n",
        "    # Show the statistics\r\n",
        "    print(\"Statistics (min., avg., median, max., std.)\")\r\n",
        "    print([np.min(y), np.mean(y), np.median(y), np.max(y), np.std(y)])\r\n",
        "    print('parameter:')\r\n",
        "    print(par)\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HGz1LCT-Fm7w"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UECsv67ZeRDi"
      },
      "source": [
        "### In the following part, the accuracy of KNN for training and validation data is assessed, and printed. Moreover, the optimal value of KNN's hyper-parameters is presented after running the following code. Needless to say, initially, the previous part should be run, and afterward, you can assess KNN in this part."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f5i-T2004XTE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f5fc4ed1-4487-47fa-f309-05b0112168db"
      },
      "source": [
        "#Check the accuracy of KNN\r\n",
        "X = par\r\n",
        "\r\n",
        "#leaf_size\r\n",
        "h1 = math.floor((X[0]*100)) + 1\r\n",
        "\r\n",
        "#algorithm\r\n",
        "if X[1]<0.25:\r\n",
        "    h2 = 'auto'\r\n",
        "elif X[1] >= 0.25 and X[1] < 0.5:\r\n",
        "    h2 = 'ball_tree'\r\n",
        "elif X[1] >= 0.5 and X[1] < 0.75:\r\n",
        "    h2 = 'kd_tree'\r\n",
        "else:\r\n",
        "    h2 = 'brute'\r\n",
        "\r\n",
        "\r\n",
        "#p\r\n",
        "h3 = math.floor((X[2])*2) + 1\r\n",
        "\r\n",
        "# n_neighbors\r\n",
        "h4 = math.floor(X[3]*100) + 4\r\n",
        "\r\n",
        "KNN = KNeighborsClassifier(leaf_size = h1, algorithm = h2, p = h3, n_neighbors = h4)\r\n",
        "KNN.fit(X_train, Y_train)\r\n",
        "y_pred_val_KNN = KNN.predict(X_test)\r\n",
        "y_pred_train_KNN = KNN.predict(X_train)\r\n",
        "accuracy_score_KNN_val = accuracy_score(y_pred_val_KNN, Y_test)\r\n",
        "accuracy_score_KNN_train = accuracy_score(y_pred_train_KNN, Y_train)\r\n",
        "\r\n",
        "\r\n",
        "print('Validation accuracy-KNN', accuracy_score_KNN_val)\r\n",
        "print('Training accuracy-KNN', accuracy_score_KNN_train)\r\n",
        "print('leaf size: ', h1, ' algorithm: ', h2, ' p: ', h3, ' n_neighbors: ', h4)\r\n",
        "\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:25: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/neighbors/_base.py:414: UserWarning: cannot use tree with sparse input: using brute force\n",
            "  warnings.warn(\"cannot use tree with sparse input: \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Validation accuracy 0.4766666666666667\n",
            "Training accuracy 0.5832835820895522\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QTu75drWGMr6"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ttBYUn15eZno"
      },
      "source": [
        "#### In the following part, Logistic regression (LR) is modeled, and its hyper-parameters are tuned by COA."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lsf423dr7bdY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1f15e2ea-072f-4917-d1a9-b359c9dfdd9d"
      },
      "source": [
        "## Logistic regression ## Optimized by COA\r\n",
        "from sklearn.linear_model import LogisticRegression\r\n",
        "\r\n",
        "def Sphere(X):\r\n",
        "\r\n",
        "## Defining the range of hyperparameters\r\n",
        "#C\r\n",
        "    h1 = (X[0]+1e-10)*20\r\n",
        "\r\n",
        "#Solver\r\n",
        "    if X[1]<0.2:\r\n",
        "        h2 = 'newton-cg'\r\n",
        "    elif X[1] >= 0.2 and X[1] < 0.4:\r\n",
        "        h2 = 'lbfgs'\r\n",
        "    elif X[1] >= 0.4 and X[1] < 0.6:\r\n",
        "        h2 = 'liblinear'\r\n",
        "    elif X[1] >= 0.6 and X[1] < 0.8:\r\n",
        "        h2 = 'sag'\r\n",
        "    else:\r\n",
        "        h2 = 'saga'\r\n",
        "\r\n",
        "\r\n",
        "#tol\r\n",
        "    h3 = X[2]+1e-12\r\n",
        "\r\n",
        "\r\n",
        "    LR = LogisticRegression(C = h1, solver = h2, tol = h3)\r\n",
        "    LR.fit(X_train, Y_train)\r\n",
        "    y_pred_test_LR = LR.predict(X_test)\r\n",
        "    accuracy_score_LR = accuracy_score(y_pred_test_LR, Y_test)\r\n",
        "    y= -accuracy_score_LR        \r\n",
        "\r\n",
        "\r\n",
        "    return y\r\n",
        "\r\n",
        "if __name__==\"__main__\":\r\n",
        "\r\n",
        "    import time\r\n",
        "    # Objective function definition\r\n",
        "\r\n",
        "\r\n",
        "    fobj = Sphere           # Function\r\n",
        "    d = 3                  # Problem dimension\r\n",
        "    lu = np.zeros((2, d))   # Boundaires\r\n",
        "    lu[0, :] = 0          # Lower boundaires\r\n",
        "    lu[1, :] = 1           # Upper boundaries\r\n",
        "\r\n",
        "    # COA parameters\r\n",
        "    n_packs = 8            # Number of Packs\r\n",
        "    n_coy = 3               # Number of coyotes\r\n",
        "    nfevalmax = 2       # Stopping criteria: maximum number of function evaluations\r\n",
        "\r\n",
        "    # Experimanetal variables\r\n",
        "    n_exper = 1             # Number of experiments\r\n",
        "    t = time.time()         # Time counter (and initial value)\r\n",
        "    y = np.zeros(n_exper)   # Experiments costs (for stats.)\r\n",
        "    for i in range(n_exper):\r\n",
        "        # Apply the COA to the problem with the defined parameters\r\n",
        "        gbest, par = COA(fobj, lu, nfevalmax, n_packs, n_coy)\r\n",
        "        # Keep the global best\r\n",
        "        y[i] = gbest\r\n",
        "        # Show the result (objective cost and time)\r\n",
        "        print(\"Experiment \", i+1, \", Best: \", gbest, \", time (s): \", time.time()-t)\r\n",
        "        t = time.time()\r\n",
        "\r\n",
        "    # Show the statistics\r\n",
        "    print(\"Statistics (min., avg., median, max., std.)\")\r\n",
        "    print([np.min(y), np.mean(y), np.median(y), np.max(y), np.std(y)])\r\n",
        "    print('parameter:')\r\n",
        "    print(par)\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Experiment  1 , Best:  -0.435 , time (s):  557.5335052013397\n",
            "Statistics (min., avg., median, max., std.)\n",
            "[-0.435, -0.435, -0.435, -0.435, 0.0]\n",
            "parameter:\n",
            "[0.90921717 0.86037446 0.24645592]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aBz8eETAFzpW"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TqKVeS65egyP"
      },
      "source": [
        "### In the following part, the accuracy of Logistic Regression (LR) for training and validation data is assessed, and printed. Moreover, the optimal value of LR's hyper-parameters is presented after running the following code. Needless to say, initially, the previous part should be run, and afterward, you can assess LR in this part."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vlwDBtfmz_Ah",
        "outputId": "dc4025e6-581c-4621-be11-48def528f183"
      },
      "source": [
        "#Check the accuracy of LR\r\n",
        "X = par\r\n",
        "\r\n",
        "#C\r\n",
        "h1 = (X[0]+1e-10)*20\r\n",
        "\r\n",
        "#Solver\r\n",
        "if X[1]<0.2:\r\n",
        "    h2 = 'newton-cg'\r\n",
        "elif X[1] >= 0.2 and X[1] < 0.4:\r\n",
        "    h2 = 'lbfgs'\r\n",
        "elif X[1] >= 0.4 and X[1] < 0.6:\r\n",
        "    h2 = 'liblinear'\r\n",
        "elif X[1] >= 0.6 and X[1] < 0.8:\r\n",
        "    h2 = 'sag'\r\n",
        "else:\r\n",
        "    h2 = 'saga'\r\n",
        "\r\n",
        "#tol\r\n",
        "h3 = X[2]+1e-12\r\n",
        "\r\n",
        "\r\n",
        "LR = LogisticRegression(C = h1, solver = h2, tol = h3)\r\n",
        "LR.fit(X_train, Y_train)\r\n",
        "y_pred_val_LR = LR.predict(X_test)\r\n",
        "y_pred_train_LR = LR.predict(X_train)\r\n",
        "accuracy_score_LR_val = accuracy_score(y_pred_val_LR, Y_test)\r\n",
        "accuracy_score_LR_train = accuracy_score(y_pred_train_LR, Y_train)\r\n",
        "\r\n",
        "\r\n",
        "print('Training accuracy-LR', accuracy_score_LR_train)\r\n",
        "print('Validation accuracy-LR', accuracy_score_LR_val)\r\n",
        "print('C: ', h1, ' solver: ', h2, ' tol: ', h3)\r\n",
        "\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Validation accuracy-LR 0.421\n",
            "Training accuracy-LR 0.5398888888888889\n",
            "C:  18.184343487900986  solver:  saga  tol:  0.24645592253494067\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3taiMEw0eppm"
      },
      "source": [
        "# In the following part, Kernalized Support Vector Machine (SVM) is modeled, and its hyper-parameters are tuned by COA."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7jSLkTm61Tz4",
        "outputId": "66581117-3fa5-4288-91cc-84bddc02f4fc"
      },
      "source": [
        "## Kernalized Support Vector Machine (SVM)'s hyper-parameters are optimized by COA ##\r\n",
        "from sklearn import svm\r\n",
        "\r\n",
        "\r\n",
        "def Sphere(X):\r\n",
        "\r\n",
        "#Defining the range of hyperparameters\r\n",
        "\r\n",
        "#C\r\n",
        "    h1 = (X[0]+1e-10)*100\r\n",
        "\r\n",
        "#Kernel\r\n",
        "    if X[1]<0.25:\r\n",
        "        h2 = 'linear'\r\n",
        "    elif X[1] >= 0.25 and X[1] < 0.5:\r\n",
        "        h2 = 'poly'\r\n",
        "    elif X[1] >= 0.5 and X[1] < 0.75:\r\n",
        "        h2 = 'rbf'\r\n",
        "    else:\r\n",
        "        h2 = 'sigmoid'\r\n",
        "\r\n",
        "#gama\r\n",
        "    if X[2]<0.5:\r\n",
        "        h3 = 'scale'\r\n",
        "    else:\r\n",
        "        h3 = 'auto'\r\n",
        "\r\n",
        "#tol\r\n",
        "    h4 = X[3]+1e-12\r\n",
        "\r\n",
        "\r\n",
        "\r\n",
        "    SVM = svm.SVC(C = h1, kernel = h2, gamma = h3, tol = h4)\r\n",
        "    SVM.fit(X_train, Y_train)\r\n",
        "    y_pred_test_SVM = SVM.predict(X_test)\r\n",
        "    accuracy_score_SVM = accuracy_score(y_pred_test_SVM, Y_test)\r\n",
        "    y= -accuracy_score_SVM        \r\n",
        "\r\n",
        "    return y\r\n",
        "\r\n",
        "if __name__==\"__main__\":\r\n",
        "\r\n",
        "    import time\r\n",
        "    # Objective function definition\r\n",
        "\r\n",
        "\r\n",
        "    fobj = Sphere           # Function\r\n",
        "    d = 4                  # Problem dimension\r\n",
        "    lu = np.zeros((2, d))   # Boundaires\r\n",
        "    lu[0, :] = 0          # Lower boundaires\r\n",
        "    lu[1, :] = 1           # Upper boundaries\r\n",
        "\r\n",
        "    # COA parameters\r\n",
        "    n_packs = 8            # Number of Packs\r\n",
        "    n_coy = 3               # Number of coyotes\r\n",
        "    nfevalmax = 2       # Stopping criteria: maximum number of function evaluations\r\n",
        "\r\n",
        "    # Experimanetal variables\r\n",
        "    n_exper = 1             # Number of experiments\r\n",
        "    t = time.time()         # Time counter (and initial value)\r\n",
        "    y = np.zeros(n_exper)   # Experiments costs (for stats.)\r\n",
        "    for i in range(n_exper):\r\n",
        "        # Apply the COA to the problem with the defined parameters\r\n",
        "        gbest, par = COA(fobj, lu, nfevalmax, n_packs, n_coy)\r\n",
        "        # Keep the global best\r\n",
        "        y[i] = gbest\r\n",
        "        # Show the result (objective cost and time)\r\n",
        "        print(\"Experiment \", i+1, \", Best: \", gbest, \", time (s): \", time.time()-t)\r\n",
        "        t = time.time()\r\n",
        "\r\n",
        "    # Show the statistics\r\n",
        "    print(\"Statistics (min., avg., median, max., std.)\")\r\n",
        "    print([np.min(y), np.mean(y), np.median(y), np.max(y), np.std(y)])\r\n",
        "    print('parameter:')\r\n",
        "    print(par)\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Experiment  1 , Best:  -0.541 , time (s):  2073.1783089637756\n",
            "Statistics (min., avg., median, max., std.)\n",
            "[-0.541, -0.541, -0.541, -0.541, 0.0]\n",
            "parameter:\n",
            "[0.5534643  0.68298537 0.04805757 0.19232164]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "07Y18v_hF8-g"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RMv1AOR3e0iD"
      },
      "source": [
        "### In the following part, the accuracy of Kernalized Support Vector Machine (SVM) for training and validation data is assessed, and printed. Moreover, the optimal value of SVM's hyper-parameters is presented after running the following code. Needless to say, initially, the previous part should be run, and afterward, you can assess SVM in this part."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kXhuDBvz9nmU",
        "outputId": "f58bc55d-ebf0-4b04-90a2-585f8ee3ccbe"
      },
      "source": [
        "#Check the accuracy of SVM\r\n",
        "X = par\r\n",
        "\r\n",
        "#C\r\n",
        "h1 = (X[0]+1e-10)*100\r\n",
        "\r\n",
        "#Kernel\r\n",
        "if X[1]<0.25:\r\n",
        "    h2 = 'linear'\r\n",
        "elif X[1] >= 0.25 and X[1] < 0.5:\r\n",
        "    h2 = 'poly'\r\n",
        "elif X[1] >= 0.5 and X[1] < 0.75:\r\n",
        "    h2 = 'rbf'\r\n",
        "else:\r\n",
        "    h2 = 'sigmoid'\r\n",
        "\r\n",
        "#gama\r\n",
        "if X[2]<0.5:\r\n",
        "    h3 = 'scale'\r\n",
        "else:\r\n",
        "    h3 = 'auto'\r\n",
        "\r\n",
        "#tol\r\n",
        "h4 = X[3]+1e-12\r\n",
        "\r\n",
        "\r\n",
        "\r\n",
        "SVM = svm.SVC(C = h1, kernel = h2, gamma = h3, tol = h4)\r\n",
        "SVM.fit(X_train, Y_train)\r\n",
        "y_pred_val_SVM = SVM.predict(X_test)\r\n",
        "y_pred_train_SVM = SVM.predict(X_train)\r\n",
        "accuracy_score_SVM_val = accuracy_score(y_pred_val_SVM, Y_test)\r\n",
        "accuracy_score_SVM_train = accuracy_score(y_pred_train_SVM, Y_train)\r\n",
        "\r\n",
        "\r\n",
        "print('Training accuracy-SVM', accuracy_score_SVM_train)\r\n",
        "print('Validation accuracy-SVM', accuracy_score_SVM_val)\r\n",
        "print('C: ', h1, ' kernel: ', h2, ' gamma: ', h3, ' tol: ', h4)\r\n",
        "\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Training accuracy-SVM 1.0\n",
            "Validation accuracy-SVM 0.541\n",
            "C:  55.3464299179948  kernel:  rbf  gamma:  scale  tol:  0.1923216443525038\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}