finetune.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-31T16:43:58.248636Z",
     "iopub.status.busy": "2022-05-31T16:43:58.242200Z",
     "iopub.status.idle": "2022-05-31T16:44:00.475539Z",
     "shell.execute_reply": "2022-05-31T16:44:00.476697Z"
    }
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from transformers import (\n",
    "    AutoTokenizer,\n",
    "    AutoModelForMaskedLM,\n",
    "    Trainer,\n",
    "    TrainingArguments,\n",
    "    DataCollatorForLanguageModeling,\n",
    ")\n",
    "from datasets import Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-31T16:44:00.488633Z",
     "iopub.status.busy": "2022-05-31T16:44:00.487326Z",
     "iopub.status.idle": "2022-05-31T16:44:00.521354Z",
     "shell.execute_reply": "2022-05-31T16:44:00.520069Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(\"w_dict.json\", \"r\") as f:\n",
    "    w_dict = json.load(f)\n",
    "with open(\"ap_dict.json\", \"r\") as f:\n",
    "    ap_dict = json.load(f)\n",
    "with open(\"technique_dict.json\", \"r\") as f:\n",
    "    technique_dict = json.load(f)\n",
    "with open(\"tactic_dict.json\", \"r\") as f:\n",
    "    tactic_dict = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-31T16:44:00.531587Z",
     "iopub.status.busy": "2022-05-31T16:44:00.530573Z",
     "iopub.status.idle": "2022-05-31T16:44:00.562301Z",
     "shell.execute_reply": "2022-05-31T16:44:00.561342Z"
    }
   },
   "outputs": [],
   "source": [
    "f = open(\"cwe_mitigation_ids_temp.json\")\n",
    "w_mitigation = json.load(f)\n",
    "\n",
    "f = open(\"capec_mitigation_temp.json\")\n",
    "ap_mitigation = json.load(f)\n",
    "\n",
    "f = open(\"technique_mitigation_temp.json\")\n",
    "technique_mitigation = json.load(f)\n",
    "\n",
    "f = open(\"technique_detection_temp.json\")\n",
    "technique_detection = json.load(f)\n",
    "\n",
    "f = open(\"capec_detection_temp.json\")\n",
    "ap_detection = json.load(f)\n",
    "\n",
    "f = open(\"cwe_detection_temp.json\")\n",
    "w_detection = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-31T16:44:00.583104Z",
     "iopub.status.busy": "2022-05-31T16:44:00.577722Z",
     "iopub.status.idle": "2022-06-01T07:48:20.152631Z",
     "shell.execute_reply": "2022-06-01T07:48:20.151450Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "baa1d9d33d1f4c6889c233fa8e46a2be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74429fe2fafa4bca85b1dc9d458e5ea1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "/home/ubuntu/venv_BRON_ML/lib/python3.6/site-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  FutureWarning,\n",
      "***** Running training *****\n",
      "  Num examples = 3809\n",
      "  Num Epochs = 50\n",
      "  Instantaneous batch size per device = 2\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 2\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 95250\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='95250' max='95250' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [95250/95250 15:04:13, Epoch 50/50]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.037800</td>\n",
       "      <td>3.071460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2.818700</td>\n",
       "      <td>2.728717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2.523200</td>\n",
       "      <td>2.731335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2.429700</td>\n",
       "      <td>2.810951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>2.366800</td>\n",
       "      <td>2.610152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2.237100</td>\n",
       "      <td>2.765339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>1.995100</td>\n",
       "      <td>2.579362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2.004600</td>\n",
       "      <td>2.766116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>1.900000</td>\n",
       "      <td>2.929121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>1.747700</td>\n",
       "      <td>2.646270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>1.632000</td>\n",
       "      <td>2.708073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>1.631600</td>\n",
       "      <td>2.444237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>1.470600</td>\n",
       "      <td>2.735118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>1.492900</td>\n",
       "      <td>2.760572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>1.457700</td>\n",
       "      <td>2.572687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>1.354200</td>\n",
       "      <td>2.510020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>1.460000</td>\n",
       "      <td>2.470235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>1.377500</td>\n",
       "      <td>2.682118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>1.230100</td>\n",
       "      <td>2.592304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>1.289500</td>\n",
       "      <td>2.731010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>1.186600</td>\n",
       "      <td>2.680697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>1.034500</td>\n",
       "      <td>2.803011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>1.003400</td>\n",
       "      <td>2.597025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>1.030500</td>\n",
       "      <td>2.884346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>0.870000</td>\n",
       "      <td>2.679439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>0.761800</td>\n",
       "      <td>2.830223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>1.086600</td>\n",
       "      <td>2.924690</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>0.826300</td>\n",
       "      <td>2.744296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>0.856500</td>\n",
       "      <td>2.512988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>0.754500</td>\n",
       "      <td>2.680164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>0.803100</td>\n",
       "      <td>2.596526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>0.690300</td>\n",
       "      <td>2.503499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>0.625300</td>\n",
       "      <td>2.703051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34</td>\n",
       "      <td>0.654000</td>\n",
       "      <td>2.360031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>0.598000</td>\n",
       "      <td>2.258506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>0.662300</td>\n",
       "      <td>2.424518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>0.516800</td>\n",
       "      <td>2.594737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>0.528400</td>\n",
       "      <td>2.690280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39</td>\n",
       "      <td>0.503900</td>\n",
       "      <td>2.483672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>0.475200</td>\n",
       "      <td>2.717351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41</td>\n",
       "      <td>0.602100</td>\n",
       "      <td>2.680578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42</td>\n",
       "      <td>0.463200</td>\n",
       "      <td>2.705710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43</td>\n",
       "      <td>0.495400</td>\n",
       "      <td>2.457725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44</td>\n",
       "      <td>0.441400</td>\n",
       "      <td>2.459729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>0.364200</td>\n",
       "      <td>2.655340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>0.343300</td>\n",
       "      <td>2.637520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>0.404800</td>\n",
       "      <td>2.409842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>0.382000</td>\n",
       "      <td>2.856293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49</td>\n",
       "      <td>0.332300</td>\n",
       "      <td>2.594057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>0.398100</td>\n",
       "      <td>2.635175</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-1905\n",
      "Configuration saved in bert_base/checkpoint-1905/config.json\n",
      "Model weights saved in bert_base/checkpoint-1905/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-3810\n",
      "Configuration saved in bert_base/checkpoint-3810/config.json\n",
      "Model weights saved in bert_base/checkpoint-3810/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-1905] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-5715\n",
      "Configuration saved in bert_base/checkpoint-5715/config.json\n",
      "Model weights saved in bert_base/checkpoint-5715/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-7620\n",
      "Configuration saved in bert_base/checkpoint-7620/config.json\n",
      "Model weights saved in bert_base/checkpoint-7620/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-5715] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-9525\n",
      "Configuration saved in bert_base/checkpoint-9525/config.json\n",
      "Model weights saved in bert_base/checkpoint-9525/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-3810] due to args.save_total_limit\n",
      "Deleting older checkpoint [bert_base/checkpoint-7620] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-11430\n",
      "Configuration saved in bert_base/checkpoint-11430/config.json\n",
      "Model weights saved in bert_base/checkpoint-11430/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-13335\n",
      "Configuration saved in bert_base/checkpoint-13335/config.json\n",
      "Model weights saved in bert_base/checkpoint-13335/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-9525] due to args.save_total_limit\n",
      "Deleting older checkpoint [bert_base/checkpoint-11430] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-15240\n",
      "Configuration saved in bert_base/checkpoint-15240/config.json\n",
      "Model weights saved in bert_base/checkpoint-15240/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-17145\n",
      "Configuration saved in bert_base/checkpoint-17145/config.json\n",
      "Model weights saved in bert_base/checkpoint-17145/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-15240] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-19050\n",
      "Configuration saved in bert_base/checkpoint-19050/config.json\n",
      "Model weights saved in bert_base/checkpoint-19050/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-17145] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-20955\n",
      "Configuration saved in bert_base/checkpoint-20955/config.json\n",
      "Model weights saved in bert_base/checkpoint-20955/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-19050] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-22860\n",
      "Configuration saved in bert_base/checkpoint-22860/config.json\n",
      "Model weights saved in bert_base/checkpoint-22860/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-13335] due to args.save_total_limit\n",
      "Deleting older checkpoint [bert_base/checkpoint-20955] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-24765\n",
      "Configuration saved in bert_base/checkpoint-24765/config.json\n",
      "Model weights saved in bert_base/checkpoint-24765/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-26670\n",
      "Configuration saved in bert_base/checkpoint-26670/config.json\n",
      "Model weights saved in bert_base/checkpoint-26670/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-24765] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-28575\n",
      "Configuration saved in bert_base/checkpoint-28575/config.json\n",
      "Model weights saved in bert_base/checkpoint-28575/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-26670] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-30480\n",
      "Configuration saved in bert_base/checkpoint-30480/config.json\n",
      "Model weights saved in bert_base/checkpoint-30480/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-28575] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-32385\n",
      "Configuration saved in bert_base/checkpoint-32385/config.json\n",
      "Model weights saved in bert_base/checkpoint-32385/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-30480] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-34290\n",
      "Configuration saved in bert_base/checkpoint-34290/config.json\n",
      "Model weights saved in bert_base/checkpoint-34290/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-32385] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-36195\n",
      "Configuration saved in bert_base/checkpoint-36195/config.json\n",
      "Model weights saved in bert_base/checkpoint-36195/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-34290] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-38100\n",
      "Configuration saved in bert_base/checkpoint-38100/config.json\n",
      "Model weights saved in bert_base/checkpoint-38100/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-36195] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-40005\n",
      "Configuration saved in bert_base/checkpoint-40005/config.json\n",
      "Model weights saved in bert_base/checkpoint-40005/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-38100] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-41910\n",
      "Configuration saved in bert_base/checkpoint-41910/config.json\n",
      "Model weights saved in bert_base/checkpoint-41910/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-40005] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-43815\n",
      "Configuration saved in bert_base/checkpoint-43815/config.json\n",
      "Model weights saved in bert_base/checkpoint-43815/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-41910] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-45720\n",
      "Configuration saved in bert_base/checkpoint-45720/config.json\n",
      "Model weights saved in bert_base/checkpoint-45720/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-43815] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-47625\n",
      "Configuration saved in bert_base/checkpoint-47625/config.json\n",
      "Model weights saved in bert_base/checkpoint-47625/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-45720] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-49530\n",
      "Configuration saved in bert_base/checkpoint-49530/config.json\n",
      "Model weights saved in bert_base/checkpoint-49530/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-47625] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-51435\n",
      "Configuration saved in bert_base/checkpoint-51435/config.json\n",
      "Model weights saved in bert_base/checkpoint-51435/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-49530] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-53340\n",
      "Configuration saved in bert_base/checkpoint-53340/config.json\n",
      "Model weights saved in bert_base/checkpoint-53340/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-51435] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-55245\n",
      "Configuration saved in bert_base/checkpoint-55245/config.json\n",
      "Model weights saved in bert_base/checkpoint-55245/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-53340] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-57150\n",
      "Configuration saved in bert_base/checkpoint-57150/config.json\n",
      "Model weights saved in bert_base/checkpoint-57150/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-55245] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-59055\n",
      "Configuration saved in bert_base/checkpoint-59055/config.json\n",
      "Model weights saved in bert_base/checkpoint-59055/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-57150] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-60960\n",
      "Configuration saved in bert_base/checkpoint-60960/config.json\n",
      "Model weights saved in bert_base/checkpoint-60960/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-59055] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-62865\n",
      "Configuration saved in bert_base/checkpoint-62865/config.json\n",
      "Model weights saved in bert_base/checkpoint-62865/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-60960] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-64770\n",
      "Configuration saved in bert_base/checkpoint-64770/config.json\n",
      "Model weights saved in bert_base/checkpoint-64770/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-22860] due to args.save_total_limit\n",
      "Deleting older checkpoint [bert_base/checkpoint-62865] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-66675\n",
      "Configuration saved in bert_base/checkpoint-66675/config.json\n",
      "Model weights saved in bert_base/checkpoint-66675/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-64770] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-68580\n",
      "Configuration saved in bert_base/checkpoint-68580/config.json\n",
      "Model weights saved in bert_base/checkpoint-68580/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-70485\n",
      "Configuration saved in bert_base/checkpoint-70485/config.json\n",
      "Model weights saved in bert_base/checkpoint-70485/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-68580] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-72390\n",
      "Configuration saved in bert_base/checkpoint-72390/config.json\n",
      "Model weights saved in bert_base/checkpoint-72390/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-70485] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-74295\n",
      "Configuration saved in bert_base/checkpoint-74295/config.json\n",
      "Model weights saved in bert_base/checkpoint-74295/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-72390] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-76200\n",
      "Configuration saved in bert_base/checkpoint-76200/config.json\n",
      "Model weights saved in bert_base/checkpoint-76200/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-74295] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-78105\n",
      "Configuration saved in bert_base/checkpoint-78105/config.json\n",
      "Model weights saved in bert_base/checkpoint-78105/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-76200] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-80010\n",
      "Configuration saved in bert_base/checkpoint-80010/config.json\n",
      "Model weights saved in bert_base/checkpoint-80010/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-78105] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-81915\n",
      "Configuration saved in bert_base/checkpoint-81915/config.json\n",
      "Model weights saved in bert_base/checkpoint-81915/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-80010] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-83820\n",
      "Configuration saved in bert_base/checkpoint-83820/config.json\n",
      "Model weights saved in bert_base/checkpoint-83820/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-81915] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-85725\n",
      "Configuration saved in bert_base/checkpoint-85725/config.json\n",
      "Model weights saved in bert_base/checkpoint-85725/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-83820] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-87630\n",
      "Configuration saved in bert_base/checkpoint-87630/config.json\n",
      "Model weights saved in bert_base/checkpoint-87630/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-85725] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-89535\n",
      "Configuration saved in bert_base/checkpoint-89535/config.json\n",
      "Model weights saved in bert_base/checkpoint-89535/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-87630] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-91440\n",
      "Configuration saved in bert_base/checkpoint-91440/config.json\n",
      "Model weights saved in bert_base/checkpoint-91440/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-89535] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-93345\n",
      "Configuration saved in bert_base/checkpoint-93345/config.json\n",
      "Model weights saved in bert_base/checkpoint-93345/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-91440] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 424\n",
      "  Batch size = 2\n",
      "Saving model checkpoint to bert_base/checkpoint-95250\n",
      "Configuration saved in bert_base/checkpoint-95250/config.json\n",
      "Model weights saved in bert_base/checkpoint-95250/pytorch_model.bin\n",
      "Deleting older checkpoint [bert_base/checkpoint-93345] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from bert_base/checkpoint-66675 (score: 2.2585062980651855).\n",
      "Saving model checkpoint to bert_base\n",
      "Configuration saved in bert_base/config.json\n",
      "Model weights saved in bert_base/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n",
    "\n",
    "\n",
    "def tokenize_function(examples):\n",
    "    return tokenizer(examples[\"text\"], truncation=True)\n",
    "\n",
    "\n",
    "device = \"cpu\"\n",
    "\n",
    "data = set()\n",
    "\n",
    "for w in w_dict:\n",
    "    data.add(w_dict[w][\"name\"])\n",
    "for ap in ap_dict:\n",
    "    data.add(ap_dict[ap][\"name\"])\n",
    "for tech in technique_dict:\n",
    "    data.add(technique_dict[tech][\"name\"])\n",
    "for tac in tactic_dict:\n",
    "    data.add(tactic_dict[tac][\"name\"])\n",
    "for mitigation in w_mitigation:\n",
    "    data.add(mitigation[\"metadata\"][\"Description\"])\n",
    "for detection in w_detection:\n",
    "    data.add(detection[\"metadata\"][\"Description\"])\n",
    "for mitigation in ap_mitigation:\n",
    "    data.add(mitigation[\"metadata\"])\n",
    "for detection in ap_detection:\n",
    "    data.add(detection[\"metadata\"])\n",
    "for mitigation in technique_mitigation:\n",
    "    data.add(mitigation[\"name\"])\n",
    "for detection in technique_detection:\n",
    "    data.add(detection[\"name\"])\n",
    "\n",
    "dataset = {\"text\": list(data)}\n",
    "dataset = Dataset.from_dict(dataset)\n",
    "dataset = dataset.train_test_split(test_size=0.1)\n",
    "\n",
    "tokenized_datasets = dataset.map(\n",
    "    tokenize_function, batched=True, remove_columns=[\"text\"]\n",
    ")\n",
    "\n",
    "model = AutoModelForMaskedLM.from_pretrained(\"bert-base-uncased\")\n",
    "\n",
    "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer)\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    \"bert_base\",\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    save_strategy=\"epoch\",\n",
    "    per_device_train_batch_size=2,\n",
    "    per_device_eval_batch_size=2,\n",
    "    num_train_epochs=50.0,\n",
    "    save_total_limit=1,\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model=\"eval_loss\",\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_datasets[\"train\"],\n",
    "    eval_dataset=tokenized_datasets[\"test\"],\n",
    "    data_collator=data_collator,\n",
    ")\n",
    "\n",
    "trainer.train()\n",
    "trainer.save_model()"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "98b0a9b7b4eaaa670588a142fd0a9b87eaafe866f1db4228be72b4211d12040f"
  },
  "kernelspec": {
   "display_name": "Python 3.7.10 64-bit ('base': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "015e2f37014a4fba9461366d3a533291": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "05480ee0c2d2400085b5ea360133a7e1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "1be6d5b68b364ba49840e96b8a1b1e9c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_05480ee0c2d2400085b5ea360133a7e1",
       "placeholder": "​",
       "style": "IPY_MODEL_4e8c797865974a478945037614e5ac4b",
       "value": "100%"
      }
     },
     "392677d93be34925a88fc899700ad581": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4a99086f87304988a6efec1ca9513c24": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_9a94126b238a488f8c8e0be42bf14746",
       "max": 4.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_015e2f37014a4fba9461366d3a533291",
       "value": 4.0
      }
     },
     "4e8c797865974a478945037614e5ac4b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "50c323f1deac4323abd89a10e6bde8da": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "5c59629045264e45854ec87997e8087c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "655bea80e91b4fd1b6cb3c785f1f5d27": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_f0b3acd29140454c98a4e8bd942bc4b3",
       "placeholder": "​",
       "style": "IPY_MODEL_5c59629045264e45854ec87997e8087c",
       "value": " 4/4 [00:00&lt;00:00,  9.35ba/s]"
      }
     },
     "74429fe2fafa4bca85b1dc9d458e5ea1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_1be6d5b68b364ba49840e96b8a1b1e9c",
        "IPY_MODEL_b88a5483b4174fbda4d788ea8dace9bf",
        "IPY_MODEL_c54b07c42b2949a08d41a68624ea8a78"
       ],
       "layout": "IPY_MODEL_dc7a23f4aca54f749f793f16a187f42b"
      }
     },
     "9a94126b238a488f8c8e0be42bf14746": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b88a5483b4174fbda4d788ea8dace9bf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_dd416da61f924af480616c4458f1a69e",
       "max": 1.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_50c323f1deac4323abd89a10e6bde8da",
       "value": 1.0
      }
     },
     "baa1d9d33d1f4c6889c233fa8e46a2be": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_f847b9ad11d9488a9803d947c0500fa0",
        "IPY_MODEL_4a99086f87304988a6efec1ca9513c24",
        "IPY_MODEL_655bea80e91b4fd1b6cb3c785f1f5d27"
       ],
       "layout": "IPY_MODEL_bae75d4f45794b8684994ba93935cc07"
      }
     },
     "bae75d4f45794b8684994ba93935cc07": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c54b07c42b2949a08d41a68624ea8a78": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_fc52adf41bc0434681e9996a486265b7",
       "placeholder": "​",
       "style": "IPY_MODEL_da983d7d4be7421bb335a8da634aa08f",
       "value": " 1/1 [00:00&lt;00:00, 17.19ba/s]"
      }
     },
     "da983d7d4be7421bb335a8da634aa08f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "dc7a23f4aca54f749f793f16a187f42b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "dd416da61f924af480616c4458f1a69e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "deb78f79a61e43d5bfbe6ac41943fa6f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "f0b3acd29140454c98a4e8bd942bc4b3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f847b9ad11d9488a9803d947c0500fa0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_392677d93be34925a88fc899700ad581",
       "placeholder": "​",
       "style": "IPY_MODEL_deb78f79a61e43d5bfbe6ac41943fa6f",
       "value": "100%"
      }
     },
     "fc52adf41bc0434681e9996a486265b7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}