实体类型分类.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "实体类型分类.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V3WXHPM4VD5d",
        "outputId": "5f455db2-2efd-49d6-8833-4812f6370fa1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-05-29 05:45:43--  https://www.dropbox.com/s/x0vjwjeuaawo6yr/entitytyping.zip?dl=0\n",
            "Resolving www.dropbox.com (www.dropbox.com)... 162.125.1.18, 2620:100:6016:18::a27d:112\n",
            "Connecting to www.dropbox.com (www.dropbox.com)|162.125.1.18|:443... connected.\n",
            "HTTP request sent, awaiting response... 301 Moved Permanently\n",
            "Location: /s/raw/x0vjwjeuaawo6yr/entitytyping.zip [following]\n",
            "--2022-05-29 05:45:43--  https://www.dropbox.com/s/raw/x0vjwjeuaawo6yr/entitytyping.zip\n",
            "Reusing existing connection to www.dropbox.com:443.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://ucf4fac902d7053de2b87abfed8d.dl.dropboxusercontent.com/cd/0/inline/BmLBF44L1I8yAEjPirhj_6J-bjcSmm5XSLh_9Y42LeLVImPnEbzxvamu2xV9kVb81G-NPZb88OmlkuNGWjRS3x5kWJ0chwJ3LV82KozqDGC2nggMZ0oeZS-1h2gvyuv5vvVtNJC2ajngwY5mGqvQxpUcL1WdYvRGo1NX7fU42TNDgQ/file# [following]\n",
            "--2022-05-29 05:45:43--  https://ucf4fac902d7053de2b87abfed8d.dl.dropboxusercontent.com/cd/0/inline/BmLBF44L1I8yAEjPirhj_6J-bjcSmm5XSLh_9Y42LeLVImPnEbzxvamu2xV9kVb81G-NPZb88OmlkuNGWjRS3x5kWJ0chwJ3LV82KozqDGC2nggMZ0oeZS-1h2gvyuv5vvVtNJC2ajngwY5mGqvQxpUcL1WdYvRGo1NX7fU42TNDgQ/file\n",
            "Resolving ucf4fac902d7053de2b87abfed8d.dl.dropboxusercontent.com (ucf4fac902d7053de2b87abfed8d.dl.dropboxusercontent.com)... 162.125.1.15, 2620:100:6022:15::a27d:420f\n",
            "Connecting to ucf4fac902d7053de2b87abfed8d.dl.dropboxusercontent.com (ucf4fac902d7053de2b87abfed8d.dl.dropboxusercontent.com)|162.125.1.15|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: /cd/0/inline2/BmLgCVxMmQ7vMT9wFISG8OqKNU7tWs_N4dcBvbQpbFlBY96eAAsHRRd-vAc1v7q_-Cdl374_5otUzwkx3AIj3ZfCqIcmafxrFPdacgpEFB4tVKW2ncpsIplAXJTVBTYQm3BQBWgB_YBWamynOhwCvLm7KLO6A6kDRzdpJjdJamZ4xoIDC8x1uugm9w2lSRQwM1h4P2CuZ7u8FTakFEUeRKhLNHGZzqTSHEF_wGzsryYsengLd4g7vY1PKNxfv7BVg38t7BVGZlX-ZEAFgZIr-wQXcrs4avVt1Zl9FiERAp9v9rqyTv-Lw7OVRGQ88rtxafjlk_JwHvx5er2I8aYR2BcqwEoxGk2mOxqT2dSL_FdJq_XUjvaafzx6mIWdI1iYliAasX3aSakram8FGQfe6-QDhLegDd5dC04foZ8gHQZ08w/file [following]\n",
            "--2022-05-29 05:45:44--  https://ucf4fac902d7053de2b87abfed8d.dl.dropboxusercontent.com/cd/0/inline2/BmLgCVxMmQ7vMT9wFISG8OqKNU7tWs_N4dcBvbQpbFlBY96eAAsHRRd-vAc1v7q_-Cdl374_5otUzwkx3AIj3ZfCqIcmafxrFPdacgpEFB4tVKW2ncpsIplAXJTVBTYQm3BQBWgB_YBWamynOhwCvLm7KLO6A6kDRzdpJjdJamZ4xoIDC8x1uugm9w2lSRQwM1h4P2CuZ7u8FTakFEUeRKhLNHGZzqTSHEF_wGzsryYsengLd4g7vY1PKNxfv7BVg38t7BVGZlX-ZEAFgZIr-wQXcrs4avVt1Zl9FiERAp9v9rqyTv-Lw7OVRGQ88rtxafjlk_JwHvx5er2I8aYR2BcqwEoxGk2mOxqT2dSL_FdJq_XUjvaafzx6mIWdI1iYliAasX3aSakram8FGQfe6-QDhLegDd5dC04foZ8gHQZ08w/file\n",
            "Reusing existing connection to ucf4fac902d7053de2b87abfed8d.dl.dropboxusercontent.com:443.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 14339073 (14M) [application/zip]\n",
            "Saving to: ‘entitytyping.zip’\n",
            "\n",
            "entitytyping.zip    100%[===================>]  13.67M  --.-KB/s    in 0.1s    \n",
            "\n",
            "2022-05-29 05:45:44 (141 MB/s) - ‘entitytyping.zip’ saved [14339073/14339073]\n",
            "\n",
            "Archive:  entitytyping.zip\n",
            "replace template.txt? [y]es, [n]o, [A]ll, [N]one, [r]ename: y\n",
            "  inflating: template.txt            \n",
            "replace Few-Nerd/supervised/dev.txt? [y]es, [n]o, [A]ll, [N]one, [r]ename: y\n",
            "  inflating: Few-Nerd/supervised/dev.txt  \n",
            "replace Few-Nerd/supervised/test.txt? [y]es, [n]o, [A]ll, [N]one, [r]ename: y\n",
            "  inflating: Few-Nerd/supervised/test.txt  \n",
            "replace Few-Nerd/supervised/train.txt? [y]es, [n]o, [A]ll, [N]one, [r]ename: y\n",
            "  inflating: Few-Nerd/supervised/train.txt  \n",
            "replace verbalizer.txt? [y]es, [n]o, [A]ll, [N]one, [r]ename: ty\n",
            "error:  invalid response [ty]\n",
            "replace verbalizer.txt? [y]es, [n]o, [A]ll, [N]one, [r]ename: y\n",
            "  inflating: verbalizer.txt          \n"
          ]
        }
      ],
      "source": [
        "!wget https://www.dropbox.com/s/x0vjwjeuaawo6yr/entitytyping.zip?dl=0  -O entitytyping.zip\n",
        "!unzip entitytyping.zip"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install openprompt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GZ9d_0FuVevr",
        "outputId": "1860c767-c640-4600-9b9a-7fe6abfe5537"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting openprompt\n",
            "  Downloading openprompt-1.0.0-py3-none-any.whl (146 kB)\n",
            "\u001b[K     |████████████████████████████████| 146 kB 8.5 MB/s \n",
            "\u001b[?25hCollecting yacs\n",
            "  Downloading yacs-0.1.8-py3-none-any.whl (14 kB)\n",
            "Collecting datasets\n",
            "  Downloading datasets-2.2.2-py3-none-any.whl (346 kB)\n",
            "\u001b[K     |████████████████████████████████| 346 kB 47.5 MB/s \n",
            "\u001b[?25hCollecting rouge==1.0.0\n",
            "  Downloading rouge-1.0.0-py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (from openprompt) (3.2.5)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from openprompt) (0.3.5.1)\n",
            "Collecting transformers>=4.10.0\n",
            "  Downloading transformers-4.19.2-py3-none-any.whl (4.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.2 MB 45.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from openprompt) (1.4.1)\n",
            "Collecting tensorboardX\n",
            "  Downloading tensorboardX-2.5-py2.py3-none-any.whl (125 kB)\n",
            "\u001b[K     |████████████████████████████████| 125 kB 52.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.62.2 in /usr/local/lib/python3.7/dist-packages (from openprompt) (4.64.0)\n",
            "Requirement already satisfied: pyarrow in /usr/local/lib/python3.7/dist-packages (from openprompt) (6.0.1)\n",
            "Collecting sentencepiece==0.1.96\n",
            "  Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 44.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from rouge==1.0.0->openprompt) (1.15.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers>=4.10.0->openprompt) (21.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers>=4.10.0->openprompt) (3.7.0)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.7.0-py3-none-any.whl (86 kB)\n",
            "\u001b[K     |████████████████████████████████| 86 kB 5.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers>=4.10.0->openprompt) (2019.12.20)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 38.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers>=4.10.0->openprompt) (1.21.6)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers>=4.10.0->openprompt) (4.11.3)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 41.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers>=4.10.0->openprompt) (2.23.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers>=4.10.0->openprompt) (4.2.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers>=4.10.0->openprompt) (3.0.9)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[K     |████████████████████████████████| 212 kB 32.2 MB/s \n",
            "\u001b[?25hCollecting fsspec[http]>=2021.05.0\n",
            "  Downloading fsspec-2022.5.0-py3-none-any.whl (140 kB)\n",
            "\u001b[K     |████████████████████████████████| 140 kB 52.1 MB/s \n",
            "\u001b[?25hCollecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets->openprompt) (1.3.5)\n",
            "Collecting dill\n",
            "  Downloading dill-0.3.4-py2.py3-none-any.whl (86 kB)\n",
            "\u001b[K     |████████████████████████████████| 86 kB 6.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets->openprompt) (0.70.12.2)\n",
            "Collecting aiohttp\n",
            "  Downloading aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 42.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers>=4.10.0->openprompt) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers>=4.10.0->openprompt) (2022.5.18.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers>=4.10.0->openprompt) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers>=4.10.0->openprompt) (2.10)\n",
            "Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1\n",
            "  Downloading urllib3-1.25.11-py2.py3-none-any.whl (127 kB)\n",
            "\u001b[K     |████████████████████████████████| 127 kB 54.0 MB/s \n",
            "\u001b[?25hCollecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n",
            "\u001b[K     |████████████████████████████████| 271 kB 54.2 MB/s \n",
            "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.3.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (144 kB)\n",
            "\u001b[K     |████████████████████████████████| 144 kB 55.7 MB/s \n",
            "\u001b[?25hCollecting asynctest==0.13.0\n",
            "  Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-6.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (94 kB)\n",
            "\u001b[K     |████████████████████████████████| 94 kB 3.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets->openprompt) (2.0.12)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets->openprompt) (21.4.0)\n",
            "Collecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers>=4.10.0->openprompt) (3.8.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets->openprompt) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets->openprompt) (2022.1)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorboardX->openprompt) (3.17.3)\n",
            "Installing collected packages: multidict, frozenlist, yarl, urllib3, asynctest, async-timeout, aiosignal, pyyaml, fsspec, dill, aiohttp, xxhash, tokenizers, responses, huggingface-hub, yacs, transformers, tensorboardX, sentencepiece, rouge, datasets, openprompt\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "  Attempting uninstall: dill\n",
            "    Found existing installation: dill 0.3.5.1\n",
            "    Uninstalling dill-0.3.5.1:\n",
            "      Successfully uninstalled dill-0.3.5.1\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "Successfully installed aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.2 asynctest-0.13.0 datasets-2.2.2 dill-0.3.4 frozenlist-1.3.0 fsspec-2022.5.0 huggingface-hub-0.7.0 multidict-6.0.2 openprompt-1.0.0 pyyaml-6.0 responses-0.18.0 rouge-1.0.0 sentencepiece-0.1.96 tensorboardX-2.5 tokenizers-0.12.1 transformers-4.19.2 urllib3-1.25.11 xxhash-3.0.0 yacs-0.1.8 yarl-1.7.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "\n",
        "import torch\n",
        "from openprompt.data_utils.typing_dataset import PROCESSORS\n",
        "import argparse\n",
        "\n",
        "from openprompt.prompts import ManualTemplate, ManualVerbalizer\n",
        "from tqdm import tqdm\n",
        "\n",
        "parser = argparse.ArgumentParser(\"\")\n",
        "\n",
        "parser.add_argument(\"--shot\", type=int, default=5)\n",
        "parser.add_argument(\"--seed\", type=int, default=144)\n",
        "\n",
        "parser.add_argument(\"--plm_eval_mode\", action=\"store_true\")\n",
        "parser.add_argument(\"--model\", type=str, default='roberta')\n",
        "parser.add_argument(\"--model_name_or_path\", default='roberta-large')\n",
        "parser.add_argument(\"--result_file\", type=str, default=\"sfs_scripts/results_fewshot_manual_kpt.txt\")\n",
        "parser.add_argument(\"--openprompt_path\", type=str, default=\"datasets\")\n",
        "\n",
        "parser.add_argument(\"--verbalizer\", default='kpt',type=str)\n",
        "parser.add_argument(\"--calibration\", default='--calibration',action=\"store_true\")\n",
        "parser.add_argument(\"--nocut\", action=\"store_true\")\n",
        "parser.add_argument(\"--filter\", default=\"tfidf_filter\", type=str)\n",
        "parser.add_argument(\"--template_id\", default=1,type=int)\n",
        "parser.add_argument(\"--max_token_split\", default=-1, type=int)\n",
        "parser.add_argument(\"--dataset\",default='agnews',type=str)\n",
        "parser.add_argument(\"--write_filter_record\", action=\"store_true\")\n",
        "args = parser.parse_args(args=[])\n",
        "\n",
        "# base_path = \"Few-Nerd\"\n",
        "#\n",
        "dataset_name = \"FewNERD\"\n",
        "# dataset_path = os.path.join(base_path, dataset_name)\n",
        "processor = PROCESSORS[dataset_name.lower()]()\n",
        "dataset_path='Few-Nerd'\n",
        "dataset={}\n",
        "\n",
        "train_dataset = processor.get_train_examples(dataset_path)"
      ],
      "metadata": {
        "id": "3XO7XHQiVkfI"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Zero-shot"
      ],
      "metadata": {
        "id": "OPBXkG2FVKlB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dev_dataset = processor.get_dev_examples(dataset_path)\n",
        "test_dataset = processor.get_test_examples(dataset_path)\n",
        "class_labels=processor.get_labels()\n",
        "scriptsbase = \"\"\n",
        "scriptformat = \"txt\"\n",
        "cutoff=0.5 if (not args.nocut) else 0.0\n",
        "max_seq_l = 128\n",
        "batch_s = 10\n",
        "\n",
        "dataset['train']=train_dataset\n",
        "dataset['test']=test_dataset\n",
        "dataset['support']=dev_dataset"
      ],
      "metadata": {
        "id": "vJhs8B3XVo_W"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 加载预训练模型"
      ],
      "metadata": {
        "id": "6FXOlYYpYq-J"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from openprompt.utils.reproduciblity import set_seed\n",
        "set_seed()\n",
        "\n",
        "from openprompt.plms import load_plm\n",
        "plm, tokenizer, model_config, WrapperClass = load_plm(args.model, args.model_name_or_path)\n",
        "\n",
        "mytemplate = ManualTemplate(tokenizer=tokenizer).from_file(f\"template.txt\", choice=args.template_id)\n",
        "\n",
        "myverbalizer = ManualVerbalizer(tokenizer, classes=class_labels).from_file(f\"verbalizer.{scriptformat}\")\n"
      ],
      "metadata": {
        "id": "dJ0waM8UVuW4"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 实体类型分类问题"
      ],
      "metadata": {
        "id": "dplI-pdPYwuk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from openprompt import PromptModel, PromptDataLoader,PromptForClassification\n",
        "\n",
        "use_cuda = True\n",
        "prompt_model = PromptForClassification(plm=plm,template=mytemplate,\n",
        "                           verbalizer=myverbalizer,freeze_plm=False, plm_eval_mode=args.plm_eval_mode)\n",
        "\n",
        "if use_cuda:\n",
        "    prompt_model=  prompt_model.cuda()\n"
      ],
      "metadata": {
        "id": "NwMNPUC8Vw_G"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "myrecord= \"\"\n",
        "if args.write_filter_record:\n",
        "    record_prefix = \"=\"*20+\"\\n\"\n",
        "    record_prefix += f\"dataset {args.dataset}\\t\"\n",
        "    record_prefix += f\"temp {args.template_id}\\t\"\n",
        "    record_prefix += f\"seed {args.seed}\\t\"\n",
        "    record_prefix += f\"cali {args.calibration}\\t\"\n",
        "    record_prefix += f\"filt {args.filter}\\t\"\n",
        "    record_prefix += \"\\n\"\n",
        "    myrecord = record_prefix +myrecord\n",
        "    with open(\"../sfs_scripts/filter_record_file.txt\",'a')  as fout_rec:\n",
        "        fout_rec.write(myrecord)\n",
        "    exit()\n",
        "\n",
        "\n",
        "test_dataloader = PromptDataLoader(dataset=dataset[\"test\"], template=mytemplate, tokenizer=tokenizer,\n",
        "    tokenizer_wrapper_class=WrapperClass, max_seq_length=max_seq_l, decoder_max_length=3,\n",
        "    batch_size=batch_s,shuffle=False, teacher_forcing=False, predict_eos_token=False,\n",
        "    truncate_method=\"tail\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lnsH9EpRV24j",
        "outputId": "016e9530-fd91-4fa4-cc70-b722a4906b59"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "tokenizing: 96902it [02:30, 642.48it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "allpreds = []\n",
        "alllabels = []\n",
        "pbar = tqdm(test_dataloader)\n",
        "for step, inputs in enumerate(pbar):\n",
        "    if use_cuda:\n",
        "        inputs = inputs.cuda()\n",
        "    logits = prompt_model(inputs)#bs*labels_num\n",
        "    labels = inputs['label']\n",
        "    alllabels.extend(labels.cpu().tolist())\n",
        "    allpreds.extend(torch.argmax(logits, dim=-1).cpu().tolist())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pY8pUtXvV5Ih",
        "outputId": "0605184f-8b25-45f9-80b6-286f49a05499"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 48451/48451 [31:51<00:00, 25.34it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "acc = sum([int(i==j) for i,j in zip(allpreds, alllabels)])/len(allpreds)\n",
        "recall=sum([int(i==j) for i,j in zip(allpreds, alllabels)])/len(alllabels)\n",
        "micro_f1=2*acc*recall/(acc+recall)"
      ],
      "metadata": {
        "id": "SNjFtbhqV8ec"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(acc,recall,micro_f1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3klcRQyWg6Rs",
        "outputId": "3c95ced5-9a43-4626-ca11-71c9282ceffa"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.1915233947699738 0.1915233947699738 0.1915233947699738\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# few-shot"
      ],
      "metadata": {
        "id": "jpCa5JrUhJ4y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataloader = PromptDataLoader(dataset=dataset[\"train\"], template=mytemplate, tokenizer=tokenizer,\n",
        "    tokenizer_wrapper_class=WrapperClass, max_seq_length=max_seq_l, decoder_max_length=3,\n",
        "    batch_size=batch_s,shuffle=True, teacher_forcing=False, predict_eos_token=False,\n",
        "    truncate_method=\"tail\")"
      ],
      "metadata": {
        "id": "CJSg56xAj_Fi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "\n",
        "validation_dataloader = PromptDataLoader(dataset=dataset[\"support\"], template=mytemplate, tokenizer=tokenizer,\n",
        "    tokenizer_wrapper_class=WrapperClass, max_seq_length=max_seq_l, decoder_max_length=3,\n",
        "    batch_size=batch_s,shuffle=False, teacher_forcing=False, predict_eos_token=False,\n",
        "    truncate_method=\"tail\")\n",
        "\n",
        "\n",
        "\n",
        "test_dataloader = PromptDataLoader(dataset=dataset[\"test\"], template=mytemplate, tokenizer=tokenizer,\n",
        "    tokenizer_wrapper_class=WrapperClass, max_seq_length=max_seq_l, decoder_max_length=3,\n",
        "    batch_size=batch_s,shuffle=False, teacher_forcing=False, predict_eos_token=False,\n",
        "    truncate_method=\"tail\")\n",
        "#['In the early 1930s the band moved to the Grill Room of the Taft Hotel in New York ; the band was renamed ``\n",
        "# George Hall and His Hotel Taft Orchestra `` .', '. In this sentence,', ' Grill Room', ' is a', '<mask>', '.']\n",
        "from random import random\n",
        "\n",
        "def evaluate(prompt_model, dataloader, desc):\n",
        "    prompt_model.eval()\n",
        "    allpreds = []\n",
        "    alllabels = []\n",
        "    pbar = tqdm(dataloader, desc=desc)\n",
        "    for step, inputs in enumerate(pbar):\n",
        "        if use_cuda:\n",
        "            inputs = inputs.cuda()\n",
        "        logits = prompt_model(inputs)\n",
        "        labels = inputs['label']\n",
        "        alllabels.extend(labels.cpu().tolist())\n",
        "        allpreds.extend(torch.argmax(logits, dim=-1).cpu().tolist())\n",
        "    acc = sum([int(i==j) for i,j in zip(allpreds, alllabels)])/len(allpreds)\n",
        "    return acc\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "'''\n",
        "\n",
        "assert processor.get_num_labels() == 66\n",
        "assert processor.get_labels() == [\n",
        "    \"person-actor\", \"person-director\", \"person-artist/author\", \"person-athlete\", \"person-politician\", \"person-scholar\", \"person-soldier\", \"person-other\",\n",
        "    \"organization-showorganization\", \"organization-religion\", \"organization-company\", \"organization-sportsteam\", \"organization-education\", \"organization-government/governmentagency\", \"organization-media/newspaper\", \"organization-politicalparty\", \"organization-sportsleague\", \"organization-other\",\n",
        "    \"location-GPE\", \"location-road/railway/highway/transit\", \"location-bodiesofwater\", \"location-park\", \"location-mountain\", \"location-island\", \"location-other\",\n",
        "    \"product-software\", \"product-food\", \"product-game\", \"product-ship\", \"product-train\", \"product-airplane\", \"product-car\", \"product-weapon\", \"product-other\",\n",
        "    \"building-theater\", \"building-sportsfacility\", \"building-airport\", \"building-hospital\", \"building-library\", \"building-hotel\", \"building-restaurant\", \"building-other\",\n",
        "    \"event-sportsevent\", \"event-attack/battle/war/militaryconflict\", \"event-disaster\", \"event-election\", \"event-protest\", \"event-other\",\n",
        "    \"art-music\", \"art-writtenart\", \"art-film\", \"art-painting\", \"art-broadcastprogram\", \"art-other\",\n",
        "    \"other-biologything\", \"other-chemicalthing\", \"other-livingthing\", \"other-astronomything\", \"other-god\", \"other-law\", \"other-award\", \"other-disease\", \"other-medical\", \"other-language\", \"other-currency\", \"other-educationaldegree\",\n",
        "]\n",
        "\n",
        "\n",
        "assert processor.get_num_labels() == 66\n",
        "assert processor.get_labels() == [\n",
        "    \"person-actor\", \"person-director\", \"person-artist/author\", \"person-athlete\", \"person-politician\", \"person-scholar\", \"person-soldier\", \"person-other\",\n",
        "    \"organization-showorganization\", \"organization-religion\", \"organization-company\", \"organization-sportsteam\", \"organization-education\", \"organization-government/governmentagency\", \"organization-media/newspaper\", \"organization-politicalparty\", \"organization-sportsleague\", \"organization-other\",\n",
        "    \"location-GPE\", \"location-road/railway/highway/transit\", \"location-bodiesofwater\", \"location-park\", \"location-mountain\", \"location-island\", \"location-other\",\n",
        "    \"product-software\", \"product-food\", \"product-game\", \"product-ship\", \"product-train\", \"product-airplane\", \"product-car\", \"product-weapon\", \"product-other\",\n",
        "    \"building-theater\", \"building-sportsfacility\", \"building-airport\", \"building-hospital\", \"building-library\", \"building-hotel\", \"building-restaurant\", \"building-other\",\n",
        "    \"event-sportsevent\", \"event-attack/battle/war/militaryconflict\", \"event-disaster\", \"event-election\", \"event-protest\", \"event-other\",\n",
        "    \"art-music\", \"art-writtenart\", \"art-film\", \"art-painting\", \"art-broadcastprogram\", \"art-other\",\n",
        "    \"other-biologything\", \"other-chemicalthing\", \"other-livingthing\", \"other-astronomything\", \"other-god\", \"other-law\", \"other-award\", \"other-disease\", \"other-medical\", \"other-language\", \"other-currency\", \"other-educationaldegree\",\n",
        "]\n",
        "assert dev_dataset[0].text_a == \"The final stage in the development of the Skyfox was the production of a model with tricycle landing gear to better cater for the pilot training market .\"\n",
        "assert dev_dataset[0].meta[\"entity\"] == \"Skyfox\"\n",
        "assert dev_dataset[0].label == 30\n",
        "'''\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "hu44_MD7hMxS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import random"
      ],
      "metadata": {
        "id": "y_yIHyNFk_x_"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "this_run_unicode=str(random.randint(0, 1e10))\n",
        "from transformers import  AdamW, get_linear_schedule_with_warmup\n",
        "loss_func = torch.nn.CrossEntropyLoss()\n",
        "optimizer1 = AdamW(prompt_model.parameters(), lr=3e-5)\n",
        "tot_loss = 0\n",
        "log_loss = 0\n",
        "best_val_acc = 0\n",
        "max_epochs =5\n",
        "for epoch in range(max_epochs):\n",
        "    tot_loss = 0\n",
        "    prompt_model.train()\n",
        "    for step, inputs in enumerate(train_dataloader):\n",
        "        if use_cuda:\n",
        "            inputs = inputs.cuda()\n",
        "        logits = prompt_model(inputs)#bs*class_nums\n",
        "        labels = inputs['label']#bs*1\n",
        "        loss = loss_func(logits, labels)\n",
        "        loss.requires_grad_(True)\n",
        "        loss.backward()\n",
        "        torch.nn.utils.clip_grad_norm_(prompt_model.parameters(), 1.0)#梯度裁剪\n",
        "        tot_loss += loss.item()\n",
        "        optimizer1.step()\n",
        "        optimizer1.zero_grad()\n",
        "        if step!=0 and step%200==0:\n",
        "          print('step={},average loss={}'.format(step,tot_loss/step))\n",
        "\n",
        "    val_acc = evaluate(prompt_model, validation_dataloader, desc=\"Valid\")\n",
        "    if val_acc>=best_val_acc:\n",
        "        torch.save(prompt_model.state_dict(),f\"{this_run_unicode}.ckpt\")\n",
        "        best_val_acc = val_acc\n",
        "    print(\"Epoch {}, val_acc {}\".format(epoch, val_acc), flush=True)\n",
        "\n",
        "prompt_model.load_state_dict(torch.load(f\"{this_run_unicode}.ckpt\"))\n",
        "prompt_model = prompt_model.cuda()\n",
        "test_acc = evaluate(prompt_model, test_dataloader, desc=\"Test\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "mwVxELWBk889",
        "outputId": "b44d541d-cbb0-4c75-92eb-9ff5fc3327c5"
      },
      "execution_count": 30,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "step=200,average loss=3.3661122047901153\n",
            "step=400,average loss=3.4840178912878037\n",
            "step=600,average loss=3.3994029732545217\n",
            "step=800,average loss=3.387121944874525\n",
            "step=1000,average loss=3.4056431230306625\n",
            "step=1200,average loss=3.4003532695770264\n",
            "step=1400,average loss=3.408081677726337\n",
            "step=1600,average loss=3.4201036425679923\n",
            "step=1800,average loss=3.377756201426188\n",
            "step=2000,average loss=3.364802974998951\n",
            "step=2200,average loss=3.3527766661210494\n",
            "step=2400,average loss=3.3474711544811724\n",
            "step=2600,average loss=3.35005833534094\n",
            "step=2800,average loss=3.324537343766008\n",
            "step=3000,average loss=3.3304373717308042\n",
            "step=3200,average loss=3.3687009147927167\n",
            "step=3400,average loss=3.4157961751783597\n",
            "step=3600,average loss=3.454931202828884\n",
            "step=3800,average loss=3.4877211680224067\n",
            "step=4000,average loss=3.5181861983835696\n",
            "step=4200,average loss=3.5420341714507058\n",
            "step=4400,average loss=3.5632577844912356\n",
            "step=4600,average loss=3.58414865942105\n",
            "step=4800,average loss=3.602685950671633\n",
            "step=5000,average loss=3.622217558979988\n",
            "step=5200,average loss=3.636581980242179\n",
            "step=5400,average loss=3.6484200226156798\n",
            "step=5600,average loss=3.663029300974948\n",
            "step=5800,average loss=3.676095934756871\n",
            "step=6000,average loss=3.6882821044723193\n",
            "step=6200,average loss=3.7007355508688957\n",
            "step=6400,average loss=3.7090383843146264\n",
            "step=6600,average loss=3.717627835472425\n",
            "step=6800,average loss=3.727707470918403\n",
            "step=7000,average loss=3.73544861137867\n",
            "step=7200,average loss=3.7424074100620217\n",
            "step=7400,average loss=3.7517336802063763\n",
            "step=7600,average loss=3.7613023130046694\n",
            "step=7800,average loss=3.768577638543569\n",
            "step=8000,average loss=3.7743781725019216\n",
            "step=8200,average loss=3.7797972522276204\n",
            "step=8400,average loss=3.785868287015529\n",
            "step=8600,average loss=3.790675272400989\n",
            "step=8800,average loss=3.7957350150428035\n",
            "step=9000,average loss=3.800427019106017\n",
            "step=9200,average loss=3.8043516959185184\n",
            "step=9400,average loss=3.808678377879427\n",
            "step=9600,average loss=3.8139025408402087\n",
            "step=9800,average loss=3.818754714228669\n",
            "step=10000,average loss=3.822791948568821\n",
            "step=10200,average loss=3.8266977195529375\n",
            "step=10400,average loss=3.8302798280005272\n",
            "step=10600,average loss=3.8338110100660683\n",
            "step=10800,average loss=3.8353643365149144\n",
            "step=11000,average loss=3.839446520079266\n",
            "step=11200,average loss=3.8432779458058732\n",
            "step=11400,average loss=3.8461312130354997\n",
            "step=11600,average loss=3.8488413576524834\n",
            "step=11800,average loss=3.85080518123457\n",
            "step=12000,average loss=3.852743185609579\n",
            "step=12200,average loss=3.8556015170109075\n",
            "step=12400,average loss=3.85913502705674\n",
            "step=12600,average loss=3.8618452218884514\n",
            "step=12800,average loss=3.8636139032710344\n",
            "step=13000,average loss=3.865688522898234\n",
            "step=13200,average loss=3.8679461431774227\n",
            "step=13400,average loss=3.8693323159128874\n",
            "step=13600,average loss=3.8716023969212\n",
            "step=13800,average loss=3.873262090812559\n",
            "step=14000,average loss=3.8752593872462002\n",
            "step=14200,average loss=3.8766123394982914\n",
            "step=14400,average loss=3.8790114055822293\n",
            "step=14600,average loss=3.881073576807976\n",
            "step=14800,average loss=3.883105477811517\n",
            "step=15000,average loss=3.88376929778258\n",
            "step=15200,average loss=3.8861842772129336\n",
            "step=15400,average loss=3.887882768790443\n",
            "step=15600,average loss=3.8890002213609525\n",
            "step=15800,average loss=3.889521362411825\n",
            "step=16000,average loss=3.8905102027878167\n",
            "step=16200,average loss=3.89094565058196\n",
            "step=16400,average loss=3.8920293255622793\n",
            "step=16600,average loss=3.8937446339949067\n",
            "step=16800,average loss=3.894463654153404\n",
            "step=17000,average loss=3.895019955950625\n",
            "step=17200,average loss=3.89611102741125\n",
            "step=17400,average loss=3.897152198779172\n",
            "step=17600,average loss=3.8981350406936626\n",
            "step=17800,average loss=3.8991462696201347\n",
            "step=18000,average loss=3.900378417087926\n",
            "step=18200,average loss=3.9015238471018088\n",
            "step=18400,average loss=3.902368338788333\n",
            "step=18600,average loss=3.9029778073359562\n",
            "step=18800,average loss=3.903571160556154\n",
            "step=19000,average loss=3.9043663818898953\n",
            "step=19200,average loss=3.9049711657253403\n",
            "step=19400,average loss=3.9057968879667753\n",
            "step=19600,average loss=3.907096285911239\n",
            "step=19800,average loss=3.9075502842664718\n",
            "step=20000,average loss=3.908254309517145\n",
            "step=20200,average loss=3.908800003227621\n",
            "step=20400,average loss=3.9094629019089773\n",
            "step=20600,average loss=3.9098894919643126\n",
            "step=20800,average loss=3.910463006903346\n",
            "step=21000,average loss=3.9111237228087017\n",
            "step=21200,average loss=3.912008875389144\n",
            "step=21400,average loss=3.9123991958170294\n",
            "step=21600,average loss=3.9129537365116454\n",
            "step=21800,average loss=3.9135689750559832\n",
            "step=22000,average loss=3.9141212982101874\n",
            "step=22200,average loss=3.9148089538017907\n",
            "step=22400,average loss=3.915203525908291\n",
            "step=22600,average loss=3.9155259261774806\n",
            "step=22800,average loss=3.9154034489550087\n",
            "step=23000,average loss=3.915780613116596\n",
            "step=23200,average loss=3.915895126980954\n",
            "step=23400,average loss=3.9162169010058427\n",
            "step=23600,average loss=3.916752123605397\n",
            "step=23800,average loss=3.9167636909254457\n",
            "step=24000,average loss=3.916942340736588\n",
            "step=24200,average loss=3.9172727703407775\n",
            "step=24400,average loss=3.9175054844627617\n",
            "step=24600,average loss=3.9178197006723745\n",
            "step=24800,average loss=3.918476424827691\n",
            "step=25000,average loss=3.9187718772172926\n",
            "step=25200,average loss=3.919108870374778\n",
            "step=25400,average loss=3.91953648049531\n",
            "step=25600,average loss=3.919909213637002\n",
            "step=25800,average loss=3.920802273468454\n",
            "step=26000,average loss=3.9215871852773887\n",
            "step=26200,average loss=3.9223036900321944\n",
            "step=26400,average loss=3.9228307041480686\n",
            "step=26600,average loss=3.923389733577133\n",
            "step=26800,average loss=3.9234604044859087\n",
            "step=27000,average loss=3.923851356961109\n",
            "step=27200,average loss=3.9239077206613384\n",
            "step=27400,average loss=3.924038301380011\n",
            "step=27600,average loss=3.9247953587640887\n",
            "step=27800,average loss=3.924713989286114\n",
            "step=28000,average loss=3.924900944467102\n",
            "step=28200,average loss=3.9255639664205253\n",
            "step=28400,average loss=3.9259950891053172\n",
            "step=28600,average loss=3.9262965413115243\n",
            "step=28800,average loss=3.9271969144749974\n",
            "step=29000,average loss=3.9277925406776624\n",
            "step=29200,average loss=3.9279269261270353\n",
            "step=29400,average loss=3.928312971847398\n",
            "step=29600,average loss=3.9286452427143987\n",
            "step=29800,average loss=3.9288299882611972\n",
            "step=30000,average loss=3.928985901916027\n",
            "step=30200,average loss=3.9289088853266065\n",
            "step=30400,average loss=3.9290823489702063\n",
            "step=30600,average loss=3.9291103310016244\n",
            "step=30800,average loss=3.9292892507183086\n",
            "step=31000,average loss=3.9293089822530747\n",
            "step=31200,average loss=3.929653855099892\n",
            "step=31400,average loss=3.9297204095201126\n",
            "step=31600,average loss=3.9298013863571084\n",
            "step=31800,average loss=3.9296611877394922\n",
            "step=32000,average loss=3.930070475343615\n",
            "step=32200,average loss=3.9299642609254173\n",
            "step=32400,average loss=3.9306027165423205\n",
            "step=32600,average loss=3.9309138939241692\n",
            "step=32800,average loss=3.931211161297269\n",
            "step=33000,average loss=3.9313776122909605\n",
            "step=33200,average loss=3.9313215030568194\n",
            "step=33400,average loss=3.931486796322697\n",
            "step=33600,average loss=3.9314433949866463\n",
            "step=33800,average loss=3.931668610209544\n",
            "step=34000,average loss=3.9321209777488426\n",
            "step=34200,average loss=3.9323111718683914\n",
            "step=34400,average loss=3.932526964874462\n",
            "step=34600,average loss=3.932906522058338\n",
            "step=34800,average loss=3.9331013495654896\n",
            "step=35000,average loss=3.9330255558116094\n",
            "step=35200,average loss=3.933285998867994\n",
            "step=35400,average loss=3.9332291088568963\n",
            "step=35600,average loss=3.933343750590019\n",
            "step=35800,average loss=3.9334548861387724\n",
            "step=36000,average loss=3.933542718605863\n",
            "step=36200,average loss=3.933581761612418\n",
            "step=36400,average loss=3.9340459153449143\n",
            "step=36600,average loss=3.934128017044458\n",
            "step=36800,average loss=3.934275593696081\n",
            "step=37000,average loss=3.934667817892255\n",
            "step=37200,average loss=3.9347703113703316\n",
            "step=37400,average loss=3.9349557370521167\n",
            "step=37600,average loss=3.935182454164358\n",
            "step=37800,average loss=3.935501853388453\n",
            "step=38000,average loss=3.9356359040580298\n",
            "step=38200,average loss=3.935651258096021\n",
            "step=38400,average loss=3.9357997347569715\n",
            "step=38600,average loss=3.935975567278467\n",
            "step=38800,average loss=3.9361407116915763\n",
            "step=39000,average loss=3.936206367220634\n",
            "step=39200,average loss=3.9362563794516787\n",
            "step=39400,average loss=3.9363803273378895\n",
            "step=39600,average loss=3.9363973561531367\n",
            "step=39800,average loss=3.936559865705332\n",
            "step=40000,average loss=3.936737094387412\n",
            "step=40200,average loss=3.9367400998410895\n",
            "step=40400,average loss=3.9367866466629624\n",
            "step=40600,average loss=3.936682096346846\n",
            "step=40800,average loss=3.9370680240819267\n",
            "step=41000,average loss=3.937304999232292\n",
            "step=41200,average loss=3.9375507713550504\n",
            "step=41400,average loss=3.9376972407908832\n",
            "step=41600,average loss=3.937870569587327\n",
            "step=41800,average loss=3.937810180395414\n",
            "step=42000,average loss=3.9378028878172238\n",
            "step=42200,average loss=3.938022617776812\n",
            "step=42400,average loss=3.938057519779453\n",
            "step=42600,average loss=3.9380902211570965\n",
            "step=42800,average loss=3.9383817831851613\n",
            "step=43000,average loss=3.9387382559748585\n",
            "step=43200,average loss=3.939025265102585\n",
            "step=43400,average loss=3.939275945350871\n",
            "step=43600,average loss=3.9393774771334926\n",
            "step=43800,average loss=3.939496120598218\n",
            "step=44000,average loss=3.939591005653143\n",
            "step=44200,average loss=3.939574508966364\n",
            "step=44400,average loss=3.9393501139680542\n",
            "step=44600,average loss=3.9393108995372406\n",
            "step=44800,average loss=3.9395121255809706\n",
            "step=45000,average loss=3.9397415066904493\n",
            "step=45200,average loss=3.9399150355595403\n",
            "step=45400,average loss=3.9397657428885346\n",
            "step=45600,average loss=3.9400129097778547\n",
            "step=45800,average loss=3.9400797356814796\n",
            "step=46000,average loss=3.9401944956805397\n",
            "step=46200,average loss=3.9403543128518317\n",
            "step=46400,average loss=3.9403654861013435\n",
            "step=46600,average loss=3.940427818638572\n",
            "step=46800,average loss=3.940684054995704\n",
            "step=47000,average loss=3.9406269225490855\n",
            "step=47200,average loss=3.940792358378738\n",
            "step=47400,average loss=3.9409657913547025\n",
            "step=47600,average loss=3.9409816096635426\n",
            "step=47800,average loss=3.9410885340673656\n",
            "step=48000,average loss=3.9414362492586177\n",
            "step=48200,average loss=3.941497731028256\n",
            "step=48400,average loss=3.9418309596722776\n",
            "step=48600,average loss=3.941880426669317\n",
            "step=48800,average loss=3.941972985362909\n",
            "step=49000,average loss=3.942171032158696\n",
            "step=49200,average loss=3.9424152314977916\n",
            "step=49400,average loss=3.9423740961073865\n",
            "step=49600,average loss=3.9424896644945107\n",
            "step=49800,average loss=3.942195745124874\n",
            "step=50000,average loss=3.94240471732378\n",
            "step=50200,average loss=3.942727603966971\n",
            "step=50400,average loss=3.9429176939550845\n",
            "step=50600,average loss=3.9428982306021476\n",
            "step=50800,average loss=3.9428809730251007\n",
            "step=51000,average loss=3.9429785360051137\n",
            "step=51200,average loss=3.942905910902191\n",
            "step=51400,average loss=3.943114439381236\n",
            "step=51600,average loss=3.9432086533793185\n",
            "step=51800,average loss=3.943395327101803\n",
            "step=52000,average loss=3.9431772498007005\n",
            "step=52200,average loss=3.9433175821756494\n",
            "step=52400,average loss=3.9434109385982725\n",
            "step=52600,average loss=3.943522288157913\n",
            "step=52800,average loss=3.9436116684995817\n",
            "step=53000,average loss=3.943943240181455\n",
            "step=53200,average loss=3.9441430463266554\n",
            "step=53400,average loss=3.944162684507138\n",
            "step=53600,average loss=3.9439710957634806\n",
            "step=53800,average loss=3.9441790752140564\n",
            "step=54000,average loss=3.9439258663146584\n",
            "step=54200,average loss=3.9439832807884887\n",
            "step=54400,average loss=3.944066468444379\n",
            "step=54600,average loss=3.9440998934192972\n",
            "step=54800,average loss=3.9443181296617444\n",
            "step=55000,average loss=3.944357454670559\n",
            "step=55200,average loss=3.944388321916694\n",
            "step=55400,average loss=3.944593801384368\n",
            "step=55600,average loss=3.9446357402291228\n",
            "step=55800,average loss=3.9446402376975636\n",
            "step=56000,average loss=3.9447213966782604\n",
            "step=56200,average loss=3.944928936841649\n",
            "step=56400,average loss=3.9449845747723646\n",
            "step=56600,average loss=3.9450724436727094\n",
            "step=56800,average loss=3.9449130896342472\n",
            "step=57000,average loss=3.944764485156327\n",
            "step=57200,average loss=3.9449931460431404\n",
            "step=57400,average loss=3.944902029533835\n",
            "step=57600,average loss=3.944977582525462\n",
            "step=57800,average loss=3.9450737859695426\n",
            "step=58000,average loss=3.9452940573342916\n",
            "step=58200,average loss=3.9454268880664687\n",
            "step=58400,average loss=3.9454867665265523\n",
            "step=58600,average loss=3.945420222967965\n",
            "step=58800,average loss=3.945506448808576\n",
            "step=59000,average loss=3.945301759620844\n",
            "step=59200,average loss=3.9455341572918603\n",
            "step=59400,average loss=3.9455420171752924\n",
            "step=59600,average loss=3.945642006767276\n",
            "step=59800,average loss=3.9454713225504228\n",
            "step=60000,average loss=3.9454251932005087\n",
            "step=60200,average loss=3.945437604674073\n",
            "step=60400,average loss=3.945441605833587\n",
            "step=60600,average loss=3.9455237704908885\n",
            "step=60800,average loss=3.945838124706949\n",
            "step=61000,average loss=3.945923046207819\n",
            "step=61200,average loss=3.9458959329732104\n",
            "step=61400,average loss=3.945933279146589\n",
            "step=61600,average loss=3.9458430399658617\n",
            "step=61800,average loss=3.9458680711112746\n",
            "step=62000,average loss=3.945889057045983\n",
            "step=62200,average loss=3.946018253359764\n",
            "step=62400,average loss=3.9460488061931653\n",
            "step=62600,average loss=3.9463514838222498\n",
            "step=62800,average loss=3.9463182374569263\n",
            "step=63000,average loss=3.946446742540314\n",
            "step=63200,average loss=3.946425055173002\n",
            "step=63400,average loss=3.946338140597855\n",
            "step=63600,average loss=3.9462928916804447\n",
            "step=63800,average loss=3.9462202828973063\n",
            "step=64000,average loss=3.946333715727553\n",
            "step=64200,average loss=3.9458674052552642\n",
            "step=64400,average loss=3.9459759204154428\n",
            "step=64600,average loss=3.9461531095257483\n",
            "step=64800,average loss=3.946084640543034\n",
            "step=65000,average loss=3.946111997488829\n",
            "step=65200,average loss=3.9461602122268062\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-30-a1b2f7804fa4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip_grad_norm_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprompt_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1.0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m#梯度裁剪\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m         \u001b[0mtot_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m         \u001b[0moptimizer1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0moptimizer1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    }
  ]
}