赛事提交的代码.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "本方法提供了两种运行思路，一种是原始数据处理，另一种使用我们已经处理好的进行训练\n",
    "原始数据处理起来相对较慢，可以先尝试使用我们已经处理好的数据进行验证。\n",
    "\n",
    "\n",
    "如果需要在B榜数据进行验证的话。需要在标题为 处理A榜数据哪里改成B榜数据的信息\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 库所需要的版本要求\n",
    "sklearn->1.1.2\n",
    "pandas->1.1.4\n",
    "lightgbm->3.3.2 ->这个比较关键"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 导入所需要的库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from glob import glob\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import auc\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "\n",
    "# svm分类器\n",
    "from sklearn.model_selection import cross_val_score, GridSearchCV\n",
    "from sklearn.metrics import accuracy_score\n",
    "import sklearn.metrics\n",
    "# 机器学习算法相关包\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from sklearn.model_selection import train_test_split\n",
    "model_undersample = RandomUnderSampler(random_state=42)\n",
    "from lightgbm import LGBMClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据部分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 28389/28389 [00:00<00:00, 36117.06it/s]\n"
     ]
    }
   ],
   "source": [
    "# 以下这部分是赛事提供的没做任何修改\n",
    "data_path = 'Train'  # 存放数据的路径\n",
    "pkl_files = glob(data_path + '/*.pkl')\n",
    "len(pkl_files)\n",
    "\n",
    "ind_pkl_files = []  # 存放标签为0的文件\n",
    "ood_pkl_files = []  # 存放标签为1的文件\n",
    "for each_path in tqdm(pkl_files):\n",
    "    # 打开pkl文件\n",
    "    pic = open(each_path, 'rb')\n",
    "    this_pkl_file = pickle.load(pic)\n",
    "    if this_pkl_file[1]['label'] == '00':\n",
    "        ind_pkl_files.append(each_path)\n",
    "    else:\n",
    "        ood_pkl_files.append(each_path)\n",
    "\n",
    "random.seed(0)\n",
    "random.shuffle(ind_pkl_files)\n",
    "random.shuffle(ood_pkl_files)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 下面这部分开始处理异常数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4654/4654 [00:05<00:00, 902.05it/s]\n"
     ]
    }
   ],
   "source": [
    "data_df_norm = []  # 将结果保存为df\n",
    "# 所需的特征 也是赛事提供的，这里需要所有的特征\n",
    "columns = ['volt','current','soc','max_single_volt','min_single_volt','max_temp','min_temp','timestamp'] \n",
    "index = 1\n",
    "for each_pkl in tqdm(ood_pkl_files):\n",
    "    pic = open(each_pkl, 'rb')\n",
    "    item = pickle.load(pic)\n",
    "    data = item[0]\n",
    "    temp_df = pd.DataFrame(data, columns=columns)\n",
    "    data_label = item[1]['label']                # 打上标签\n",
    "    data_mileage = float(item[1]['mileage'])     # 考虑mileage特征\n",
    "    temp_df['pkl'] = each_pkl                    # 对应的文件名字\n",
    "    temp_df['mileage'] = data_mileage \n",
    "    temp_df['label'] = data_label\n",
    "    temp_df['custom_index'] = index              # 每条数据有一个索引 用来标识每条数据\n",
    "\n",
    "    index += 1\n",
    "    data_df_norm.append(temp_df)\n",
    "\n",
    "data_df_norm = pd.concat(data_df_norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4654/4654 [00:00<00:00, 20198.37it/s]\n"
     ]
    }
   ],
   "source": [
    "data_df_group = data_df_norm.groupby('custom_index')   # 按照每条数据对应的索引进行分组，方便后面对每条数据进行特征处理\n",
    "group_data = []\n",
    "for idx, data_ in tqdm(data_df_group):\n",
    "    group_data.append(data_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def helper(x):\n",
    "    '''\n",
    "    x: 特征 \n",
    "    比如 volt，会有序列的信息 比如第一时刻到256时刻的volt数据\n",
    "    对这序列数据取到 最大值、最小值、均值、求和、median、方差\n",
    "    这些特征用来反映这条时序数据的信息\n",
    "    \n",
    "    \n",
    "    后续会对每一个特征都会进行这样的操作，另外还会引入新的特征序列，比如相对volt时序信息等等\n",
    "    '''\n",
    "    if isinstance(x, type(np.array([1,2,]))):\n",
    "        x = pd.DataFrame(x)\n",
    "    max_num = x.max()\n",
    "    min_num = x.min()\n",
    "    mean_num = x.mean()\n",
    "    sum_num = x.sum()\n",
    "    median_num = float(x.median())\n",
    "    std_num = float(x.std())\n",
    "\n",
    "    res = []\n",
    "    for i in [max_num, min_num, mean_num, sum_num, median_num, std_num]:\n",
    "        res.append(float(i))\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_features(m, ):\n",
    "    '''\n",
    "    m: 表示每一条数据  序列长度*['volt','current','soc','max_single_volt','min_single_volt','max_temp','min_temp','timestamp', 'mileage']\n",
    "    \n",
    "    以下的思路是先对每一个已有的序列特征根据helper提取到他的一些标量化特征来表示这条序列特征的信息\n",
    "    '''\n",
    "    \n",
    "    # 所有的特征信息\n",
    "    features = []  \n",
    "    # 遍历已有的特征，注意这里比赛事给的特征多了一个 mileage\n",
    "    for i in columns:\n",
    "        l = helper(m[i])  # 对每一个序列特征提取他的标量特征\n",
    "        features += l     # 把它加到features\n",
    "        \n",
    "    # 以下是相对volt信息，也就是上一时刻的volt减去前一时刻的volt，构造的新的特征信息，这里做了一定的扩大用来放大特征\n",
    "    # 这里基本上就是假设有1-256 时刻 就是。2-256的特征减去 1-255\n",
    "    one_last_volt = np.array(m['volt'][1:])*10\n",
    "    zero_last_volt = np.array(m['volt'][0:-1])*10\n",
    "    new_volt = one_last_volt - zero_last_volt\n",
    "    new_volt_rel = np.hstack([new_volt, np.array(m['volt'].iloc[-1]) - np.array(m['volt'].iloc[0])])\n",
    "    # 相对信息的最大最小 mean/sum/median\n",
    "    features+=helper(new_volt_rel)  # 对序列特征进行标量化处理\n",
    "    \n",
    "    # 同理这里是max_single_volt的相对特征，做了缩放\n",
    "    one_last_volt = np.array(m['max_single_volt'][1:])*1000\n",
    "    zero_last_volt = np.array(m['max_single_volt'][0:-1])*1000\n",
    "    new_volt = one_last_volt - zero_last_volt\n",
    "    new_max_single_volt = np.hstack([new_volt,np.array(m['max_single_volt'].iloc[-1]) - np.array(m['max_single_volt'].iloc[0])])\n",
    "    features+=helper(new_max_single_volt)\n",
    "    \n",
    "    # 同理这里是min_single_volt的相对特征，做了缩放\n",
    "    one_last_volt = np.array(m['min_single_volt'][1:])*1000\n",
    "    zero_last_volt = np.array(m['min_single_volt'][0:-1])*1000\n",
    "    new_volt = one_last_volt - zero_last_volt\n",
    "    new_min_single_volt = np.hstack([new_volt,np.array(m['min_single_volt'].iloc[-1]) - np.array(m['min_single_volt'].iloc[0])])\n",
    "    features+=helper(new_min_single_volt)\n",
    "    \n",
    "    # 这里对温度做了缩放信息\n",
    "    new_max_temp_vari = m['max_temp'] - 200\n",
    "    new_min_temp_vari = m['min_temp'] - 160\n",
    "    features += helper(new_max_temp_vari)\n",
    "    features += helper(new_min_temp_vari)\n",
    "    \n",
    "    # 这里是特征变化的信息 也就是最后一个时刻和第一个时刻的每一个特征的变化特征。\n",
    "    last_one_diff = list(m[columns].iloc[-1] - m[columns].iloc[0])\n",
    "    features += last_one_diff\n",
    "    \n",
    "    # 最大volt和最小volt的差特征\n",
    "    max_min_volt_change = m['max_single_volt'] - m['min_single_volt']\n",
    "    features += helper(max_min_volt_change)\n",
    "    # 最大温度和最小温度的差特征\n",
    "    max_min_temp_change = m['max_temp'] - m['min_temp']\n",
    "    features += helper(max_min_temp_change)\n",
    "    \n",
    "    return features\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 以下是利用上面两个函数对异常时序数据进行标量化特征处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "核心想法描述：\n",
    "由于给定的数据集正负样本极其不均衡，正样本数据极其稀少（相对负样本来说）。所以首先需要扩充数据来提升预测效果。\n",
    "给定的数据是1-256条序列数据。为了扩充数据，需要对数据做切分，将一条数据变成多条数据，同时保证数据的时序性。\n",
    "我们按照将一条数据划分成4条数据。\n",
    "\n",
    "由于是时序数据，所以取间断时刻构成的数据也算时序数据，而且出现异常的状态肯定是一段时间的所以间断时刻不会改变它是异常的本质\n",
    "比如现在有12条数据。我们按照每3个时刻为间断构建一条新的数据：\n",
    "具体如下： \\\\ \n",
    "\\\\ \\[1,2,3,4,5,6,7,8,9,10,11,12\\]\\\\\n",
    "\\[0, 4, 8\\]  \\\\\n",
    "\\[1, 5, 9\\]  \\\\ \n",
    "\\[2, 6, 10\\] \\\\\n",
    "\\[3, 7, 11\\] \\\\\n",
    "这样一条时序数据被划分成了4条新的时序数据，\n",
    "同样的1-256会被划分成4条新的数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4654it [02:22, 32.66it/s]\n"
     ]
    }
   ],
   "source": [
    "new_df = []   # 存储结果\n",
    "columns = ['volt','current','soc','max_single_volt','min_single_volt','max_temp',\n",
    "           'min_temp','timestamp', 'mileage',]   # 这里就是需要处理的序列特征多了一个 mileage\n",
    "for idx, m in tqdm(enumerate(group_data)):  # 之前按照每条数据对应的索引进行了分组，现在遍历每一条数据\n",
    "    for i in range(4):                      # 这里是我们核心的想法，参考上面的描述\n",
    "        df = m.iloc[i::4]\n",
    "        features = get_features(df)         # 特征处理\n",
    "        temp_df = pd.DataFrame(features).T  # 将其保存\n",
    "        temp_df['label'] = 1                # 打标签\n",
    "        new_df.append(temp_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 以上就是异常数据的处理结果了存储到了new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>96</th>\n",
       "      <th>97</th>\n",
       "      <th>98</th>\n",
       "      <th>99</th>\n",
       "      <th>100</th>\n",
       "      <th>101</th>\n",
       "      <th>102</th>\n",
       "      <th>103</th>\n",
       "      <th>104</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>152.3</td>\n",
       "      <td>149.3</td>\n",
       "      <td>151.529687</td>\n",
       "      <td>9697.9</td>\n",
       "      <td>151.55</td>\n",
       "      <td>0.572014</td>\n",
       "      <td>-0.8</td>\n",
       "      <td>-24.2</td>\n",
       "      <td>-23.159375</td>\n",
       "      <td>-1482.2</td>\n",
       "      <td>...</td>\n",
       "      <td>0.41</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.000684</td>\n",
       "      <td>30.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>29.90625</td>\n",
       "      <td>1914.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>0.75</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 106 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0      1           2       3       4         5    6     7          8  \\\n",
       "0  152.3  149.3  151.529687  9697.9  151.55  0.572014 -0.8 -24.2 -23.159375   \n",
       "\n",
       "        9  ...    96     97        98    99   100       101     102   103  \\\n",
       "0 -1482.2  ...  0.41  0.006  0.000684  30.0  24.0  29.90625  1914.0  30.0   \n",
       "\n",
       "    104  label  \n",
       "0  0.75      1  \n",
       "\n",
       "[1 rows x 106 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 以下是正常数据的处理过程和异常处理一样"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 23735/23735 [00:25<00:00, 928.24it/s]\n"
     ]
    }
   ],
   "source": [
    "data_df_norm = []\n",
    "columns = ['volt','current','soc','max_single_volt','min_single_volt','max_temp','min_temp','timestamp', ]\n",
    "index = 1\n",
    "for each_pkl in tqdm(ind_pkl_files):\n",
    "    pic = open(each_pkl, 'rb')\n",
    "    item = pickle.load(pic)\n",
    "    data = item[0]\n",
    "    temp_df = pd.DataFrame(data, columns=columns)\n",
    "    data_label = item[1]['label']\n",
    "    data_mileage = float(item[1]['mileage'])\n",
    "    temp_df['pkl'] = each_pkl\n",
    "    temp_df['mileage'] = data_mileage\n",
    "    temp_df['label'] = data_label\n",
    "    temp_df['custom_index'] = index\n",
    "    \n",
    "    index += 1\n",
    "    data_df_norm.append(temp_df)\n",
    "\n",
    "data_df_norm = pd.concat(data_df_norm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 分组"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 23735/23735 [00:01<00:00, 15032.76it/s]\n"
     ]
    }
   ],
   "source": [
    "data_df_group = data_df_norm.groupby('custom_index')\n",
    "group_data = []\n",
    "for idx, data_ in tqdm(data_df_group):\n",
    "    group_data.append(data_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 划分数据时序数据标量化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23735it [12:10, 32.49it/s]\n"
     ]
    }
   ],
   "source": [
    "columns = ['volt','current','soc','max_single_volt','min_single_volt','max_temp','min_temp',\n",
    "           'timestamp', 'mileage']\n",
    "norm_feature_df = []\n",
    "for idx, m in tqdm(enumerate(group_data)):\n",
    "    for i in range(4):\n",
    "        df = m.iloc[i::4]\n",
    "        features = get_features(df)\n",
    "        temp_df = pd.DataFrame(features).T\n",
    "        temp_df['label'] = 0\n",
    "        norm_feature_df.append(temp_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 以下是处理A榜数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6234"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = 'Test_A'  # 存放数据的路径\n",
    "pkl_files = glob(data_path + '/*.pkl')\n",
    "len(pkl_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6234/6234 [00:05<00:00, 1143.62it/s]\n"
     ]
    }
   ],
   "source": [
    "test_data = []\n",
    "columns = ['volt','current','soc','max_single_volt','min_single_volt','max_temp','min_temp','timestamp', ]\n",
    "index = 1\n",
    "for each_pkl in tqdm(pkl_files):\n",
    "    pic = open(each_pkl, 'rb')\n",
    "    item = pickle.load(pic)\n",
    "    data = item[0]\n",
    "    temp_df = pd.DataFrame(data, columns=columns)\n",
    "    data_mileage = float(item[1]['mileage'])\n",
    "    temp_df['pkl'] = each_pkl\n",
    "    temp_df['mileage'] = data_mileage\n",
    "    temp_df['custom_index'] = index\n",
    "    \n",
    "    index += 1\n",
    "    test_data.append(temp_df)\n",
    "\n",
    "test_data = pd.concat(test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 同样的数据会被划分成4组，所以需要数据分别存放，将最后的4组数据分别得到的预测结果取均值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6234/6234 [00:00<00:00, 24745.25it/s]\n",
      "6234it [03:11, 32.55it/s]\n"
     ]
    }
   ],
   "source": [
    "columns = ['volt','current','soc','max_single_volt','min_single_volt','max_temp','min_temp','timestamp', \n",
    "           'mileage',]\n",
    "#           'current_soc', 'max_temp_min_temp', 'volt_current', 'max_vol_max_temp', 'current_mileage']\n",
    "data_df_group = test_data.groupby('custom_index')\n",
    "\n",
    "group_data = []\n",
    "test_df_odd = []\n",
    "test_df_even = []\n",
    "for idx, data_ in tqdm(data_df_group):\n",
    "    group_data.append(data_)\n",
    "\n",
    "test_df1 = []\n",
    "test_df2 = []\n",
    "test_df3 = []\n",
    "test_df4 = []\n",
    "for idx, m in tqdm(enumerate(group_data)):\n",
    "    \n",
    "    for i in range(4):\n",
    "        \n",
    "        df = m.iloc[i::4]\n",
    "        features = get_features(df)\n",
    "        temp_df = pd.DataFrame(features).T\n",
    "        name = np.array(df['pkl'])\n",
    "        temp_df['pkl'] = name[0]\n",
    "#         temp_df['label'] = 0\n",
    "        if i==0:\n",
    "            test_df1.append(temp_df)\n",
    "        elif i==1:\n",
    "            test_df2.append(temp_df)\n",
    "        elif i==2:\n",
    "            test_df3.append(temp_df)\n",
    "        else:\n",
    "            test_df4.append(temp_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# 4组数据分别存放\n",
    "test_df1 = pd.concat(test_df1)\n",
    "test_df2 = pd.concat(test_df2)\n",
    "test_df3 = pd.concat(test_df3)\n",
    "test_df4 = pd.concat(test_df4)\n",
    "# 存放到一个list\n",
    "test_df = [test_df1, test_df2, test_df3, test_df4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 整个上述部分就是原始数据处理的部分了，下面是直接读取我们已经处理过的训练数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(\"lgb_feature_10-1-90.pkl\", 'rb') as f:\n",
    "    data_new = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 可以判断 原始数据处理后的数据是否和我们提供的已处理数据是否一致 data在下面生成。\n",
    "(data_new != data).sum().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 以下是训练训练部分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将正常和异常合并\n",
    "data = new_df + norm_feature_df\n",
    "data = pd.concat(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>96</th>\n",
       "      <th>97</th>\n",
       "      <th>98</th>\n",
       "      <th>99</th>\n",
       "      <th>100</th>\n",
       "      <th>101</th>\n",
       "      <th>102</th>\n",
       "      <th>103</th>\n",
       "      <th>104</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>152.3</td>\n",
       "      <td>149.3</td>\n",
       "      <td>151.529687</td>\n",
       "      <td>9697.9</td>\n",
       "      <td>151.55</td>\n",
       "      <td>0.572014</td>\n",
       "      <td>-0.8</td>\n",
       "      <td>-24.2</td>\n",
       "      <td>-23.159375</td>\n",
       "      <td>-1482.2</td>\n",
       "      <td>...</td>\n",
       "      <td>0.410</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.000684</td>\n",
       "      <td>30.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>29.90625</td>\n",
       "      <td>1914.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>152.3</td>\n",
       "      <td>149.5</td>\n",
       "      <td>151.546875</td>\n",
       "      <td>9699.0</td>\n",
       "      <td>151.55</td>\n",
       "      <td>0.540420</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>-24.2</td>\n",
       "      <td>-23.228125</td>\n",
       "      <td>-1486.6</td>\n",
       "      <td>...</td>\n",
       "      <td>0.406</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.000648</td>\n",
       "      <td>30.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>30.00000</td>\n",
       "      <td>1920.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>152.3</td>\n",
       "      <td>149.5</td>\n",
       "      <td>151.560937</td>\n",
       "      <td>9699.9</td>\n",
       "      <td>151.55</td>\n",
       "      <td>0.517202</td>\n",
       "      <td>-4.5</td>\n",
       "      <td>-24.2</td>\n",
       "      <td>-23.443750</td>\n",
       "      <td>-1500.4</td>\n",
       "      <td>...</td>\n",
       "      <td>0.412</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.000687</td>\n",
       "      <td>30.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>29.90625</td>\n",
       "      <td>1914.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>152.3</td>\n",
       "      <td>149.5</td>\n",
       "      <td>151.562500</td>\n",
       "      <td>9700.0</td>\n",
       "      <td>151.55</td>\n",
       "      <td>0.515321</td>\n",
       "      <td>-4.5</td>\n",
       "      <td>-24.2</td>\n",
       "      <td>-23.453125</td>\n",
       "      <td>-1501.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.407</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.000698</td>\n",
       "      <td>36.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>30.09375</td>\n",
       "      <td>1926.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>1.305893</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>161.7</td>\n",
       "      <td>158.3</td>\n",
       "      <td>159.853125</td>\n",
       "      <td>10230.6</td>\n",
       "      <td>159.60</td>\n",
       "      <td>0.943140</td>\n",
       "      <td>-46.0</td>\n",
       "      <td>-46.0</td>\n",
       "      <td>-46.000000</td>\n",
       "      <td>-2944.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.708</td>\n",
       "      <td>0.011</td>\n",
       "      <td>0.000432</td>\n",
       "      <td>24.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>19.12500</td>\n",
       "      <td>1224.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.360387</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>158.2</td>\n",
       "      <td>157.6</td>\n",
       "      <td>157.889063</td>\n",
       "      <td>10104.9</td>\n",
       "      <td>157.80</td>\n",
       "      <td>0.221909</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>-6.1</td>\n",
       "      <td>-6.078125</td>\n",
       "      <td>-389.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.683</td>\n",
       "      <td>0.011</td>\n",
       "      <td>0.000592</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.00000</td>\n",
       "      <td>384.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>153.0</td>\n",
       "      <td>152.8</td>\n",
       "      <td>152.903125</td>\n",
       "      <td>9785.8</td>\n",
       "      <td>152.90</td>\n",
       "      <td>0.077600</td>\n",
       "      <td>-4.3</td>\n",
       "      <td>-4.4</td>\n",
       "      <td>-4.398438</td>\n",
       "      <td>-281.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.199</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.000567</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.00000</td>\n",
       "      <td>768.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>153.0</td>\n",
       "      <td>152.8</td>\n",
       "      <td>152.904687</td>\n",
       "      <td>9785.9</td>\n",
       "      <td>152.90</td>\n",
       "      <td>0.078538</td>\n",
       "      <td>-4.4</td>\n",
       "      <td>-4.4</td>\n",
       "      <td>-4.400000</td>\n",
       "      <td>-281.6</td>\n",
       "      <td>...</td>\n",
       "      <td>0.200</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.000604</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.00000</td>\n",
       "      <td>768.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>153.0</td>\n",
       "      <td>152.8</td>\n",
       "      <td>152.903125</td>\n",
       "      <td>9785.8</td>\n",
       "      <td>152.90</td>\n",
       "      <td>0.077600</td>\n",
       "      <td>-4.4</td>\n",
       "      <td>-4.4</td>\n",
       "      <td>-4.400000</td>\n",
       "      <td>-281.6</td>\n",
       "      <td>...</td>\n",
       "      <td>0.203</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.000579</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.00000</td>\n",
       "      <td>768.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>153.0</td>\n",
       "      <td>152.8</td>\n",
       "      <td>152.906250</td>\n",
       "      <td>9786.0</td>\n",
       "      <td>152.90</td>\n",
       "      <td>0.077408</td>\n",
       "      <td>-4.4</td>\n",
       "      <td>-4.4</td>\n",
       "      <td>-4.400000</td>\n",
       "      <td>-281.6</td>\n",
       "      <td>...</td>\n",
       "      <td>0.199</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.000669</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.00000</td>\n",
       "      <td>768.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>113556 rows × 106 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        0      1           2        3       4         5     6     7  \\\n",
       "0   152.3  149.3  151.529687   9697.9  151.55  0.572014  -0.8 -24.2   \n",
       "0   152.3  149.5  151.546875   9699.0  151.55  0.540420  -3.0 -24.2   \n",
       "0   152.3  149.5  151.560937   9699.9  151.55  0.517202  -4.5 -24.2   \n",
       "0   152.3  149.5  151.562500   9700.0  151.55  0.515321  -4.5 -24.2   \n",
       "0   161.7  158.3  159.853125  10230.6  159.60  0.943140 -46.0 -46.0   \n",
       "..    ...    ...         ...      ...     ...       ...   ...   ...   \n",
       "0   158.2  157.6  157.889063  10104.9  157.80  0.221909  -6.0  -6.1   \n",
       "0   153.0  152.8  152.903125   9785.8  152.90  0.077600  -4.3  -4.4   \n",
       "0   153.0  152.8  152.904687   9785.9  152.90  0.078538  -4.4  -4.4   \n",
       "0   153.0  152.8  152.903125   9785.8  152.90  0.077600  -4.4  -4.4   \n",
       "0   153.0  152.8  152.906250   9786.0  152.90  0.077408  -4.4  -4.4   \n",
       "\n",
       "            8       9  ...     96     97        98    99   100       101  \\\n",
       "0  -23.159375 -1482.2  ...  0.410  0.006  0.000684  30.0  24.0  29.90625   \n",
       "0  -23.228125 -1486.6  ...  0.406  0.006  0.000648  30.0  30.0  30.00000   \n",
       "0  -23.443750 -1500.4  ...  0.412  0.006  0.000687  30.0  24.0  29.90625   \n",
       "0  -23.453125 -1501.0  ...  0.407  0.006  0.000698  36.0  24.0  30.09375   \n",
       "0  -46.000000 -2944.0  ...  0.708  0.011  0.000432  24.0  18.0  19.12500   \n",
       "..        ...     ...  ...    ...    ...       ...   ...   ...       ...   \n",
       "0   -6.078125  -389.0  ...  0.683  0.011  0.000592   6.0   6.0   6.00000   \n",
       "0   -4.398438  -281.5  ...  0.199  0.003  0.000567  12.0  12.0  12.00000   \n",
       "0   -4.400000  -281.6  ...  0.200  0.003  0.000604  12.0  12.0  12.00000   \n",
       "0   -4.400000  -281.6  ...  0.203  0.003  0.000579  12.0  12.0  12.00000   \n",
       "0   -4.400000  -281.6  ...  0.199  0.003  0.000669  12.0  12.0  12.00000   \n",
       "\n",
       "       102   103       104  label  \n",
       "0   1914.0  30.0  0.750000      1  \n",
       "0   1920.0  30.0  0.000000      1  \n",
       "0   1914.0  30.0  0.750000      1  \n",
       "0   1926.0  30.0  1.305893      1  \n",
       "0   1224.0  18.0  2.360387      1  \n",
       "..     ...   ...       ...    ...  \n",
       "0    384.0   6.0  0.000000      0  \n",
       "0    768.0  12.0  0.000000      0  \n",
       "0    768.0  12.0  0.000000      0  \n",
       "0    768.0  12.0  0.000000      0  \n",
       "0    768.0  12.0  0.000000      0  \n",
       "\n",
       "[113556 rows x 106 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 总共会构建出来105个特征，需要对其进行筛选，\n",
    "features_index = [int(i) for i in ['1',  \n",
    "                                   '6', '8','11','10', \n",
    "                                   '12', '16', '15', '17', \n",
    "                                   '23', \n",
    "                                   '30', '34','32', \n",
    "                                   '36', '41', '40', \n",
    "                                   '47',   \n",
    "                                   '48', '51' ,'52', '53',  \n",
    "                                   '55', '54',   \n",
    "                                   '61',   \n",
    "                                   '72', '73', '74', \n",
    "                                   '79', '80', '78', \n",
    "                                   '89', '91', \n",
    "                                   '100', '103']] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 标签数据 划分\n",
    "target = 'label'\n",
    "x = data.iloc[:, features_index]\n",
    "y = LabelEncoder().fit_transform(data[target])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "欠采样前 113556 113556\n",
      "欠采样后 37232 37232\n"
     ]
    }
   ],
   "source": [
    "# 使用欠采样方法\n",
    "print('欠采样前', len(x), len(y))\n",
    "x, y = model_undersample.fit_resample(x, y)\n",
    "print('欠采样后', len(x), len(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 划分训练测试数据\n",
    "train_x, test_x, train_y, test_y = train_test_split(x, y, test_size=0.20, random_state=42, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.5, subsample=1.0 will be ignored. Current value: bagging_fraction=0.5\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10\n",
      "model5：\n",
      " LGBMClassifier(bagging_fraction=0.5, bagging_freq=10, boosting_type='rf',\n",
      "               feature_fraction=0.5, is_unbalance=False, learning_rate=0.0001,\n",
      "               max_depth=6, num_leaves=64, random_state=2, reg_alpha=0.01,\n",
      "               reg_lambda=0.02) \n",
      "auc: 0.9987597833542866\n"
     ]
    }
   ],
   "source": [
    "# 基础模型选择为LGBMClassifier\n",
    "model5 = LGBMClassifier(boosting_type='rf', learning_rate=0.0001,max_depth=6,num_leaves=2**6,\n",
    "                       reg_lambda=0.02,random_state=2, is_unbalance=False,\n",
    "                       bagging_freq=10,\n",
    "                       bagging_fraction= 0.5,\n",
    "                       feature_fraction=0.5,\n",
    "                       reg_alpha=0.01)\n",
    "model5.fit(x, y)\n",
    "pred5_score = model5.predict_proba(test_x)[:, 1]\n",
    "fpr, tpr, thresholds = metrics.roc_curve(test_y, pred5_score, pos_label=1)\n",
    "print('model5：\\n', model5, '\\nauc:', metrics.auc(fpr, tpr))\n",
    "\n",
    "# 正确的预测结果auc: 0.9987597833542866，如果不对可能导致A榜结果预测不准确，\n",
    "# 处理办法可以参考我们提供已经处理好的数据，或者将LGBMClassifier包升级 满足版本要求\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算A榜的预测结果，将4组预测结果取平均值或者最后一组\n",
    "def compute_test():\n",
    "    score = 0\n",
    "    for i in test_df:\n",
    "        score = model5.predict_proba(i.iloc[:, features_index])[:, 1]\n",
    "    \n",
    "    # score /= 4\n",
    "    submision_df = pd.DataFrame()\n",
    "    submision_df['file_name'] = test_df1['pkl'].apply(lambda x:x[7:])\n",
    "    submision_df['score'] = score\n",
    "    submision_df.to_csv('submision.csv', index=False)\n",
    "    return submision_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "submision_df = compute_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7096.pkl</td>\n",
       "      <td>0.154747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6421.pkl</td>\n",
       "      <td>0.179055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10799.pkl</td>\n",
       "      <td>0.122404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3655.pkl</td>\n",
       "      <td>0.122300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10947.pkl</td>\n",
       "      <td>0.121699</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4739.pkl</td>\n",
       "      <td>0.137857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8268.pkl</td>\n",
       "      <td>0.125588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7863.pkl</td>\n",
       "      <td>0.120588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6809.pkl</td>\n",
       "      <td>0.127282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8242.pkl</td>\n",
       "      <td>0.119230</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   file_name     score\n",
       "0   7096.pkl  0.154747\n",
       "0   6421.pkl  0.179055\n",
       "0  10799.pkl  0.122404\n",
       "0   3655.pkl  0.122300\n",
       "0  10947.pkl  0.121699\n",
       "0   4739.pkl  0.137857\n",
       "0   8268.pkl  0.125588\n",
       "0   7863.pkl  0.120588\n",
       "0   6809.pkl  0.127282\n",
       "0   8242.pkl  0.119230"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submision_df[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}