2_matrix_generation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reduce_mem_usage(df):\n",
    "    \"\"\" iterate through all the columns of a dataframe and modify the data type\n",
    "        to reduce memory usage.        \n",
    "    \"\"\" \n",
    "    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "        \n",
    "        if col_type != object:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else:\n",
    "            df[col] = df[col].astype('category')\n",
    "\n",
    "    \n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31/31 [07:23<00:00, 14.04s/it]\n"
     ]
    }
   ],
   "source": [
    "data = []\n",
    "for i in tqdm(range(1, 32)):\n",
    "    data.append(reduce_mem_usage(pd.read_csv('../data/train_table_%02d.gz'%i)))\n",
    "data = pd.concat(data)\n",
    "data = pd.concat([data, reduce_mem_usage(pd.read_csv('../data/test_table.gz'))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = LabelEncoder()\n",
    "data['id'] = encoder.fit_transform(data['link'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[['link','id']].to_csv('map.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 先把每天的矩阵给复原 15583 * 720\n",
    "mat = np.zeros(shape=(32, 15584, 800)) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "idmap = pd.read_csv('map.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "idmap = dict(zip(idmap['link'], idmap['id']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/31 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "  3%|▎         | 1/31 [02:09<1:04:46, 129.54s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 2/31 [04:20<1:02:49, 129.99s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 10%|▉         | 3/31 [06:31<1:00:49, 130.33s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 13%|█▎        | 4/31 [08:44<59:01, 131.15s/it]  \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 16%|█▌        | 5/31 [10:58<57:13, 132.07s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 6/31 [13:06<54:24, 130.58s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 23%|██▎       | 7/31 [15:05<50:55, 127.33s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 26%|██▌       | 8/31 [19:13<1:02:39, 163.48s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 29%|██▉       | 9/31 [23:21<1:09:10, 188.67s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 32%|███▏      | 10/31 [27:27<1:12:08, 206.11s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 35%|███▌      | 11/31 [31:37<1:13:02, 219.10s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 39%|███▊      | 12/31 [35:47<1:12:20, 228.47s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 42%|████▏     | 13/31 [39:42<1:09:04, 230.27s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 45%|████▌     | 14/31 [43:26<1:04:46, 228.60s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 48%|████▊     | 15/31 [49:30<1:11:46, 269.17s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 52%|█████▏    | 16/31 [55:36<1:14:34, 298.31s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 55%|█████▍    | 17/31 [1:01:43<1:14:23, 318.86s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 58%|█████▊    | 18/31 [1:07:53<1:12:25, 334.26s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 61%|██████▏   | 19/31 [1:14:03<1:08:57, 344.80s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 65%|██████▍   | 20/31 [1:19:50<1:03:19, 345.42s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 68%|██████▊   | 21/31 [1:25:16<56:36, 339.64s/it]  \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 71%|███████   | 22/31 [1:33:11<57:03, 380.40s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 74%|███████▍  | 23/31 [1:41:05<54:27, 408.44s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 77%|███████▋  | 24/31 [1:48:54<49:45, 426.52s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████  | 25/31 [1:56:42<43:53, 438.99s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 84%|████████▍ | 26/31 [2:04:42<37:36, 451.27s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 87%|████████▋ | 27/31 [2:12:02<29:51, 447.98s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 90%|█████████ | 28/31 [2:18:59<21:55, 438.61s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▎| 29/31 [2:28:43<16:04, 482.11s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " 97%|█████████▋| 30/31 [2:38:24<08:31, 511.95s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 31/31 [2:47:52<00:00, 528.70s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A"
     ]
    }
   ],
   "source": [
    "for date in tqdm(range(0, 31)):\n",
    "    data = pd.read_csv('../data/train_table_%02d.gz'%(date+1))\n",
    "    data['id'] = data['link'].apply(lambda x:idmap[x])\n",
    "    for i, row in data.iterrows():\n",
    "        link = int(row['id'])\n",
    "        label = row['label']\n",
    "        if label != -1:\n",
    "            mat[date, link, int(row['predict_time'])] = label\n",
    "        for loc in range(1, 6):\n",
    "            mat[date, link, int(row['current_feature_time_%d'%loc])] = int(row['current_feature_status_%d'%loc])\n",
    "        for w in range(1, 5):\n",
    "            week = (5 - w) * 7\n",
    "            if date - week >= 0:\n",
    "                for loc in range(1, 6):\n",
    "                    origin = mat[date - week, link, int(row['history%d_feature_time_%d'%(w, loc)])]\n",
    "                    value = int(row['history%d_feature_status_%d'%(w, loc)])\n",
    "                    if origin != -1 and origin != value:\n",
    "                        print(origin, value, w)\n",
    "                    else:\n",
    "                        mat[date - week, link, int(row['history%d_feature_time_%d'%(w, loc)])] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "date = 31\n",
    "data = pd.read_csv('../data/test_table.gz')\n",
    "data['id'] = data['link'].apply(lambda x:idmap[x])\n",
    "count = 0\n",
    "for i, row in data.iterrows():\n",
    "    link = int(row['id'])\n",
    "    label = row['label']\n",
    "    if label != -1:\n",
    "        mat[date, link, int(row['predict_time'])] = label\n",
    "    for loc in range(1, 6):\n",
    "        mat[date, link, int(row['current_feature_time_%d'%loc])] = int(row['current_feature_status_%d'%loc])\n",
    "    for w in range(1, 5):\n",
    "        week = (5 - w) * 7\n",
    "        if date - week >= 0:\n",
    "            for loc in range(1, 6):\n",
    "                origin = mat[date - week, link, int(row['history%d_feature_time_%d'%(w, loc)])]\n",
    "                value = int(row['history%d_feature_status_%d'%(w, loc)])\n",
    "                if origin != -1 and origin != value:\n",
    "                    count += 1\n",
    "                else:\n",
    "                    mat[date - week, link, int(row['history%d_feature_time_%d'%(w, loc)])] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('mat.npy',mat)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}