数据处理-2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "处理Feature，并结合第1部分里得到的不匹配的节点，生成新的Feature，并保存。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path\n",
    "base_path = './final_data'\n",
    "publish_path = ''\n",
    "\n",
    "diff_node_path = os.path.join(base_path, publish_path, 'diff_nodes.csv')\n",
    "train_nodes_path = os.path.join(base_path, publish_path, 'train_nodes.csv')\n",
    "val_nodes_path = os.path.join(base_path, publish_path, 'validation_nodes.csv')\n",
    "test_nodes_path = os.path.join(base_path, publish_path, 'test_nodes.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 处理节点的特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_node(line):\n",
    "    nid, feat_json, label = line.strip().split('\\\"')\n",
    "    \n",
    "    feat_list = [float(feat[1:-1]) for feat in feat_json[1:-1].split(', ')]\n",
    "    \n",
    "    if len(feat_list) != 300:\n",
    "        print('此行数据有问题 {}'.format(line))\n",
    "    \n",
    "    return nid[:-1], feat_list, label[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed 100000 train rows\n",
      "Processed 200000 train rows\n",
      "Processed 300000 train rows\n",
      "Processed 400000 train rows\n",
      "Processed 500000 train rows\n",
      "Processed 600000 train rows\n",
      "Processed 700000 train rows\n",
      "Processed 800000 train rows\n",
      "Processed 900000 train rows\n",
      "Processed 1000000 train rows\n",
      "Processed 1100000 train rows\n",
      "Processed 1200000 train rows\n",
      "Processed 1300000 train rows\n",
      "Processed 1400000 train rows\n",
      "Processed 1500000 train rows\n",
      "Processed 1600000 train rows\n",
      "Processed 1700000 train rows\n",
      "Processed 1800000 train rows\n",
      "Processed 1900000 train rows\n",
      "Processed 2000000 train rows\n",
      "Processed 2100000 train rows\n",
      "Processed 2200000 train rows\n",
      "Processed 2300000 train rows\n",
      "Processed 2400000 train rows\n",
      "Processed 2500000 train rows\n",
      "Processed 2600000 train rows\n",
      "Processed 2700000 train rows\n",
      "Processed 2800000 train rows\n",
      "Processed 2900000 train rows\n",
      "Processed 3000000 train rows\n",
      "Processed 100000 validation rows\n",
      "Processed 200000 validation rows\n",
      "Processed 300000 validation rows\n",
      "Processed 400000 validation rows\n",
      "Processed 500000 validation rows\n",
      "Processed 100000 validation rows\n",
      "Processed 200000 validation rows\n",
      "Processed 300000 validation rows\n",
      "Processed 400000 validation rows\n",
      "Processed 500000 validation rows\n",
      "Processed 600000 validation rows\n",
      "Processed 700000 validation rows\n",
      "Processed 800000 validation rows\n",
      "Processed 900000 validation rows\n",
      "Processed 1000000 validation rows\n",
      "Processed 1100000 validation rows\n",
      "Processed 1200000 validation rows\n",
      "Processed 1300000 validation rows\n",
      "Processed 1400000 validation rows\n",
      "Processed 1500000 validation rows\n",
      "Processed 1600000 validation rows\n"
     ]
    }
   ],
   "source": [
    "# 下面处理特征Feature并存储亿备后用\n",
    "# nid_list = []\n",
    "feature_list = []\n",
    "\n",
    "with open(train_nodes_path, 'r') as f:\n",
    "    i = 0\n",
    "    \n",
    "    for line in f:\n",
    "        if i > 0:\n",
    "            _, features, _ = process_node(line)\n",
    "#             nid_list.append(nid)\n",
    "            feature_list.append(features)\n",
    "        i += 1\n",
    "        if i % 100000 == 0:\n",
    "            print('Processed {} train rows'.format(i))\n",
    "    \n",
    "with open(val_nodes_path, 'r') as f:\n",
    "    i = 0\n",
    "    \n",
    "    for line in f:\n",
    "        if i > 0:\n",
    "            _, features, _ = process_node(line)\n",
    "#             nid_list.append(nid)\n",
    "            feature_list.append(features)\n",
    "        i += 1\n",
    "        if i % 100000 == 0:\n",
    "            print('Processed {} validation rows'.format(i))\n",
    "\n",
    "with open(test_nodes_path, 'r') as f:\n",
    "    i = 0\n",
    "    \n",
    "    for line in f:\n",
    "        if i > 0:\n",
    "            _, features, _ = process_node(line)\n",
    "#             nid_list.append(nid)\n",
    "            feature_list.append(features)\n",
    "        i += 1\n",
    "        if i % 100000 == 0:\n",
    "            print('Processed {} validation rows'.format(i))\n",
    "            \n",
    "\n",
    "# nid_arr = np.array(nid_list)\n",
    "feat_arr = np.array(feature_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 删除list以节省内存\n",
    "del feature_list\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 给未匹配上的节点造300维的特征，这里用其他所有节点的300维的平均值来作为他们的特征\n",
    "# 更好的方法是用每个节点的所有邻居的特征的平均，这里就不搞这么复杂了。\n",
    "diff_node_feat_arr = np.tile(np.mean(feat_arr, axis=0),(424, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5346177, 300)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feat_arr = np.concatenate((feat_arr, diff_node_feat_arr), axis=0)\n",
    "feat_arr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用Numpy保存特征为.npy格式，以节省存储空间和提高读写速度\n",
    "with open(os.path.join(base_path, publish_path, './features.npy'), 'wb') as f:\n",
    "    np.save(f, feat_arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.65447729e-01, -2.06784075e-01,  1.23800340e-01, -1.27744982e-01,\n",
       "       -5.46123311e-02, -8.18064214e-02,  3.57811122e-02,  8.80050055e-02,\n",
       "       -7.53288820e-02, -1.56775955e+00,  7.59375579e-02,  2.18963792e-01,\n",
       "       -3.47929514e-02,  2.75621000e-01,  2.74354380e-01, -3.23156998e-02,\n",
       "        3.95653434e-03, -4.01394397e-01,  5.45779787e-02,  2.11846699e-01,\n",
       "        6.27020868e-02, -2.99849561e-02,  7.04652388e-02,  4.45794317e-02,\n",
       "        8.12155046e-02, -2.73458110e-01,  7.17670784e-02, -5.83930657e-02,\n",
       "       -1.11664584e-01, -1.53301753e-01,  1.14251968e-01, -4.06278533e-02,\n",
       "        8.93757684e-02, -6.68480415e-02, -1.56442087e-02, -6.66905498e-02,\n",
       "        1.78514793e-01, -4.33929428e-02, -7.51053499e-02,  3.38231255e-01,\n",
       "        9.47836468e-02,  1.02571917e-01,  1.67372718e-01,  2.18996915e-02,\n",
       "        3.26381817e-02, -1.33619296e-01,  4.64139408e-02, -2.56323592e-01,\n",
       "        2.10549769e-02,  1.00766371e-01, -3.48834411e-02,  1.14433107e-02,\n",
       "       -4.04550295e-02, -6.87207509e-03, -3.71818352e-02, -2.68626975e-02,\n",
       "        2.61101772e-02, -7.26619699e-02,  8.30491757e-02, -6.48526865e-02,\n",
       "        9.43357236e-02,  4.83663156e-02,  1.29575336e-01, -1.94613399e-01,\n",
       "       -1.51744897e-01,  1.66828756e-01, -8.84592620e-02, -5.51867959e-02,\n",
       "       -1.17060972e-01, -8.37361022e-02, -2.17924777e-01,  2.06457754e-02,\n",
       "       -8.65103058e-02, -6.08056352e-02, -1.87352803e-02,  3.48874388e-02,\n",
       "       -2.02109228e-02, -1.22501547e-01,  7.39078025e-02, -1.63570759e-01,\n",
       "       -4.12248718e-02, -1.65271649e-02,  1.71800065e-01,  7.20659005e-02,\n",
       "       -6.04165818e-03,  1.99682659e-01,  1.94429006e-01, -8.11011010e-01,\n",
       "       -1.36286321e-01, -3.24001369e-02,  3.32838476e-02,  5.56948316e-02,\n",
       "       -4.90884706e-02, -6.62709143e-02, -1.12345222e-01,  1.92429656e-01,\n",
       "       -9.49748709e-02,  5.77257802e-02, -1.00265583e-01,  1.14468919e-01,\n",
       "       -1.18696963e-01, -1.45284359e-01,  5.77327850e-02,  1.13016629e-01,\n",
       "       -8.83497660e-02, -2.51696642e-01, -1.08337553e-02, -1.13580774e-01,\n",
       "       -1.07927182e-01, -1.10399743e-01,  1.86270459e-02,  2.55797645e-01,\n",
       "        1.83104955e-02, -3.57915547e-02, -1.46023842e-02,  1.66247392e-01,\n",
       "       -1.34331869e-01, -3.87905189e-02,  7.79452573e-02, -6.53826593e-02,\n",
       "        5.54363431e-02, -1.11054822e-02, -3.04751116e-02,  5.27066349e-02,\n",
       "        2.46076949e-01, -8.24572990e-02,  5.31251173e-03,  7.55993269e-02,\n",
       "       -8.75015790e-02,  9.30560632e-02,  5.48892013e-02, -1.47337055e-01,\n",
       "       -1.17164227e-01, -2.91028992e-02, -1.18727596e-01,  3.23994336e-02,\n",
       "        5.78619035e-02, -3.95620353e-02,  7.95022458e-02, -1.19908039e-01,\n",
       "        6.81885378e-01,  1.32162521e-01, -1.88713909e-01, -3.58391059e-02,\n",
       "        5.95323313e-02, -4.31908932e-02,  1.62250514e-01,  1.20175104e-02,\n",
       "        4.16568126e-02,  8.33570918e-02, -1.10243283e-01,  1.07902021e-01,\n",
       "       -1.26521323e-01,  1.12844246e-02,  6.27730104e-02,  6.21901895e-02,\n",
       "       -3.31287205e-02, -4.89059913e-02,  8.82485052e-02, -9.12004499e-03,\n",
       "       -2.78961646e-01,  1.30393703e-02, -1.09494982e-02,  2.03854946e-02,\n",
       "        1.45022683e-01,  9.52451547e-02, -3.04579473e-01, -3.10128689e-02,\n",
       "       -1.28773056e-01,  9.44338157e-02, -1.68580855e-01, -9.29804489e-02,\n",
       "        1.75168860e-01,  3.98155035e-03,  2.62491455e-01, -2.29299249e-02,\n",
       "        7.55326010e-02, -4.60569529e-03, -2.69577316e-01,  9.22820247e-02,\n",
       "       -9.37035479e-03, -1.37233857e-02,  8.74457899e-02,  2.20330087e-01,\n",
       "        1.27206997e-01,  1.18574719e-01, -2.17939269e-02, -1.44181295e-01,\n",
       "        1.17681978e-01,  4.03423371e-02,  1.81489639e-01, -6.74334578e-02,\n",
       "        5.99498639e-02, -1.21470137e-01,  1.72224031e-02, -4.85130683e-02,\n",
       "        1.42239483e-01,  5.65276395e-02,  5.13538791e-02, -1.29299502e-01,\n",
       "       -2.77287999e-02,  2.50927458e-01,  1.57139345e-01, -1.48072144e-01,\n",
       "       -7.81678795e-02, -1.33583645e-01,  8.35795455e-02,  6.10525267e-03,\n",
       "        2.16065613e-01,  1.04309584e-01, -3.36767889e-02,  9.85715248e-02,\n",
       "       -1.67005564e-01,  3.21130874e-01, -1.30199183e-01, -1.63167346e-01,\n",
       "        2.05649133e-01, -3.84456056e-01,  2.30536569e-01, -3.37762492e-02,\n",
       "        1.73634280e-01, -1.32487147e-03,  5.50683771e-02, -1.77121633e-01,\n",
       "       -1.49468776e-01, -6.28062374e-02, -1.93994703e-02, -3.28240465e-02,\n",
       "       -2.16794804e-01, -2.88400047e-01,  1.89540815e-01,  1.99809282e-02,\n",
       "       -7.90708427e-02, -1.69437169e-02, -3.44156972e-02,  1.86782901e-01,\n",
       "       -5.24401635e-02,  5.13464736e-02, -6.49783338e-02, -5.14453631e-02,\n",
       "       -6.95958507e-02, -2.00027258e-01, -1.35496420e-01, -1.15528203e-01,\n",
       "       -1.03333056e-01,  1.80223617e-01,  2.28424752e-02,  1.98229670e-01,\n",
       "        2.53902424e-01, -6.73627459e-03, -6.66882332e-02, -4.53740885e-02,\n",
       "       -1.05108677e-01,  1.95218148e-01, -1.42062693e-01, -3.95023739e-01,\n",
       "        4.07923915e-02, -1.95471586e-02, -1.70232511e-01,  4.88997903e-02,\n",
       "       -2.38473615e-01, -2.25298512e-01,  1.85316232e-01, -1.72017794e-01,\n",
       "        4.80221384e-02,  1.36545263e-01,  2.71561983e-02, -9.85245569e-03,\n",
       "       -6.51411777e-02, -5.96135742e-02, -3.45868388e-01,  1.33200947e-01,\n",
       "        8.08588423e-02,  4.07913703e-02,  7.81983720e-02,  3.44882696e-02,\n",
       "       -3.12555459e-02, -9.88200834e-02,  4.82478827e-02, -1.09225497e-01,\n",
       "       -4.16300596e-03, -1.85741084e-01, -1.63361108e-01,  1.46834327e-01,\n",
       "       -2.29509476e-02, -1.66409098e-02, -6.68053783e-02,  6.98620927e-02,\n",
       "       -1.08873888e-01, -3.30613799e-02,  3.94694049e-01, -1.08729760e-01,\n",
       "       -1.98334768e-01, -2.05030462e-01, -2.44849524e-02,  6.76704016e-02,\n",
       "        1.32814178e-01,  8.41047450e-02, -2.20487337e-01, -2.54510220e-01])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feat_arr[5345753,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}