als_thing/compute_feature_vector.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>repoid</th>\n",
       "      <th>uname</th>\n",
       "      <th>reponame</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>24466870</td>\n",
       "      <td>59996401</td>\n",
       "      <td>zzkkui</td>\n",
       "      <td>yapi</td>\n",
       "      <td>2018-03-13T01:07:02Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24466870</td>\n",
       "      <td>5239185</td>\n",
       "      <td>zzkkui</td>\n",
       "      <td>quill</td>\n",
       "      <td>2018-03-13T01:07:02Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24466870</td>\n",
       "      <td>76567547</td>\n",
       "      <td>zzkkui</td>\n",
       "      <td>vue-quill-editor</td>\n",
       "      <td>2018-03-13T01:07:02Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24466870</td>\n",
       "      <td>105479936</td>\n",
       "      <td>zzkkui</td>\n",
       "      <td>react-email-editor</td>\n",
       "      <td>2018-03-13T01:07:02Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>24466870</td>\n",
       "      <td>16179237</td>\n",
       "      <td>zzkkui</td>\n",
       "      <td>virtual-dom</td>\n",
       "      <td>2018-03-13T01:07:02Z</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        uid     repoid   uname            reponame                  date\n",
       "0  24466870   59996401  zzkkui                yapi  2018-03-13T01:07:02Z\n",
       "1  24466870    5239185  zzkkui               quill  2018-03-13T01:07:02Z\n",
       "2  24466870   76567547  zzkkui    vue-quill-editor  2018-03-13T01:07:02Z\n",
       "3  24466870  105479936  zzkkui  react-email-editor  2018-03-13T01:07:02Z\n",
       "4  24466870   16179237  zzkkui         virtual-dom  2018-03-13T01:07:02Z"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_file = pd.read_csv('data/train.csv', index_col=None, sep=',') \\\n",
    "    .drop_duplicates()\n",
    "train_file.columns = ['uid', 'repoid', 'uname', 'reponame', 'date']\n",
    "train_file = train_file.drop_duplicates()\n",
    "train_file.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>repoid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>24466870</td>\n",
       "      <td>59996401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24466870</td>\n",
       "      <td>5239185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24466870</td>\n",
       "      <td>76567547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24466870</td>\n",
       "      <td>105479936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>24466870</td>\n",
       "      <td>16179237</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        uid     repoid\n",
       "0  24466870   59996401\n",
       "1  24466870    5239185\n",
       "2  24466870   76567547\n",
       "3  24466870  105479936\n",
       "4  24466870   16179237"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# take only uid / repoid\n",
    "train_file = train_file[['uid', 'repoid']]\n",
    "train_file.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark import SparkContext\n",
    "from pyspark.sql import SQLContext\n",
    "from pyspark.sql.functions import reverse\n",
    "\n",
    "from pyspark.mllib.recommendation import ALS\n",
    "from pyspark.mllib.linalg.distributed import CoordinateMatrix, MatrixEntry, RowMatrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = SparkContext(appName=\"RR\")\n",
    "sqlContext = SQLContext(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def first_two_column(row):\n",
    "    return (int(row[0]), int(row[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_rdd = sqlContext.createDataFrame(train_file).rdd.map(first_two_column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# returns\n",
    "# 1. idx -> keys\n",
    "# 2. idx -> feature_vecs\n",
    "def train(training_rdd):\n",
    "    # training_rdd - (uid, prod_id)\n",
    "    \n",
    "    # this takes a short while too\n",
    "    model = ALS.trainImplicit( \\\n",
    "        training_rdd.map(lambda rr: (rr[0], rr[1], 1)),\n",
    "        rank=16,\n",
    "        iterations=10,\n",
    "        lambda_=0.1,\n",
    "        alpha=80.0\n",
    "    )\n",
    "    return model.productFeatures()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def similarity(feature_vecs, columnSimilarities_threshold):\n",
    "    # transpose `prod_features_rdd`\n",
    "    def transpose(rm):\n",
    "        cm = CoordinateMatrix(\n",
    "            rm.rows.zipWithIndex().flatMap(\n",
    "                lambda x: [MatrixEntry(x[1], j, v) for j, v in enumerate(x[0])]\n",
    "            )\n",
    "        )\n",
    "        return cm.transpose().toRowMatrix()\n",
    "    rowmat = RowMatrix(feature_vecs)\n",
    "    colmat = transpose(rowmat)\n",
    "    sims = colmat.columnSimilarities(columnSimilarities_threshold)\n",
    "    return sims"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def most_similar(sims):\n",
    "    # list of `MatrixEntry(5, 649, 0.2423914277654137)`\n",
    "    sim_entries = sims.entries\n",
    "\n",
    "    # duplicate sim_entries to get all pairs of (i, j, val)\n",
    "    # list of `(23, (66, -0.14212846655328612))`\n",
    "    # n entries for every product === total length n*n\n",
    "    sim_entries_full = sim_entries.flatMap(lambda rr: ((rr.i, (rr.j, rr.value)), (rr.j, (rr.i, rr.value))))\n",
    "\n",
    "    def foldByKeyCompare(r1, r2):\n",
    "        if r1 is None:\n",
    "            return r2\n",
    "        return r1 if r1[1] > r2[1] else r2\n",
    "    \n",
    "    # for each prod, find prod with highest similarity\n",
    "    # list of `(23, (66, -0.14212846655328612))`, 1 entry for every product\n",
    "    sim_entries_highest = sim_entries_full.foldByKey(None, foldByKeyCompare)\n",
    "    \n",
    "    return sim_entries_highest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def join_key(keys, most_similars):\n",
    "    '''\n",
    "    keys - list of repo ids\n",
    "        [28, 12044, 147180, 284868 ...]\n",
    "    most_similars - most similar repo for each repo, expressed as index in `keys`\n",
    "        (428, (743, 0.0031531581486397564))\n",
    "    '''\n",
    "    \n",
    "    # order keys\n",
    "    # [(0, 2421308), (1, 5218272), (2, 5691060), (3, 8193932) ...]\n",
    "    orderedKeys = keys.zipWithIndex().map(lambda rr: (rr[1], rr[0]))\n",
    "    \n",
    "    # join with one repo\n",
    "    # reshape to expose another repo index\n",
    "    # `(28, (14325757, 0.0020377382811490874))`\n",
    "    sim_entries_left_half = most_similars \\\n",
    "        .join(orderedKeys) \\\n",
    "        .map(lambda rr: (rr[1][0][0], (rr[1][1], rr[1][0][1])))\n",
    "    \n",
    "    # join another repo\n",
    "    # reshape to flat result\n",
    "    # `(109245282, 14325757, 0.014610917357937986)`\n",
    "    repo_pairs = sim_entries_left_half.join(orderedKeys).map(lambda rr: (rr[1][0][0], rr[1][1], rr[1][0][1]))\n",
    "    \n",
    "    return repo_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def timer(func, desc):\n",
    "    import time\n",
    "    start = time.time()\n",
    "    result = func()\n",
    "    end = time.time()\n",
    "    print(desc, \":\", end - start)\n",
    "    return result\n",
    "\n",
    "def bind(func, *params):\n",
    "    def newfunc():\n",
    "        return func(*params)\n",
    "    return newfunc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ALS : 9.76835012435913\n"
     ]
    }
   ],
   "source": [
    "productFeatures = timer(bind(train, training_rdd), \"ALS\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy import linalg as LA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "prod_features_arr = productFeatures.values().collect()\n",
    "np_features = np.array(prod_features_arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "np_keys = np.array(productFeatures.keys().collect())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "np_la_norm = np.array(list(map(LA.norm, np_features)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('productFeatures', np_features)\n",
    "np.save('prod_feature_norm', list(np_la_norm))\n",
    "np.save('repo_ids', np_keys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(np_la_norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8187300, 16)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([8016338, 5723894, 2367101, ..., 5629124, 5331759,       0])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argsort(np_features.dot(np_features[0]) / (np_la_norm * np_la_norm[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "        0.,  0.,  0.])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np_features[0] - np_features[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cache ALS : 0.009809017181396484\n",
      "using 5 samples\n"
     ]
    }
   ],
   "source": [
    "productFeatures_small = productFeatures.sample(False, 0.01)\n",
    "timer(productFeatures_small.cache, \"cache ALS\")\n",
    "print(\"using\", len(productFeatures_small.collect()), \"samples\")\n",
    "\n",
    "# productFeatures_small = sc.parallelize([(kk, tuple(vv)) for kk, vv in [\n",
    "#     (0, (1 for ii in range(16))), \\\n",
    "#     (1, (1 for ii in range(16))), \\\n",
    "#     (2, (2 for ii in range(16))), \\\n",
    "#     (3, (ii for ii in range(16))) \\\n",
    "# ]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(12,\n",
       "  array('d', [-0.9812264442443848, -0.014173307456076145, -0.1922406405210495, 0.10215497016906738, -0.11043383181095123, 0.2534223794937134, -0.797737717628479, -0.12205567210912704, -0.6269376277923584, -0.2890850305557251, -0.4578281342983246, 0.48876720666885376, -0.002395555144175887, -0.38595688343048096, -0.2819075584411621, 0.7047893404960632])),\n",
       " (13,\n",
       "  array('d', [-0.12482436746358871, -0.03038131631910801, 0.12130644917488098, 0.15238629281520844, -0.2096806913614273, 0.2443142831325531, -0.037369176745414734, 0.14703594148159027, -0.2823255658149719, -0.12168966978788376, 0.37711790204048157, 0.14086169004440308, -0.19714556634426117, -0.3474962115287781, -0.8193628787994385, 0.21149735152721405])),\n",
       " (10,\n",
       "  array('d', [-0.9812264442443848, -0.014173307456076145, -0.1922406405210495, 0.10215497016906738, -0.11043383181095123, 0.2534223794937134, -0.797737717628479, -0.12205567210912704, -0.6269376277923584, -0.2890850305557251, -0.4578281342983246, 0.48876720666885376, -0.002395555144175887, -0.38595688343048096, -0.2819075584411621, 0.7047893404960632])),\n",
       " (14,\n",
       "  array('d', [-0.20569327473640442, -0.45948052406311035, -0.14928635954856873, -0.16120681166648865, -0.1588963121175766, 0.5270352959632874, 0.45235878229141235, -0.2029407024383545, -0.11108352988958359, 0.07381252944469452, 0.08862240612506866, 0.2555128335952759, -0.19588065147399902, 0.8397274017333984, -0.0014846274862065911, 0.2165004014968872])),\n",
       " (11,\n",
       "  array('d', [-0.9812264442443848, -0.014173307456076145, -0.1922406405210495, 0.10215497016906738, -0.11043383181095123, 0.2534223794937134, -0.797737717628479, -0.12205567210912704, -0.6269376277923584, -0.2890850305557251, -0.4578281342983246, 0.48876720666885376, -0.002395555144175887, -0.38595688343048096, -0.2819075584411621, 0.7047893404960632]))]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "productFeatures_small.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "similarity : 0.011590003967285156\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(12, 10), (13, 10), (14, 10), (11, 10), (10, 11)]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keys = productFeatures_small.keys()\n",
    "feature_vecs = productFeatures_small.values()\n",
    "\n",
    "sims = similarity(feature_vecs, 0.1)\n",
    "most_similars = most_similar(sims)\n",
    "timer(most_similars.cache, \"similarity\")\n",
    "\n",
    "repo_pairs = join_key(keys, most_similars)\n",
    "# repo_pairs.map(lambda rr: rr[:2]).collect()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(12, 10, 1.0),\n",
       " (13, 10, 0.3958634364711903),\n",
       " (14, 10, -0.0009971295093847005),\n",
       " (11, 10, 1.0),\n",
       " (10, 11, 1.0)]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "repo_pairs.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>repoid</th>\n",
       "      <th>uname</th>\n",
       "      <th>reponame</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>24466870</td>\n",
       "      <td>59996401</td>\n",
       "      <td>zzkkui</td>\n",
       "      <td>yapi</td>\n",
       "      <td>2018-03-13T01:07:02Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24466870</td>\n",
       "      <td>5239185</td>\n",
       "      <td>zzkkui</td>\n",
       "      <td>quill</td>\n",
       "      <td>2018-03-13T01:07:02Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24466870</td>\n",
       "      <td>76567547</td>\n",
       "      <td>zzkkui</td>\n",
       "      <td>vue-quill-editor</td>\n",
       "      <td>2018-03-13T01:07:02Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24466870</td>\n",
       "      <td>105479936</td>\n",
       "      <td>zzkkui</td>\n",
       "      <td>react-email-editor</td>\n",
       "      <td>2018-03-13T01:07:02Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>24466870</td>\n",
       "      <td>16179237</td>\n",
       "      <td>zzkkui</td>\n",
       "      <td>virtual-dom</td>\n",
       "      <td>2018-03-13T01:07:02Z</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        uid     repoid   uname            reponame                  date\n",
       "0  24466870   59996401  zzkkui                yapi  2018-03-13T01:07:02Z\n",
       "1  24466870    5239185  zzkkui               quill  2018-03-13T01:07:02Z\n",
       "2  24466870   76567547  zzkkui    vue-quill-editor  2018-03-13T01:07:02Z\n",
       "3  24466870  105479936  zzkkui  react-email-editor  2018-03-13T01:07:02Z\n",
       "4  24466870   16179237  zzkkui         virtual-dom  2018-03-13T01:07:02Z"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_file.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_file_names = train_file[['uname', 'reponame']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uname</th>\n",
       "      <th>reponame</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>zzkkui</td>\n",
       "      <td>yapi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>zzkkui</td>\n",
       "      <td>quill</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>zzkkui</td>\n",
       "      <td>vue-quill-editor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>zzkkui</td>\n",
       "      <td>react-email-editor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>zzkkui</td>\n",
       "      <td>virtual-dom</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    uname            reponame\n",
       "0  zzkkui                yapi\n",
       "1  zzkkui               quill\n",
       "2  zzkkui    vue-quill-editor\n",
       "3  zzkkui  react-email-editor\n",
       "4  zzkkui         virtual-dom"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_file_names.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "field reponame: Can not merge type <class 'pyspark.sql.types.StringType'> and <class 'pyspark.sql.types.DoubleType'>",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-c3f018ffd59a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msqlContext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreateDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_file_names\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/Cellar/apache-spark/2.3.0/libexec/python/pyspark/sql/context.py\u001b[0m in \u001b[0;36mcreateDataFrame\u001b[0;34m(self, data, schema, samplingRatio, verifySchema)\u001b[0m\n\u001b[1;32m    300\u001b[0m         \u001b[0mPy4JJavaError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m...\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    301\u001b[0m         \"\"\"\n\u001b[0;32m--> 302\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msparkSession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreateDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mschema\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msamplingRatio\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverifySchema\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    303\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    304\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0msince\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1.3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/apache-spark/2.3.0/libexec/python/pyspark/sql/session.py\u001b[0m in \u001b[0;36mcreateDataFrame\u001b[0;34m(self, data, schema, samplingRatio, verifySchema)\u001b[0m\n\u001b[1;32m    687\u001b[0m             \u001b[0mrdd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mschema\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_createFromRDD\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprepare\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mschema\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msamplingRatio\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    688\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 689\u001b[0;31m             \u001b[0mrdd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mschema\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_createFromLocal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprepare\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mschema\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    690\u001b[0m         \u001b[0mjrdd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jvm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSerDeUtil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoJavaArray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrdd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_to_java_object_rdd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    691\u001b[0m         \u001b[0mjdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jsparkSession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapplySchemaToPythonRDD\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjrdd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrdd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mschema\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/apache-spark/2.3.0/libexec/python/pyspark/sql/session.py\u001b[0m in \u001b[0;36m_createFromLocal\u001b[0;34m(self, data, schema)\u001b[0m\n\u001b[1;32m    408\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    409\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mschema\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mschema\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 410\u001b[0;31m             \u001b[0mstruct\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_inferSchemaFromList\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnames\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mschema\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    411\u001b[0m             \u001b[0mconverter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_create_converter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstruct\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    412\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconverter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/apache-spark/2.3.0/libexec/python/pyspark/sql/session.py\u001b[0m in \u001b[0;36m_inferSchemaFromList\u001b[0;34m(self, data, names)\u001b[0m\n\u001b[1;32m    340\u001b[0m             warnings.warn(\"inferring schema from dict is deprecated,\"\n\u001b[1;32m    341\u001b[0m                           \"please use pyspark.sql.Row instead\")\n\u001b[0;32m--> 342\u001b[0;31m         \u001b[0mschema\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreduce\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_merge_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0m_infer_schema\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnames\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mrow\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    343\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0m_has_nulltype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mschema\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    344\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Some of types cannot be determined after inferring\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/apache-spark/2.3.0/libexec/python/pyspark/sql/types.py\u001b[0m in \u001b[0;36m_merge_type\u001b[0;34m(a, b, name)\u001b[0m\n\u001b[1;32m   1131\u001b[0m         fields = [StructField(f.name, _merge_type(f.dataType, nfs.get(f.name, NullType()),\n\u001b[1;32m   1132\u001b[0m                                                   name=new_name(f.name)))\n\u001b[0;32m-> 1133\u001b[0;31m                   for f in a.fields]\n\u001b[0m\u001b[1;32m   1134\u001b[0m         \u001b[0mnames\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1135\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnfs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/apache-spark/2.3.0/libexec/python/pyspark/sql/types.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m   1131\u001b[0m         fields = [StructField(f.name, _merge_type(f.dataType, nfs.get(f.name, NullType()),\n\u001b[1;32m   1132\u001b[0m                                                   name=new_name(f.name)))\n\u001b[0;32m-> 1133\u001b[0;31m                   for f in a.fields]\n\u001b[0m\u001b[1;32m   1134\u001b[0m         \u001b[0mnames\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1135\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnfs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/apache-spark/2.3.0/libexec/python/pyspark/sql/types.py\u001b[0m in \u001b[0;36m_merge_type\u001b[0;34m(a, b, name)\u001b[0m\n\u001b[1;32m   1124\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1125\u001b[0m         \u001b[0;31m# TODO: type cast (such as int -> long)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1126\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_msg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Can not merge type %s and %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1127\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1128\u001b[0m     \u001b[0;31m# same type\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: field reponame: Can not merge type <class 'pyspark.sql.types.StringType'> and <class 'pyspark.sql.types.DoubleType'>"
     ]
    }
   ],
   "source": [
    "sqlContext.createDataFrame(train_file_names).take(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_rdd"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}