Data_process Herbsleb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import social_interaction\n",
    "#import code_interaction\n",
    "#import buggy_commit\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import csv\n",
    "import platform\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import MinMaxScaler,KBinsDiscretizer\n",
    "import statistics\n",
    "from scipy.stats import pearsonr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "607\n"
     ]
    }
   ],
   "source": [
    "if platform.system() == 'Darwin' or platform.system() == 'Linux':\n",
    "    _dir = 'Processed_data/'\n",
    "else:\n",
    "    _dir = 'Processed_data Data\\\\'\n",
    "Files = [join(_dir, f) for f in listdir(_dir) if isfile(join(_dir, f))]\n",
    "print(len(Files))\n",
    "actual_file = 0\n",
    "usable_files = 0\n",
    "size_s = 0\n",
    "size_m = 0\n",
    "size_l = 0\n",
    "result = []\n",
    "size = []\n",
    "total_df = pd.DataFrame()\n",
    "for file in Files:\n",
    "    try:\n",
    "        _df = []\n",
    "        actual_file += 1\n",
    "        df = pd.read_pickle(file)\n",
    "        buggy_commit_data_df = df.loc[0,'buggy_commit_data_df']\n",
    "        commit_data_df = df.loc[0,'commit_data_df']\n",
    "        commit_count = df.loc[0,'commit_count']\n",
    "        sg_data_df = df.loc[0,'sg_data_df']\n",
    "        cg_data_df = df.loc[0,'cg_data_df']\n",
    "        buggy_commit_data_df = buggy_commit_data_df.rename(columns = {'count':'buggy_commit_count'})\n",
    "        commit_data_df = commit_data_df.rename(columns = {'count':'commit_count'})\n",
    "        sg_data_df = sg_data_df.rename(columns = {'count':'sg_node_count'})\n",
    "        cg_data_df = cg_data_df.rename(columns = {'count':'cg_node_count'})\n",
    "        #print(\"social\",sg_data_df)\n",
    "        #print(\"Code\",cg_data_df)\n",
    "        #print(\"commit\",commit_data_df)\n",
    "        #print(\"Buggy\",buggy_commit_data_df)\n",
    "        if commit_data_df.shape[0] > 30:\n",
    "            size_l += 1\n",
    "        elif commit_data_df.shape[0] > 15:\n",
    "            size_m += 1\n",
    "        elif commit_data_df.shape[0] > 8:\n",
    "            size_s += 1\n",
    "        if commit_data_df.shape[0] < 8:\n",
    "            continue\n",
    "        join_df = pd.merge(cg_data_df, sg_data_df, on='committer')\n",
    "        join_df = pd.merge(join_df, buggy_commit_data_df, on='committer')\n",
    "        join_df = pd.merge(join_df, commit_data_df, on='committer')\n",
    "        join_df['buggy_percentage'] = join_df['buggy_commit_count']/join_df['commit_count']\n",
    "        join_df_temp = join_df.drop(labels=['committer','buggy_percentage','commit_count','buggy_commit_count'],axis=1)\n",
    "        scaler = MinMaxScaler()\n",
    "        join_df_temp = scaler.fit_transform(join_df_temp)\n",
    "        committer = join_df.committer.values.tolist()\n",
    "        buggy_percentage = join_df.buggy_percentage.values.tolist()\n",
    "        commit_count = join_df.commit_count.values.tolist()\n",
    "        buggy_commit_count = join_df.buggy_commit_count.values.tolist()\n",
    "        join_df = pd.DataFrame(join_df_temp,columns = ['cg_node_count','sg_node_count'])\n",
    "        join_df['committer'] = committer\n",
    "        join_df['buggy_percentage'] = buggy_percentage\n",
    "        join_df['commit_count'] = commit_count\n",
    "        join_df['buggy_commit_count'] = buggy_commit_count\n",
    "        total_df = pd.concat([total_df,join_df])\n",
    "        join_df.sort_values(by = ['cg_node_count','sg_node_count'],inplace=True,ascending=[False,True])\n",
    "        _sum = np.sort(join_df.cg_node_count)\n",
    "        #plt.plot(_sum)\n",
    "        #plt.show()\n",
    "        #print(join_df)\n",
    "        #print(file.split('/',1)[1],join_df.iloc[0:int(join_df.shape[0]*0.2)].cg_node_count.sum(),join_df.cg_node_count.sum(),\n",
    "        #     join_df.iloc[0:int(join_df.shape[0]*0.2)].cg_node_count.sum()/join_df.cg_node_count.sum(),join_df.shape[0])\n",
    "        #for i in range(cg_data_df.shape[0]):\n",
    "        #    buggy_commit_count = buggy_commit_data_df[buggy_commit_data_df['committer'] == cg_data_df.loc[i,'committer']]['count']\n",
    "        result.append(join_df.iloc[0:int(join_df.shape[0]*0.2)].commit_count.sum()/join_df.commit_count.sum())\n",
    "        size.append(join_df.shape[0])\n",
    "\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# binning together\n",
    "join_df_temp = total_df.drop(labels=['committer','buggy_percentage','commit_count','buggy_commit_count'],axis=1)\n",
    "scaler = KBinsDiscretizer(n_bins=3, encode='ordinal', strategy='kmeans')\n",
    "join_df_temp = scaler.fit_transform(join_df_temp)\n",
    "committer = total_df.committer.values.tolist()\n",
    "buggy_percentage = total_df.buggy_percentage.values.tolist()\n",
    "commit_count = total_df.commit_count.values.tolist()\n",
    "buggy_commit_count = total_df.buggy_commit_count.values.tolist()\n",
    "total_df = pd.DataFrame(join_df_temp,columns = ['cg_node_count','sg_node_count'])\n",
    "total_df['committer'] = committer\n",
    "total_df['buggy_percentage'] = buggy_percentage\n",
    "total_df['commit_count'] = commit_count\n",
    "total_df['buggy_commit_count'] = buggy_commit_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.6070376691654036, 0.0)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pearsonr(total_df.sg_node_count.values.tolist(),total_df.cg_node_count.values.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "607\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed_data/EMMA_final_results.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed_data/.DS_Store\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed_data/pocketio_final_results.pkl\n",
      "Processed_data/telobike_final_results.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed_data/skype-java-api_final_results.pkl\n",
      "Processed_data/Anpro_final_results.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed_data/CDOT-Dashboard_final_results.pkl\n",
      "Processed_data/moonscript_final_results.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed_data/n2n_final_results.pkl\n",
      "Processed_data/Laker-Compendium_final_results.pkl\n",
      "Processed_data/mongoose-auth_final_results.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed_data/sqlabs_final_results.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed_data/ToMaTo_final_results.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 0 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_discretization.py:159: UserWarning: Feature 1 is constant and will be replaced with 0.\n",
      "  \"replaced with 0.\" % jj)\n"
     ]
    }
   ],
   "source": [
    "if platform.system() == 'Darwin' or platform.system() == 'Linux':\n",
    "    _dir = 'Processed_data/'\n",
    "else:\n",
    "    _dir = 'Processed_data Data\\\\'\n",
    "Files = [join(_dir, f) for f in listdir(_dir) if isfile(join(_dir, f))]\n",
    "print(len(Files))\n",
    "actual_file = 0\n",
    "usable_files = 0\n",
    "size_s = 0\n",
    "size_m = 0\n",
    "size_l = 0\n",
    "result = []\n",
    "size = []\n",
    "total_df = pd.DataFrame()\n",
    "for file in Files:\n",
    "    try:\n",
    "        _df = []\n",
    "        actual_file += 1\n",
    "        df = pd.read_pickle(file)\n",
    "        buggy_commit_data_df = df.loc[0,'buggy_commit_data_df']\n",
    "        commit_data_df = df.loc[0,'commit_data_df']\n",
    "        commit_count = df.loc[0,'commit_count']\n",
    "        sg_data_df = df.loc[0,'sg_data_df']\n",
    "        cg_data_df = df.loc[0,'cg_data_df']\n",
    "        buggy_commit_data_df = buggy_commit_data_df.rename(columns = {'count':'buggy_commit_count'})\n",
    "        commit_data_df = commit_data_df.rename(columns = {'count':'commit_count'})\n",
    "        sg_data_df = sg_data_df.rename(columns = {'count':'sg_node_count'})\n",
    "        cg_data_df = cg_data_df.rename(columns = {'count':'cg_node_count'})\n",
    "        #print(\"social\",sg_data_df)\n",
    "        #print(\"Code\",cg_data_df)\n",
    "        #print(\"commit\",commit_data_df)\n",
    "        #print(\"Buggy\",buggy_commit_data_df)\n",
    "        if commit_data_df.shape[0] > 30:\n",
    "            size_l += 1\n",
    "        elif commit_data_df.shape[0] > 15:\n",
    "            size_m += 1\n",
    "        elif commit_data_df.shape[0] > 8:\n",
    "            size_s += 1\n",
    "        if commit_data_df.shape[0] < 8:\n",
    "            continue\n",
    "        join_df = pd.merge(cg_data_df, sg_data_df, on='committer')\n",
    "        join_df = pd.merge(join_df, buggy_commit_data_df, on='committer')\n",
    "        join_df = pd.merge(join_df, commit_data_df, on='committer')\n",
    "        join_df['buggy_percentage'] = join_df['buggy_commit_count']/join_df['commit_count']\n",
    "        join_df.sort_values(by = ['cg_node_count','sg_node_count'],inplace=True,ascending=[False,True])\n",
    "        join_df_temp = join_df.drop(labels=['committer','buggy_percentage','commit_count','buggy_commit_count'],axis=1)\n",
    "        scaler = KBinsDiscretizer(n_bins=3, encode='ordinal', strategy='uniform')\n",
    "        join_df_temp = scaler.fit_transform(join_df_temp)\n",
    "        committer = join_df.committer.values.tolist()\n",
    "        buggy_percentage = join_df.buggy_percentage.values.tolist()\n",
    "        commit_count = join_df.commit_count.values.tolist()\n",
    "        buggy_commit_count = join_df.buggy_commit_count.values.tolist()\n",
    "        join_df = pd.DataFrame(join_df_temp,columns = ['cg_node_count','sg_node_count'])\n",
    "        join_df['committer'] = committer\n",
    "        join_df['buggy_percentage'] = buggy_percentage\n",
    "        join_df['commit_count'] = commit_count\n",
    "        join_df['buggy_commit_count'] = buggy_commit_count\n",
    "        total_df = pd.concat([total_df,join_df])\n",
    "    except Exception as e:\n",
    "        print(file)\n",
    "        continue\n",
    "total_df.reset_index(drop=True,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "HH = []\n",
    "HM = []\n",
    "HL = []\n",
    "MH = []\n",
    "MM = []\n",
    "ML = []\n",
    "LH = []\n",
    "LM = []\n",
    "LL = []\n",
    "for i in range(total_df.shape[0]):\n",
    "    if total_df.iloc[i,0] == 2:\n",
    "        if total_df.iloc[i,1] == 2:\n",
    "            HH.append(total_df.iloc[i,3])\n",
    "        elif total_df.iloc[i,1] == 1:\n",
    "            HM.append(total_df.iloc[i,3])\n",
    "        else:\n",
    "            HL.append(total_df.iloc[i,3])\n",
    "    elif total_df.iloc[i,0] == 1:\n",
    "        if total_df.iloc[i,1] == 2:\n",
    "            MH.append(total_df.iloc[i,3])\n",
    "        elif total_df.iloc[i,1] == 1:\n",
    "            MM.append(total_df.iloc[i,3])\n",
    "        else:\n",
    "            ML.append(total_df.iloc[i,3])\n",
    "    else:\n",
    "        if total_df.iloc[i,1] == 2:\n",
    "            LH.append(total_df.iloc[i,3])\n",
    "        elif total_df.iloc[i,1] == 1:\n",
    "            LM.append(total_df.iloc[i,3])\n",
    "        else:\n",
    "            LL.append(total_df.iloc[i,3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HH 0.42010934034761976\n",
      "HM 0.47674418604651164\n",
      "HL 0.47535090722355355\n",
      "MH 0.3593165593519795\n",
      "MM 0.45554445554445555\n",
      "ML 0.5772573124205171\n",
      "LH 0.2777777777777778\n",
      "LM 0.41025641025641024\n",
      "LL 0.7\n"
     ]
    }
   ],
   "source": [
    "print('HH',statistics.median(HH))\n",
    "print('HM',statistics.median(HM))\n",
    "print('HL',statistics.median(HL))\n",
    "print('MH',statistics.median(MH))\n",
    "print('MM',statistics.median(MM))\n",
    "print('ML',statistics.median(ML))\n",
    "print('LH',statistics.median(LH))\n",
    "print('LM',statistics.median(LM))\n",
    "print('LL',statistics.median(LL))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Large\n",
    "HH 0.40824742268041236\n",
    "HM 0.4301571588286962\n",
    "HL 0.490811339198436\n",
    "MH 0.42628205128205127\n",
    "MM 0.4359506664339378\n",
    "ML 0.5\n",
    "LH 0.3152147163775071\n",
    "LM 0.34325396825396826\n",
    "LL 0.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}