Regular expressions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Date Extracter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### extracting lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dates.txt') as file:\n",
    "    for line in file:\n",
    "        line = line.rstrip()                    #rstrip() strips the string's trailing characters, default spaces\n",
    "        lines.append(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['03/25/93 Total time of visit (in minutes):',\n",
       " '6/18/85 Primary Care Doctor:',\n",
       " 'sshe plans to move as of 7/8/71 In-Home Services: None',\n",
       " '7 on 9/27/75 Audit C Score Current:',\n",
       " '2/6/96 sleep studyPain Treatment Pain Level (Numeric Scale): 7']"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### making series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "ser = pd.Series(lines, name='Text')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0           03/25/93 Total time of visit (in minutes):\n",
       "1                         6/18/85 Primary Care Doctor:\n",
       "2    sshe plans to move as of 7/8/71 In-Home Servic...\n",
       "3                  7 on 9/27/75 Audit C Score Current:\n",
       "4    2/6/96 sleep studyPain Treatment Pain Level (N...\n",
       "Name: Text, dtype: object"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ser.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### converting series to dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame = pd.DataFrame(ser)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>03/25/93 Total time of visit (in minutes):</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6/18/85 Primary Care Doctor:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sshe plans to move as of 7/8/71 In-Home Servic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7 on 9/27/75 Audit C Score Current:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2/6/96 sleep studyPain Treatment Pain Level (N...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text\n",
       "0         03/25/93 Total time of visit (in minutes):\n",
       "1                       6/18/85 Primary Care Doctor:\n",
       "2  sshe plans to move as of 7/8/71 In-Home Servic...\n",
       "3                7 on 9/27/75 Audit C Score Current:\n",
       "4  2/6/96 sleep studyPain Treatment Pain Level (N..."
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### extracting first few dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame['Dates1'] = frame['Text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0           03/25/93 Total time of visit (in minutes):\n",
       "1                         6/18/85 Primary Care Doctor:\n",
       "2    sshe plans to move as of 7/8/71 In-Home Servic...\n",
       "3                  7 on 9/27/75 Audit C Score Current:\n",
       "4    2/6/96 sleep studyPain Treatment Pain Level (N...\n",
       "Name: Dates1, dtype: object"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame['Dates1'].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### first few dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extractor1(x):\n",
    "    dt = re.search(\"\\d{1,2}[/-]\\d{1,2}[/-]\\d{2,4}\", x)\n",
    "    if(dt != None):\n",
    "        return dt.group(0)\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame['Dates1']=frame['Dates1'].apply(extractor1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Dates1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>03/25/93 Total time of visit (in minutes):</td>\n",
       "      <td>03/25/93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6/18/85 Primary Care Doctor:</td>\n",
       "      <td>6/18/85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sshe plans to move as of 7/8/71 In-Home Servic...</td>\n",
       "      <td>7/8/71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7 on 9/27/75 Audit C Score Current:</td>\n",
       "      <td>9/27/75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2/6/96 sleep studyPain Treatment Pain Level (N...</td>\n",
       "      <td>2/6/96</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text    Dates1\n",
       "0         03/25/93 Total time of visit (in minutes):  03/25/93\n",
       "1                       6/18/85 Primary Care Doctor:   6/18/85\n",
       "2  sshe plans to move as of 7/8/71 In-Home Servic...    7/8/71\n",
       "3                7 on 9/27/75 Audit C Score Current:   9/27/75\n",
       "4  2/6/96 sleep studyPain Treatment Pain Level (N...    2/6/96"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "125"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame['Dates1'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Dates1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>s The patient is a 44 year old married Caucasi...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>.10 Sep 2004 - Intake at EEC for IOP but did n...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>see above and APS eval of 26 May 1982 Social H...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>Tbooked for intake appointment at Sierra Vista...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>06 May 1972 SOS-10 Total Score:</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Text Dates1\n",
       "125  s The patient is a 44 year old married Caucasi...   None\n",
       "126  .10 Sep 2004 - Intake at EEC for IOP but did n...   None\n",
       "127  see above and APS eval of 26 May 1982 Social H...   None\n",
       "128  Tbooked for intake appointment at Sierra Vista...   None\n",
       "129                    06 May 1972 SOS-10 Total Score:   None"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame[125:200].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### extracting more dates after 124th row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extractor2(x):\n",
    "    dt2 = re.search(\"\\d{2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \\d{4}\", x)\n",
    "    if(dt2 != None):\n",
    "        return dt2.group(0)\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame['Dates2'] = frame['Text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame['Dates2']=frame['Dates2'].apply(extractor2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "69"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#frame[125:200]\n",
    "frame['Dates2'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Dates1</th>\n",
       "      <th>Dates2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>s 22 year old single Caucasian/Latino woman, u...</td>\n",
       "      <td>None</td>\n",
       "      <td>18 Jan 1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>April 11, 1990 CPT Code: 90791: No medical ser...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>MRI May 30, 2001 empty sella but no problems w...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>.Feb 18, 1994: made a phone call to Mom and Mo...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>Brother died February 18, 1981 Parental/Caregi...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>none; but currently has appt with new HJH PCP ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>.Came back to US on Jan 24 1986, saw Dr. Quack...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>July 26, 1978 Total time of visit (in minutes):</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>father was depressed inpatient at DFC December...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>May 15, 1989 SOS-10 Total Score:</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203</th>\n",
       "      <td>September 06, 1995 Total time of visit (in min...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>204</th>\n",
       "      <td>Mar. 10, 1976 CPT Code: 90791: No medical serv...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>.Got back to U.S. Jan 27, 1983.</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>Queen Hamilton in Bonita Springs courthouse.  ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>r August 12 2004 - diagnosed with Parkinson's ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>September 01, 2012 Age:</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209</th>\n",
       "      <td>July 25, 1983 Total time of visit (in minutes):</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Text Dates1       Dates2\n",
       "193  s 22 year old single Caucasian/Latino woman, u...   None  18 Jan 1995\n",
       "194  April 11, 1990 CPT Code: 90791: No medical ser...   None         None\n",
       "195  MRI May 30, 2001 empty sella but no problems w...   None         None\n",
       "196  .Feb 18, 1994: made a phone call to Mom and Mo...   None         None\n",
       "197  Brother died February 18, 1981 Parental/Caregi...   None         None\n",
       "198  none; but currently has appt with new HJH PCP ...   None         None\n",
       "199  .Came back to US on Jan 24 1986, saw Dr. Quack...   None         None\n",
       "200    July 26, 1978 Total time of visit (in minutes):   None         None\n",
       "201  father was depressed inpatient at DFC December...   None         None\n",
       "202                   May 15, 1989 SOS-10 Total Score:   None         None\n",
       "203  September 06, 1995 Total time of visit (in min...   None         None\n",
       "204  Mar. 10, 1976 CPT Code: 90791: No medical serv...   None         None\n",
       "205                    .Got back to U.S. Jan 27, 1983.   None         None\n",
       "206  Queen Hamilton in Bonita Springs courthouse.  ...   None         None\n",
       "207  r August 12 2004 - diagnosed with Parkinson's ...   None         None\n",
       "208                            September 01, 2012 Age:   None         None\n",
       "209    July 25, 1983 Total time of visit (in minutes):   None         None"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame[193:210]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### extracting after the 193rd row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extractor3(x):\n",
    "    dt3 = re.search(\"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \\d{2},* \\d{4}\", x)\n",
    "    if(dt3 != None):\n",
    "        return dt3.group(0)\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame['Dates3'] = frame['Text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame['Dates3']=frame['Dates3'].apply(extractor3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame['Dates3'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'t Allergies Sulfa (Sulfonamide Antibiotics) - Renal Toxicity : pt developed acute interstitial nephritis on Bactrim (June 2011)'"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame['Text'][229]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Dates1</th>\n",
       "      <th>Dates2</th>\n",
       "      <th>Dates3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>227</th>\n",
       "      <td>September. 15, 2011 Total time of visit (in mi...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>228</th>\n",
       "      <td>s 20 yo M carries dx of BPAD, presents for psy...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>t Allergies Sulfa (Sulfonamide Antibiotics) - ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>B/R Walnut Ridge. Raised with sister and paren...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>50 yo DWF with a history of alcohol use disord...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Text Dates1 Dates2 Dates3\n",
       "227  September. 15, 2011 Total time of visit (in mi...   None   None   None\n",
       "228  s 20 yo M carries dx of BPAD, presents for psy...   None   None   None\n",
       "229  t Allergies Sulfa (Sulfonamide Antibiotics) - ...   None   None   None\n",
       "230  B/R Walnut Ridge. Raised with sister and paren...   None   None   None\n",
       "231  50 yo DWF with a history of alcohol use disord...   None   None   None"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame[227:250].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### after 226th row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extractor4(x):\n",
    "    dt4 = re.search(\"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \\d{4}\", x)\n",
    "    if(dt4 != None):\n",
    "        return dt4.group(0)\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame['Dates4']=frame['Text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame['Dates4'] = frame['Dates4'].apply(extractor4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "171"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame['Dates4'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Dates1</th>\n",
       "      <th>Dates2</th>\n",
       "      <th>Dates3</th>\n",
       "      <th>Dates4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>328</th>\n",
       "      <td>s Pt reports long Hx of drug addiction. PLEASE...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>329</th>\n",
       "      <td>CLLC and DCF - ended in March, 2000. Psychiatr...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>330</th>\n",
       "      <td>\"-worked with Uma Dewitt (ED specialist) x 10 ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>April 1988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>331</th>\n",
       "      <td>Prozac- highest dose 40 mg; currently on 20 mg...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>December 1993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332</th>\n",
       "      <td>)last use in June 1974.Longitudinal Alcohol us...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>June 1974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333</th>\n",
       "      <td>sNovember 1997 - suicidal ideation - HHR</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>November 1997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>334</th>\n",
       "      <td>sAppendectomy in July 1986, shortly after deli...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>July 1986</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>335</th>\n",
       "      <td>Open appendectomy, as a child. 2.  Laparoscopi...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>February 1973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336</th>\n",
       "      <td>sOcella 0.03-3MG TABLET Take 1 PO QD, next pap...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>March 1978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>337</th>\n",
       "      <td>sWas on lexapro and strattera with good result...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Dec 2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>338</th>\n",
       "      <td>2Last time received treatment was from a life ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>339</th>\n",
       "      <td>Currently on Zoloft, 50 mg, Dicolenac, 100 mg,...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>340</th>\n",
       "      <td>Future-oriented; recent increase in hope; in s...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>May 1980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>341</th>\n",
       "      <td>.Nov 2007- Evaluation and treatment at CH ED/A...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Nov 2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>342</th>\n",
       "      <td>5Bethania Hospital March 1976 Hx of Outpatient...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>March 1976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>343</th>\n",
       "      <td>6/1998 Primary Care Doctor:</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344</th>\n",
       "      <td>s 52 y/o MWM h/o chronic depression, anxiety, ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>345</th>\n",
       "      <td>10/1973 Hx of Brain Injury: Yes</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>346</th>\n",
       "      <td>9/2005 Primary Care Doctor:</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>347</th>\n",
       "      <td>s 03/1980 Positive PPD: treated with INH for 6...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>348</th>\n",
       "      <td>12/2005 Family Psych History: Family History o...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>349</th>\n",
       "      <td>5/1987 Primary Care Doctor:</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Text Dates1 Dates2 Dates3  \\\n",
       "328  s Pt reports long Hx of drug addiction. PLEASE...   None   None   None   \n",
       "329  CLLC and DCF - ended in March, 2000. Psychiatr...   None   None   None   \n",
       "330  \"-worked with Uma Dewitt (ED specialist) x 10 ...   None   None   None   \n",
       "331  Prozac- highest dose 40 mg; currently on 20 mg...   None   None   None   \n",
       "332  )last use in June 1974.Longitudinal Alcohol us...   None   None   None   \n",
       "333           sNovember 1997 - suicidal ideation - HHR   None   None   None   \n",
       "334  sAppendectomy in July 1986, shortly after deli...   None   None   None   \n",
       "335  Open appendectomy, as a child. 2.  Laparoscopi...   None   None   None   \n",
       "336  sOcella 0.03-3MG TABLET Take 1 PO QD, next pap...   None   None   None   \n",
       "337  sWas on lexapro and strattera with good result...   None   None   None   \n",
       "338  2Last time received treatment was from a life ...   None   None   None   \n",
       "339  Currently on Zoloft, 50 mg, Dicolenac, 100 mg,...   None   None   None   \n",
       "340  Future-oriented; recent increase in hope; in s...   None   None   None   \n",
       "341  .Nov 2007- Evaluation and treatment at CH ED/A...   None   None   None   \n",
       "342  5Bethania Hospital March 1976 Hx of Outpatient...   None   None   None   \n",
       "343                        6/1998 Primary Care Doctor:   None   None   None   \n",
       "344  s 52 y/o MWM h/o chronic depression, anxiety, ...   None   None   None   \n",
       "345                    10/1973 Hx of Brain Injury: Yes   None   None   None   \n",
       "346                        9/2005 Primary Care Doctor:   None   None   None   \n",
       "347  s 03/1980 Positive PPD: treated with INH for 6...   None   None   None   \n",
       "348  12/2005 Family Psych History: Family History o...   None   None   None   \n",
       "349                        5/1987 Primary Care Doctor:   None   None   None   \n",
       "\n",
       "            Dates4  \n",
       "328           None  \n",
       "329           None  \n",
       "330     April 1988  \n",
       "331  December 1993  \n",
       "332      June 1974  \n",
       "333  November 1997  \n",
       "334      July 1986  \n",
       "335  February 1973  \n",
       "336     March 1978  \n",
       "337       Dec 2007  \n",
       "338           None  \n",
       "339           None  \n",
       "340       May 1980  \n",
       "341       Nov 2007  \n",
       "342     March 1976  \n",
       "343           None  \n",
       "344           None  \n",
       "345           None  \n",
       "346           None  \n",
       "347           None  \n",
       "348           None  \n",
       "349           None  "
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame[328:350]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### few more dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extractor5(x):\n",
    "    dt5 = re.search(\"\\d{1,2}[/-]\\d{4}\", x)\n",
    "    if(dt5 != None):\n",
    "        return dt5.group(0)\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame['Dates5'] = frame['Text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame['Dates5'] = frame['Dates5'].apply(extractor5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "#frame['Text'][456]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### extractor 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extractor6(x):\n",
    "    dt6 = re.search(\"\\d{4}\", x)\n",
    "    if(dt6 != None):\n",
    "        return dt6.group(0)\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame['Dates6'] = frame['Text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame['Dates6'] = frame['Dates6'].apply(extractor6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "455    1984\n",
       "456    2000\n",
       "457    2001\n",
       "458    1982\n",
       "459    1998\n",
       "Name: Dates6, dtype: object"
      ]
     },
     "execution_count": 223,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame['Dates6'][455:460]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(500, 7)"
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Dates1</th>\n",
       "      <th>Dates2</th>\n",
       "      <th>Dates3</th>\n",
       "      <th>Dates4</th>\n",
       "      <th>Dates5</th>\n",
       "      <th>Dates6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>460</th>\n",
       "      <td>1 Ex-smoker : quit 2012</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>461</th>\n",
       "      <td>. Age 16, 1991, frontal impact. out for two we...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462</th>\n",
       "      <td>sLexapro (1988-now): Good response (anxiety)</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>463</th>\n",
       "      <td>s      25 year old engaged to be married Optic...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>464</th>\n",
       "      <td>8Complications from brain hemmorage in 2016 Ax...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>465</th>\n",
       "      <td>.Age, 19, 1976, playing football, frontal impa...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>466</th>\n",
       "      <td>s1981  Swedish-American Hospital</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>467</th>\n",
       "      <td>aS/P suicide attempt 2011 Hx of Outpatient Tre...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>468</th>\n",
       "      <td>Patient has a history of suicidal ideation wit...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>469</th>\n",
       "      <td>Born and raised in Fowlerville, IN.  Parents d...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>470</th>\n",
       "      <td>y1983 Clinic Hospital, first hospitalization, ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>471</th>\n",
       "      <td>tProblems Urinary incontinence : mild urge inc...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>472</th>\n",
       "      <td>.2010 - wife; nightmares and angry outbursts; ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>473</th>\n",
       "      <td>shx of TBI (1975) ISO MVA.Medical History:</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>474</th>\n",
       "      <td>sPatient reported losing three friends that pa...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>475</th>\n",
       "      <td>TSH okay in 2015 Prior EKG:</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>476</th>\n",
       "      <td>1989 Family Psych History: Family History of S...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>477</th>\n",
       "      <td>oEnjoys animals, had a dog x 14 yrs who died i...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>478</th>\n",
       "      <td>eHistory of small right parietal subgaleal hem...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>479</th>\n",
       "      <td>sIn KEP Psychiatryfor therapy and medications ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480</th>\n",
       "      <td>1. Esophageal cancer, dx: 2013, on FOLFOX with...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>481</th>\n",
       "      <td>y1974 (all)</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>482</th>\n",
       "      <td>h/o restraining order by sister/mother in 1990...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>483</th>\n",
       "      <td>sTexas Medical Center; Oklahoma for 2 weeks; 1995</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>484</th>\n",
       "      <td>Death of former partner in 2004 by overdose as...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485</th>\n",
       "      <td>Was \"average\" student.  \"I didn't have too man...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>486</th>\n",
       "      <td>Contemplating jumping off building - 1973 - di...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>487</th>\n",
       "      <td>appendectomy s/p delivery 1992 Prior relevant ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>488</th>\n",
       "      <td>tProblems renal cell cancer : s/p nephrectomy ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489</th>\n",
       "      <td>ran own business for 35 years, sold in 1985</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490</th>\n",
       "      <td>Lab: B12 969 2007</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>)and 8mo in 2009</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>492</th>\n",
       "      <td>.Moved to USA in 1986. Suffered from malnutrit...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1986</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>493</th>\n",
       "      <td>r1978</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>494</th>\n",
       "      <td>. Went to Emerson, in Newfane Alaska. Started ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>1979 Family Psych History: Family History of S...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>therapist and friend died in ~2006 Parental/Ca...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>2008 partial thyroidectomy</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>sPt describes a history of sexual abuse as a c...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>. In 1980, patient was living in Naples and de...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1980</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Text Dates1 Dates2 Dates3  \\\n",
       "460                            1 Ex-smoker : quit 2012   None   None   None   \n",
       "461  . Age 16, 1991, frontal impact. out for two we...   None   None   None   \n",
       "462       sLexapro (1988-now): Good response (anxiety)   None   None   None   \n",
       "463  s      25 year old engaged to be married Optic...   None   None   None   \n",
       "464  8Complications from brain hemmorage in 2016 Ax...   None   None   None   \n",
       "465  .Age, 19, 1976, playing football, frontal impa...   None   None   None   \n",
       "466                   s1981  Swedish-American Hospital   None   None   None   \n",
       "467  aS/P suicide attempt 2011 Hx of Outpatient Tre...   None   None   None   \n",
       "468  Patient has a history of suicidal ideation wit...   None   None   None   \n",
       "469  Born and raised in Fowlerville, IN.  Parents d...   None   None   None   \n",
       "470  y1983 Clinic Hospital, first hospitalization, ...   None   None   None   \n",
       "471  tProblems Urinary incontinence : mild urge inc...   None   None   None   \n",
       "472  .2010 - wife; nightmares and angry outbursts; ...   None   None   None   \n",
       "473         shx of TBI (1975) ISO MVA.Medical History:   None   None   None   \n",
       "474  sPatient reported losing three friends that pa...   None   None   None   \n",
       "475                        TSH okay in 2015 Prior EKG:   None   None   None   \n",
       "476  1989 Family Psych History: Family History of S...   None   None   None   \n",
       "477  oEnjoys animals, had a dog x 14 yrs who died i...   None   None   None   \n",
       "478  eHistory of small right parietal subgaleal hem...   None   None   None   \n",
       "479  sIn KEP Psychiatryfor therapy and medications ...   None   None   None   \n",
       "480  1. Esophageal cancer, dx: 2013, on FOLFOX with...   None   None   None   \n",
       "481                                        y1974 (all)   None   None   None   \n",
       "482  h/o restraining order by sister/mother in 1990...   None   None   None   \n",
       "483  sTexas Medical Center; Oklahoma for 2 weeks; 1995   None   None   None   \n",
       "484  Death of former partner in 2004 by overdose as...   None   None   None   \n",
       "485  Was \"average\" student.  \"I didn't have too man...   None   None   None   \n",
       "486  Contemplating jumping off building - 1973 - di...   None   None   None   \n",
       "487  appendectomy s/p delivery 1992 Prior relevant ...   None   None   None   \n",
       "488  tProblems renal cell cancer : s/p nephrectomy ...   None   None   None   \n",
       "489        ran own business for 35 years, sold in 1985   None   None   None   \n",
       "490                                  Lab: B12 969 2007   None   None   None   \n",
       "491                                   )and 8mo in 2009   None   None   None   \n",
       "492  .Moved to USA in 1986. Suffered from malnutrit...   None   None   None   \n",
       "493                                              r1978   None   None   None   \n",
       "494  . Went to Emerson, in Newfane Alaska. Started ...   None   None   None   \n",
       "495  1979 Family Psych History: Family History of S...   None   None   None   \n",
       "496  therapist and friend died in ~2006 Parental/Ca...   None   None   None   \n",
       "497                         2008 partial thyroidectomy   None   None   None   \n",
       "498  sPt describes a history of sexual abuse as a c...   None   None   None   \n",
       "499  . In 1980, patient was living in Naples and de...   None   None   None   \n",
       "\n",
       "    Dates4 Dates5 Dates6  \n",
       "460   None   None   2012  \n",
       "461   None   None   1991  \n",
       "462   None   None   1988  \n",
       "463   None   None   2014  \n",
       "464   None   None   2016  \n",
       "465   None   None   1976  \n",
       "466   None   None   1981  \n",
       "467   None   None   2011  \n",
       "468   None   None   1997  \n",
       "469   None   None   2003  \n",
       "470   None   None   1983  \n",
       "471   None   None   1999  \n",
       "472   None   None   2010  \n",
       "473   None   None   1975  \n",
       "474   None   None   1972  \n",
       "475   None   None   2015  \n",
       "476   None   None   1989  \n",
       "477   None   None   1994  \n",
       "478   None   None   1993  \n",
       "479   None   None   1996  \n",
       "480   None   None   2013  \n",
       "481   None   None   1974  \n",
       "482   None   None   1990  \n",
       "483   None   None   1995  \n",
       "484   None   None   2004  \n",
       "485   None   None   1987  \n",
       "486   None   None   1973  \n",
       "487   None   None   1992  \n",
       "488   None   None   1977  \n",
       "489   None   None   1985  \n",
       "490   None   None   2007  \n",
       "491   None   None   2009  \n",
       "492   None   None   1986  \n",
       "493   None   None   1978  \n",
       "494   None   None   2002  \n",
       "495   None   None   1979  \n",
       "496   None   None   2006  \n",
       "497   None   None   2008  \n",
       "498   None   None   2005  \n",
       "499   None   None   1980  "
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame[460:500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Text', 'Dates1', 'Dates2', 'Dates3', 'Dates4', 'Dates5', 'Dates6'], dtype='object')"
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [],
   "source": [
    "#frame['Date'] = frame[['Dates1', 'Dates2', 'Dates3', 'Dates4', 'Dates5', 'Dates6']].apply(lambda x: ''.join(x), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concatenating all columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_frame = frame.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Dates1</th>\n",
       "      <th>Dates2</th>\n",
       "      <th>Dates3</th>\n",
       "      <th>Dates4</th>\n",
       "      <th>Dates5</th>\n",
       "      <th>Dates6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>03/25/93 Total time of visit (in minutes):</td>\n",
       "      <td>03/25/93</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6/18/85 Primary Care Doctor:</td>\n",
       "      <td>6/18/85</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sshe plans to move as of 7/8/71 In-Home Servic...</td>\n",
       "      <td>7/8/71</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7 on 9/27/75 Audit C Score Current:</td>\n",
       "      <td>9/27/75</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2/6/96 sleep studyPain Treatment Pain Level (N...</td>\n",
       "      <td>2/6/96</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text    Dates1 Dates2 Dates3  \\\n",
       "0         03/25/93 Total time of visit (in minutes):  03/25/93   None   None   \n",
       "1                       6/18/85 Primary Care Doctor:   6/18/85   None   None   \n",
       "2  sshe plans to move as of 7/8/71 In-Home Servic...    7/8/71   None   None   \n",
       "3                7 on 9/27/75 Audit C Score Current:   9/27/75   None   None   \n",
       "4  2/6/96 sleep studyPain Treatment Pain Level (N...    2/6/96   None   None   \n",
       "\n",
       "  Dates4 Dates5 Dates6  \n",
       "0   None   None   None  \n",
       "1   None   None   None  \n",
       "2   None   None   None  \n",
       "3   None   None   None  \n",
       "4   None   None   None  "
      ]
     },
     "execution_count": 256,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_frame.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "sep_date = pd.concat([new_frame['Dates1'], new_frame['Dates2'], new_frame['Dates3'], new_frame['Dates4'], new_frame['Dates5'], new_frame['Dates6']], axis=1, join='outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dates1    125\n",
       "Dates2     69\n",
       "Dates3     30\n",
       "Dates4    171\n",
       "Dates5    138\n",
       "Dates6    428\n",
       "dtype: int64"
      ]
     },
     "execution_count": 258,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sep_date.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [],
   "source": [
    "#new_frame['Date'] = ('' if new_frame['Dates1'].empty else new_frame['Dates1'].map(str)) + ('' if new_frame['Dates2'].empty else new_frame['Dates2'].map(str))+ ('' if new_frame['Dates3'].empty else new_frame['Dates3'].map(str))+ ('' if new_frame['Dates4'].empty else new_frame['Dates4'].map(str))+ ('' if new_frame['Dates5'].empty else new_frame['Dates5'].map(str))+ ('' if new_frame['Dates6'].empty else new_frame['Dates6'].map(str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "495    NoneNoneNoneNoneNone1979\n",
       "496    NoneNoneNoneNoneNone2006\n",
       "497    NoneNoneNoneNoneNone2008\n",
       "498    NoneNoneNoneNoneNone2005\n",
       "499    NoneNoneNoneNoneNone1980\n",
       "Name: Date, dtype: object"
      ]
     },
     "execution_count": 240,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#new_frame['Date'].tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Text', 'Dates1', 'Dates2', 'Dates3', 'Dates4', 'Dates5', 'Dates6'], dtype='object')"
      ]
     },
     "execution_count": 259,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_frame.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_frame['New'] = new_frame['Dates1'].fillna(new_frame['Dates2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_frame['New'][193:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Text', 'Dates1', 'Dates2', 'Dates3', 'Dates4', 'Dates5', 'Dates6',\n",
       "       'New'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 285,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_frame.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_frame['New1'] = new_frame['New'].fillna(new_frame['Dates3'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_frame['New1'][226:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_frame['New2'] = new_frame['New1'].fillna(new_frame['Dates4'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_frame['New3'] = new_frame['New2'].fillna(new_frame['Dates5'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_frame['New4'] = new_frame['New3'].fillna(new_frame['Dates6'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_frame['New4']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Text', 'Dates1', 'Dates2', 'Dates3', 'Dates4', 'Dates5', 'Dates6',\n",
       "       'New', 'New1', 'New2', 'New3', 'New4'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 301,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_frame.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Dates1</th>\n",
       "      <th>Dates2</th>\n",
       "      <th>Dates3</th>\n",
       "      <th>Dates4</th>\n",
       "      <th>Dates5</th>\n",
       "      <th>Dates6</th>\n",
       "      <th>New</th>\n",
       "      <th>New1</th>\n",
       "      <th>New2</th>\n",
       "      <th>New3</th>\n",
       "      <th>New4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>1979 Family Psych History: Family History of S...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1979</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>therapist and friend died in ~2006 Parental/Ca...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2006</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>2008 partial thyroidectomy</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2008</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>sPt describes a history of sexual abuse as a c...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2005</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>. In 1980, patient was living in Naples and de...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1980</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1980</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Text Dates1 Dates2 Dates3  \\\n",
       "495  1979 Family Psych History: Family History of S...   None   None   None   \n",
       "496  therapist and friend died in ~2006 Parental/Ca...   None   None   None   \n",
       "497                         2008 partial thyroidectomy   None   None   None   \n",
       "498  sPt describes a history of sexual abuse as a c...   None   None   None   \n",
       "499  . In 1980, patient was living in Naples and de...   None   None   None   \n",
       "\n",
       "    Dates4 Dates5 Dates6   New  New1  New2  New3  New4  \n",
       "495   None   None   1979  None  None  None  None  1979  \n",
       "496   None   None   2006  None  None  None  None  2006  \n",
       "497   None   None   2008  None  None  None  None  2008  \n",
       "498   None   None   2005  None  None  None  None  2005  \n",
       "499   None   None   1980  None  None  None  None  1980  "
      ]
     },
     "execution_count": 302,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_frame.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_frame.drop(['New', 'New1', 'New2', 'New3'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Text', 'Dates1', 'Dates2', 'Dates3', 'Dates4', 'Dates5', 'Dates6',\n",
       "       'New4'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 304,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_frame.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_frame.rename(columns={'New4': 'Date'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Text', 'Dates1', 'Dates2', 'Dates3', 'Dates4', 'Dates5', 'Dates6',\n",
       "       'Date'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 307,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_frame.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_frame.drop(['Dates1', 'Dates2', 'Dates3', 'Dates4', 'Dates5', 'Dates6'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Text', 'Date'], dtype='object')"
      ]
     },
     "execution_count": 309,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_frame.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>03/25/93 Total time of visit (in minutes):</td>\n",
       "      <td>03/25/93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6/18/85 Primary Care Doctor:</td>\n",
       "      <td>6/18/85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sshe plans to move as of 7/8/71 In-Home Servic...</td>\n",
       "      <td>7/8/71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7 on 9/27/75 Audit C Score Current:</td>\n",
       "      <td>9/27/75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2/6/96 sleep studyPain Treatment Pain Level (N...</td>\n",
       "      <td>2/6/96</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text      Date\n",
       "0         03/25/93 Total time of visit (in minutes):  03/25/93\n",
       "1                       6/18/85 Primary Care Doctor:   6/18/85\n",
       "2  sshe plans to move as of 7/8/71 In-Home Servic...    7/8/71\n",
       "3                7 on 9/27/75 Audit C Score Current:   9/27/75\n",
       "4  2/6/96 sleep studyPain Treatment Pain Level (N...    2/6/96"
      ]
     },
     "execution_count": 310,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_frame.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>1979 Family Psych History: Family History of S...</td>\n",
       "      <td>1979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>therapist and friend died in ~2006 Parental/Ca...</td>\n",
       "      <td>2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>2008 partial thyroidectomy</td>\n",
       "      <td>2008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>sPt describes a history of sexual abuse as a c...</td>\n",
       "      <td>2005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>. In 1980, patient was living in Naples and de...</td>\n",
       "      <td>1980</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Text  Date\n",
       "495  1979 Family Psych History: Family History of S...  1979\n",
       "496  therapist and friend died in ~2006 Parental/Ca...  2006\n",
       "497                         2008 partial thyroidectomy  2008\n",
       "498  sPt describes a history of sexual abuse as a c...  2005\n",
       "499  . In 1980, patient was living in Naples and de...  1980"
      ]
     },
     "execution_count": 311,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_frame.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}