2_TIGER_qT1_DataClean.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from os.path import expanduser, join\n",
    "from glob import glob\n",
    "import pandas as pd\n",
    "from datetime import date\n",
    "from numpy import where\n",
    "import numpy as np\n",
    "today = str(date.today())\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.width', 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Set paths\n",
    "userhome='/Users/lucindasisk/Documents/qT1_10.2020Analysis'\n",
    "home = '/Volumes/group/proc/TIGERanalysis/qT1'\n",
    "dest = home + '/Results/Raw_VoxelData2021'\n",
    "datadir = userhome +'/data/2_Extracted_Stats'\n",
    "flist = glob(datadir + '/*/ROI_voxel*GlobalR1*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Read in and reorient raw output from FSLMeanTS "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def fix_files(file1):\n",
    "    inf = file1.replace(datadir, '').replace('/ROI_voxel-contrasts', '').replace('.txt', '')\n",
    "    print(inf)\n",
    "    file1 = pd.read_csv(file1, header=None, sep = '  ', engine='python')\n",
    "    file = file1.transpose().rename(columns = {0:'L/R', 1:'A/P', 2:'S/I', 3:'Voxel Contrast'})\n",
    "    outfile = file.to_csv(dest + '/{}.csv'.format(inf), index = False)\n",
    "\n",
    "for x in range(79,len(flist)):\n",
    "    file1 = flist[x]\n",
    "    print('Fixing file {} of {}'.format(x, len(flist)))\n",
    "    fix_files(file1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lamyglist = glob(dest + '/*_LeftAmygdala.csv')\n",
    "# lcaudlist = glob(dest + '/*_LeftCaudate.csv')\n",
    "# lhipplist = glob(dest + '/*_LeftHippocampus.csv')\n",
    "# lNAcclist = glob(dest + '/*_LeftNAcc.csv')\n",
    "# lputlist = glob(dest + '/*_LeftPutamen.csv')\n",
    "# ramyglist = glob(dest + '/*_RightAmygdala.csv')\n",
    "# rcaudlist = glob(dest + '/*_RightCaudate.csv')\n",
    "# rhipplist = glob(dest + '/*_RightHippocampus.csv')\n",
    "# rNAcclist = glob(dest + '/*_RightNAcc.csv')\n",
    "# rputlist = glob(dest + '/*_RightPutamen.csv')\n",
    "\n",
    "# lcclist = glob(dest + '/*_LeftCC.csv')\n",
    "# rcclist = glob(dest + '/*_RightCC.csv')\n",
    "# # lcstlist = glob(dest + '/*_LeftCST.csv')\n",
    "# # rcstlist = glob(dest + '/*_RightCST.csv')\n",
    "# lunclist = glob(dest + '/*_LeftUF.csv')\n",
    "# runclist = glob(dest + '/*_RightUF.csv')\n",
    "# ccmajlist = glob(dest + '/*_CCMaj.csv')\n",
    "# ccminlist = glob(dest + '/*_CCMin.csv')\n",
    "global_r1list = glob(dest + '/*_GlobalR1.csv')\n",
    "llist= [global_r1list]  #lcclist, rcclist, lunclist, runclist, ccmajlist, ccminlist] #lcstlist, rcstlist\n",
    "\n",
    "# llist = [lamyglist, lcaudlist, lhipplist, lNAcclist, lputlist,\n",
    "#         ramyglist, rcaudlist, rhipplist, rNAcclist, rputlist]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_mean(tractlist):\n",
    "    mean_prop = []\n",
    "    for file in tractlist:\n",
    "        v1 = pd.read_csv(file, header=0)\n",
    "        sub = file.replace(dest+'/','').replace('.csv', '') \n",
    "        numzs = len(v1[v1['Voxel Contrast'] == 0.0])\n",
    "        numvs = len(v1)\n",
    "        v1['Voxel Contrast'] = v1['Voxel Contrast'].replace(0.0, np.nan) #Replace 0's with NaNs in tract masks\n",
    "        assert len(v1[v1['Voxel Contrast'] == 0.0]) == 0\n",
    "        mean = np.nanmean(v1['Voxel Contrast'])\n",
    "        prop_0vox = numzs/numvs #Get proportion of overhanging voxels\n",
    "        v2 = [sub, mean, prop_0vox]\n",
    "        mean_prop.append(v2)\n",
    "    return mean_prop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get Global R1 data\n",
    "gr1means = compute_mean(global_r1list)\n",
    "gr1file = pd.DataFrame(gr1means, columns = ['Global_T1', 'Global_T1_VoxelConMean', 'Global_T1_zerovoxels'])\n",
    "assert len(gr1file) == len(flist)\n",
    "gr1id = []\n",
    "for x in gr1file['Global_T1']:\n",
    "    y = str(x).replace('_GlobalR1','')\n",
    "    gr1id.append(y)\n",
    "gr1file['ID'] = gr1id\n",
    "gr1file['Global_R1'] = 1/gr1file['Global_T1_VoxelConMean']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# gr1file.sort_values(by = 'Global_T1').to_csv('/Users/lucindasisk/Box/LS_Folders/SNAPLab/Projects/TIGER_qT1/Data/Global_R1_n{}_{}.csv'.format(len(gr1file), today))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extract Data for DTI Tracts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cingulum Cingulate\n",
    "\n",
    "lccmeans = compute_mean(lcclist)\n",
    "lccfile = pd.DataFrame(lccmeans, columns = ['Left_CC', 'LCC_VoxelConMean', 'LCC_overhang'])\n",
    "lccid = []\n",
    "for x in lccfile['Left_CC']:\n",
    "    y = str(x).replace('_LeftCC','')\n",
    "    lccid.append(y)\n",
    "lccfile['ID'] = lccid\n",
    "lccfile['LCC_R1'] = 1/lccfile['LCC_VoxelConMean']\n",
    "\n",
    "\n",
    "\n",
    "rccmeans = compute_mean(rcclist)\n",
    "rccfile = pd.DataFrame(rccmeans, columns = ['Right_CC', 'RCC_VoxelConMean', 'RCC_overhang'])\n",
    "rccid = []\n",
    "for x in rccfile['Right_CC']:\n",
    "    y = str(x).replace('_RightCC','')\n",
    "    rccid.append(y)\n",
    "rccfile['ID'] = rccid\n",
    "rccfile['RCC_R1'] = 1/rccfile['RCC_VoxelConMean']\n",
    "\n",
    "# # Corticospinal\n",
    "\n",
    "# means = []\n",
    "# for file in lcstlist:\n",
    "#     compute_mean(file)\n",
    "    \n",
    "# lcstfile = pd.DataFrame(means, columns = ['Left_CST', 'LCST_VoxelConMean'])\n",
    "# lcstid = []\n",
    "# for x in lcstfile['Left_CST']:\n",
    "#     y = str(x).replace('_LeftCST','')\n",
    "#     lcstid.append(y)\n",
    "# lcstfile['ID'] = lcstid\n",
    "# lcstfile['LCST_R1'] = 1/lcstfile['LCST_VoxelConMean']\n",
    "\n",
    "\n",
    "# means = []\n",
    "# for file in rcstlist:\n",
    "#     compute_mean(file)\n",
    "  \n",
    "# rcstfile = pd.DataFrame(means, columns = ['Right_CST', 'RCST_VoxelConMean'])\n",
    "# rcstid = []\n",
    "# for x in rcstfile['Right_CST']:\n",
    "#     y = str(x).replace('_RightCST','')\n",
    "#     rcstid.append(y)\n",
    "# rcstfile['ID'] = rcstid\n",
    "# rcstfile['RCST_R1'] = 1/rcstfile['RCST_VoxelConMean']\n",
    "    \n",
    "# Uncinate Fasciculus\n",
    "luncmeans = compute_mean(lunclist)\n",
    "luncfile = pd.DataFrame(luncmeans, columns = ['Left_Unc', 'LUF_VoxelConMean', 'LUF_overhang'])\n",
    "luncid = []\n",
    "for x in luncfile['Left_Unc']:\n",
    "    y = str(x).replace('_LeftUF','')\n",
    "    luncid.append(y)\n",
    "luncfile['ID'] = luncid\n",
    "luncfile['LUF_R1'] = 1/luncfile['LUF_VoxelConMean']\n",
    "\n",
    "\n",
    "runcmeans = compute_mean(runclist)\n",
    "runcfile = pd.DataFrame(runcmeans, columns = ['Right_Unc', 'RUF_VoxelConMean', 'RUF_overhang'])\n",
    "runcid = []\n",
    "for x in runcfile['Right_Unc']:\n",
    "    y = str(x).replace('_RightUF','')\n",
    "    runcid.append(y)\n",
    "runcfile['ID'] = runcid\n",
    "runcfile['RUF_R1'] = 1/runcfile['RUF_VoxelConMean']\n",
    "\n",
    "#Corpus Callosum Major/Minor\n",
    "\n",
    "ccmajmeans = compute_mean(ccmajlist)\n",
    "ccmajfile = pd.DataFrame(ccmajmeans, columns = ['CC_Maj', 'CCMaj_VoxelConMean', 'CCMaj_overhang'])\n",
    "ccmajid = []\n",
    "for x in ccmajfile['CC_Maj']:\n",
    "    y = str(x).replace('_CCMaj','')\n",
    "    ccmajid.append(y)\n",
    "ccmajfile['ID'] = ccmajid\n",
    "ccmajfile['CCMaj_R1'] = 1/ccmajfile['CCMaj_VoxelConMean']\n",
    "\n",
    "ccminmeans = compute_mean(ccminlist)\n",
    "ccminfile = pd.DataFrame(ccminmeans, columns = ['CC_Min', 'CCMin_VoxelConMean', 'CCMin_overhang'])\n",
    "ccminid = []\n",
    "for x in ccminfile['CC_Min']:\n",
    "    y = str(x).replace('_CCMin','')\n",
    "    ccminid.append(y)\n",
    "ccminfile['ID'] = ccminid\n",
    "ccminfile['CCMin_R1'] = 1/ccminfile['CCMin_VoxelConMean']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "m1 = pd.merge(lccfile, rccfile, how='outer', on='ID')\n",
    "m2 = pd.merge(m1, luncfile, how='outer', on='ID')\n",
    "m3 = pd.merge(m2, runcfile, how='outer', on='ID')\n",
    "m4 = pd.merge(m3, ccmajfile, how='outer', on='ID')\n",
    "m5 = pd.merge(m4, ccminfile, how='outer', on='ID')\n",
    "# m4 = pd.merge(m3, lcstfile, how='outer', on='ID')\n",
    "# m5 = pd.merge(m4, rcstfile, how='outer', on='ID')\n",
    "\n",
    "\n",
    "master1=m5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_fixed = ['ID', 'LCC_R1', 'LCC_overhang', 'RCC_R1', 'RCC_overhang', 'LUF_R1', 'LUF_overhang','RUF_R1', 'RUF_overhang',\n",
    "              'CCMaj_R1', 'CCMaj_overhang', 'CCMin_R1','CCMin_overhang']    \n",
    "master = master1[cols_fixed].sort_values(by='ID')\n",
    "master['ID'] = master['ID'].str.replace('159x-T3', '159-T3') #Replace rescan x with original ID for merging\n",
    "master = master.set_index('ID').drop('42-T1', axis=0).reset_index() #Drop TIGER 42-T1 due to severe autism\n",
    "\n",
    "today = str(date.today())\n",
    "master.to_csv(home+'/Results/qT1_output_'+today+'.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read in global R1 values\n",
    "t1vals = pd.read_csv(home + '/Results/Global_R1.txt', sep = '|', header = None)\n",
    "t1vals.columns = ['ID', 'T1']\n",
    "t1vals['Global_R1'] = 1/t1vals['T1']\n",
    "t1vals.to_csv(home+'/Results/globalT1_output_'+today+'.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extract Data from Subcortical Regions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Get data for left amygdala\n",
    "# means = []\n",
    "# for file in lamyglist:\n",
    "#     compute_mean(file)\n",
    "    \n",
    "# lamygfile = pd.DataFrame(means, columns = ['Left_Amyg', 'LAmyg_VoxelConMean'])\n",
    "# lamygid = []\n",
    "# for x in lamygfile['Left_Amyg']:\n",
    "#     y = str(x).replace('_LeftAmygdala','')\n",
    "#     lamygid.append(y)\n",
    "# lamygfile['ID'] = lamygid\n",
    "# lamygfile['LAmyg_R1'] = 1/lamygfile['LAmyg_VoxelConMean']\n",
    "\n",
    "# #Get data for right amygdala\n",
    "# means = []    \n",
    "# for file in ramyglist:\n",
    "#     compute_mean(file)\n",
    "\n",
    "    \n",
    "# ramygfile = pd.DataFrame(means, columns = ['Right_Amyg', 'RAmyg_VoxelConMean'])\n",
    "# ramygid = []\n",
    "# for x in ramygfile['Right_Amyg']:\n",
    "#     y = str(x).replace('_RightAmygdala','')\n",
    "#     ramygid.append(y)\n",
    "# ramygfile['ID'] = ramygid\n",
    "# ramygfile['RAmyg_R1'] = 1/ramygfile['RAmyg_VoxelConMean']\n",
    "\n",
    "# #Get data from left caudate\n",
    "# means = []\n",
    "# for file in lcaudlist:\n",
    "#     compute_mean(file)\n",
    "\n",
    "# lcaudfile = pd.DataFrame(means, columns = ['Left_Caudate', 'LCaudate_VoxelConMean'])\n",
    "# lcaudid = []\n",
    "# for x in lcaudfile['Left_Caudate']:\n",
    "#     y = str(x).replace('_LeftCaudate','')\n",
    "#     lcaudid.append(y)\n",
    "# lcaudfile['ID'] = lcaudid\n",
    "# lcaudfile['LCaudate_R1'] = 1/lcaudfile['LCaudate_VoxelConMean']\n",
    "\n",
    "# #Get data from right caudate\n",
    "# means = []\n",
    "# for file in rcaudlist:\n",
    "#     compute_mean(file)\n",
    "\n",
    "    \n",
    "# rcaudfile = pd.DataFrame(means, columns = ['Right_Caudate', 'RCaudate_VoxelConMean'])\n",
    "# rcaudid = []\n",
    "# for x in rcaudfile['Right_Caudate']:\n",
    "#     y = str(x).replace('_RightCaudate','')\n",
    "#     rcaudid.append(y)\n",
    "# rcaudfile['ID'] = rcaudid\n",
    "# rcaudfile['RCaudate_R1'] = 1/rcaudfile['RCaudate_VoxelConMean']\n",
    "\n",
    "# #Get data from left hippocampus\n",
    "# means = []\n",
    "# for file in lhipplist:\n",
    "#     compute_mean(file)\n",
    "    \n",
    "# lhippfile = pd.DataFrame(means, columns = ['Left_Hipp', 'LHipp_VoxelConMean'])\n",
    "# lhippid = []\n",
    "# for x in lhippfile['Left_Hipp']:\n",
    "#     y = str(x).replace('_LeftHippocampus','')\n",
    "#     lhippid.append(y)\n",
    "# lhippfile['ID'] = lhippid\n",
    "# lhippfile['LHipp_R1'] = 1/lhippfile['LHipp_VoxelConMean']\n",
    "\n",
    "# #Get data from right hippocampus\n",
    "# means = []\n",
    "# for file in rhipplist:\n",
    "#     compute_mean(file)\n",
    "    \n",
    "# rhippfile = pd.DataFrame(means, columns = ['Right_Hipp', 'RHipp_VoxelConMean'])\n",
    "# rhippid = []\n",
    "# for x in rhippfile['Right_Hipp']:\n",
    "#     y = str(x).replace('_RightHippocampus','')\n",
    "#     rhippid.append(y)\n",
    "# rhippfile['ID'] = rhippid\n",
    "# rhippfile['RHipp_R1'] = 1/rhippfile['RHipp_VoxelConMean']\n",
    "\n",
    "# #Get data from Left Putamen\n",
    "# means = []\n",
    "# for file in lputlist:\n",
    "#     compute_mean(file)\n",
    "    \n",
    "# lputfile = pd.DataFrame(means, columns = ['Left_Putamen', 'LPut_VoxelConMean'])\n",
    "# lputid = []\n",
    "# for x in lputfile['Left_Putamen']:\n",
    "#     y = str(x).replace('_LeftPutamen','')\n",
    "#     lputid.append(y)\n",
    "# lputfile['ID'] = lputid\n",
    "# lputfile['LPut_R1'] = 1/lputfile['LPut_VoxelConMean']\n",
    "\n",
    "# #Get data from Right Putamen\n",
    "# means = []\n",
    "# for file in rputlist:\n",
    "#     compute_mean(file)\n",
    "\n",
    "# rputfile = pd.DataFrame(means, columns = ['Right_Putamen', 'RPut_VoxelConMean'])\n",
    "# rputid = []\n",
    "# for x in rputfile['Right_Putamen']:\n",
    "#     y = str(x).replace('_RightPutamen','')\n",
    "#     rputid.append(y)\n",
    "# rputfile['ID'] = rputid\n",
    "# rputfile['RPut_R1'] = 1/rputfile['RPut_VoxelConMean']\n",
    "\n",
    "# #Get data from Left NAcc\n",
    "# means = []\n",
    "# for file in lNAcclist:\n",
    "#     compute_mean(file)\n",
    "    \n",
    "# lNAccfile = pd.DataFrame(means, columns = ['Left_NAcc', 'LNAcc_VoxelConMean'])\n",
    "# lNAccid = []\n",
    "# for x in lNAccfile['Left_NAcc']:\n",
    "#     y = str(x).replace('_LeftNAcc','')\n",
    "#     lNAccid.append(y)\n",
    "# lNAccfile['ID'] = lNAccid\n",
    "# lNAccfile['LNAcc_R1'] = 1/lNAccfile['LNAcc_VoxelConMean']\n",
    "\n",
    "# #Get data from Right NAcc\n",
    "# means = []\n",
    "# for file in rNAcclist:\n",
    "#     compute_mean(file)\n",
    "    \n",
    "# rNAccfile = pd.DataFrame(means, columns = ['Right_NAcc', 'RNAcc_VoxelConMean'])\n",
    "# rNAccid = []\n",
    "# for x in rNAccfile['Right_NAcc']:\n",
    "#     y = str(x).replace('_RightNAcc','')\n",
    "#     rNAccid.append(y)\n",
    "# rNAccfile['ID'] = rNAccid\n",
    "# rNAccfile['RNAcc_R1'] = 1/rNAccfile['RNAcc_VoxelConMean']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# m1 = pd.merge(lamygfile, ramygfile, how='outer', on='ID')\n",
    "# m2 = pd.merge(m1, lhippfile, how='outer', on='ID')\n",
    "# m3 = pd.merge(m2, rhippfile, how='outer', on='ID')\n",
    "# m4 = pd.merge(m3, lcaudfile, how='outer', on='ID')\n",
    "# m5 = pd.merge(m4, rcaudfile, how='outer', on='ID')\n",
    "# m6 = pd.merge(m5, lputfile, how='outer', on='ID')\n",
    "# m7 = pd.merge(m6, rputfile, how='outer', on='ID')\n",
    "# m8 = pd.merge(m7, lNAccfile, how='outer', on='ID')\n",
    "# m9 = pd.merge(m8, rNAccfile, how='outer', on='ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Drop non-R1 columns\n",
    "# #master1 = m3.drop(['Right_CC', 'Left_CC', 'Right_Unc', 'Left_Unc'], axis=1)\n",
    "# # master1 = m3.drop(['Left_Amyg', 'Right_Amyg', 'Left_Hipp', 'Right_Hipp',\n",
    "# #                   'Left_Putamen', 'Right_Putamen', 'Left_Caudate', 'Right_Caudate',\n",
    "# #                   'Left_NAcc', 'Right_NAcc'], axis=1)\n",
    "# # cols = master1.columns.tolist()\n",
    "# #Reorder columns (?)\\\n",
    "\n",
    "# master1=m9\n",
    "# cols_fixed = ['ID', 'LAmyg_VoxelConMean', 'LAmyg_R1', 'RAmyg_VoxelConMean', 'RAmyg_R1', \n",
    "#               'LHipp_VoxelConMean', 'LHipp_R1', 'RHipp_VoxelConMean', 'RHipp_R1',\n",
    "#               'LCaudate_VoxelConMean', 'LCaudate_R1', 'RCaudate_VoxelConMean', 'RCaudate_R1',\n",
    "#               'LPut_VoxelConMean', 'LPut_R1', 'RPut_VoxelConMean', 'RPut_R1',\n",
    "#               'LNAcc_VoxelConMean', 'LNAcc_R1', 'RNAcc_VoxelConMean', 'RNAcc_R1'\n",
    "#              ]    \n",
    "# master = master1[cols_fixed].sort_values(by='ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# master.to_csv(home+'/Results/qT1_output_'+today+'.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}