spatial/spatial_features_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8026ccfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65afa676",
   "metadata": {},
   "source": [
    "## Save Centroid Location Info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da426d1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH_TO_PATCHES = '/deep/group/aihc-bootcamp-fall2021/lymphoma/processed/cellprofiler_out/stardist'\n",
    "TMA = 6\n",
    "\n",
    "PATCH_NUM = 4\n",
    "PATH_TO_DF = os.path.join(PATH_TO_PATCHES, f'tma_{TMA}/patches/patch_num={PATCH_NUM}/nuclei_with_patch_ids.csv')\n",
    "PATH_TO_OUTPUT_PATCHES = '/deep/group/aihc-bootcamp-fall2021/lymphoma/processed/spatial/stardist_cell_pos'\n",
    "patch_df = pd.read_csv(PATH_TO_DF)[['Location_Center_X', 'Location_Center_Y', 'group_id']]\n",
    "group_ids = set(list(patch_df['group_id']))\n",
    "grouped = patch_df.groupby(patch_df['group_id'])\n",
    "\n",
    "\n",
    "for group_id in group_ids:\n",
    "    patch_id = group_id.split('.')[0]\n",
    "    grouped.get_group(group_id).to_csv(os.path.join(PATH_TO_OUTPUT_PATCHES, f'tma_{TMA}/patches/patch_num={PATCH_NUM}/{patch_id}.csv'))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5f4363b",
   "metadata": {},
   "source": [
    "## Compute Ripley K Function Values in R"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d6d823f",
   "metadata": {},
   "source": [
    "## Concatenating R Output to Other H&E Features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc319881",
   "metadata": {},
   "source": [
    "### Read in all spatial info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "975d0f1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "patch_num = 4\n",
    "PATH1 = f'/deep/group/aihc-bootcamp-fall2021/lymphoma/processed/spatial/K/tma_1_patch_num={patch_num}.csv'\n",
    "PATH2 = f'/deep/group/aihc-bootcamp-fall2021/lymphoma/processed/spatial/K/tma_2_patch_num={patch_num}.csv'\n",
    "PATH3 = f'/deep/group/aihc-bootcamp-fall2021/lymphoma/processed/spatial/K/tma_3_patch_num={patch_num}.csv'\n",
    "PATH4 = f'/deep/group/aihc-bootcamp-fall2021/lymphoma/processed/spatial/K/tma_4_patch_num={patch_num}.csv'\n",
    "PATH5 = f'/deep/group/aihc-bootcamp-fall2021/lymphoma/processed/spatial/K/tma_5_patch_num={patch_num}.csv'\n",
    "PATH6 = f'/deep/group/aihc-bootcamp-fall2021/lymphoma/processed/spatial/K/tma_6_patch_num={patch_num}.csv'\n",
    "PATH8 = f'/deep/group/aihc-bootcamp-fall2021/lymphoma/processed/spatial/K/tma_8_patch_num={patch_num}.csv'\n",
    "\n",
    "df1 = pd.read_csv(PATH1, header=None,sep='\\n')\n",
    "df2 = pd.read_csv(PATH2, header=None,sep='\\n')\n",
    "df3 = pd.read_csv(PATH3, header=None,sep='\\n')\n",
    "df4 = pd.read_csv(PATH4, header=None,sep='\\n')\n",
    "df5 = pd.read_csv(PATH5, header=None,sep='\\n')\n",
    "df6 = pd.read_csv(PATH6, header=None,sep='\\n')\n",
    "df8 = pd.read_csv(PATH8, header=None,sep='\\n')\n",
    "df = pd.concat([df1, df2, df3, df4, df5, df6, df8])\n",
    "df.head()\n",
    "\n",
    "spat = df[0].str.split(',', expand=True)\n",
    "patch_id = [s.split(',')[-1].strip('/\"') for s in df[0]]\n",
    "spat_len = [len(s.split(',')) for s in df[0]]\n",
    "# spat.tail()\n",
    "\n",
    "# remove tma expressions from spat\n",
    "spat = spat.replace(regex=r'^.*tma.*$', value=0)\n",
    "spat['Image'] = patch_id\n",
    "\n",
    "# remove duplicates (if there are)\n",
    "spat = spat.drop_duplicates(subset=['Image'])\n",
    "\n",
    "# replace None with 0\n",
    "spat = spat.fillna(0)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "feb56365",
   "metadata": {},
   "source": [
    "### Observe Data Distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76e5023b",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(spat_len)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f351408",
   "metadata": {},
   "source": [
    "### Data Processing and Concatenation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c9f708a",
   "metadata": {},
   "outputs": [],
   "source": [
    "DIR = f'/deep/group/aihc-bootcamp-fall2021/lymphoma/processed/data_splits/custom_splits/cellprofiler/stardist_patch_num={patch_num}'\n",
    "PATH_TO_TRAIN = os.path.join(DIR, 'train.csv')\n",
    "PATH_TO_VAL = os.path.join(DIR, 'val.csv')\n",
    "PATH_TO_TEST = os.path.join(DIR, 'test.csv')\n",
    "\n",
    "PATH_TO_OUTPUT = f'/deep/group/aihc-bootcamp-fall2021/lymphoma/processed/data_splits/custom_splits/K/stardist_K_patch_num={patch_num}'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb6007de",
   "metadata": {},
   "source": [
    "#### Save Training Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86eb0682",
   "metadata": {},
   "outputs": [],
   "source": [
    "old_train = pd.read_csv(PATH_TO_TRAIN)\n",
    "new_train = old_train.set_index('Image').join(spat.set_index('Image'), how=\"inner\")\n",
    "print(len(old_train), len(new_train))\n",
    "old_features = list(old_train.columns)[1:]\n",
    "old_train.columns\n",
    "old_features\n",
    "# save the full data\n",
    "new_train.to_csv(os.path.join(PATH_TO_OUTPUT, 'full', 'train.csv'))\n",
    "new_train[old_features+[i for i in range(500)]].to_csv(os.path.join(PATH_TO_OUTPUT, 'd=500', 'train.csv'))\n",
    "# save d=500 data\n",
    "new_train[old_features+[i for i in range(500)]].to_csv(os.path.join(PATH_TO_OUTPUT, 'd=500', 'train.csv'))\n",
    "# save d=300 data\n",
    "new_train[old_features+[i for i in range(300)]].to_csv(os.path.join(PATH_TO_OUTPUT, 'd=300', 'train.csv'))\n",
    "# save d=250 data\n",
    "new_train[old_features+[i for i in range(350)]].to_csv(os.path.join(PATH_TO_OUTPUT, 'd=250', 'train.csv'))\n",
    "# save d=200 data\n",
    "new_train[old_features+[i for i in range(200)]].to_csv(os.path.join(PATH_TO_OUTPUT, 'd=200', 'train.csv'))\n",
    "# save d=100 data\n",
    "new_train[old_features+[i for i in range(100)]].to_csv(os.path.join(PATH_TO_OUTPUT, 'd=100', 'train.csv'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1736541f",
   "metadata": {},
   "source": [
    "#### Save Validation Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1bfe306",
   "metadata": {},
   "outputs": [],
   "source": [
    "old_val = pd.read_csv(PATH_TO_VAL)\n",
    "new_val = old_val.set_index('Image').join(spat.set_index('Image'), how=\"inner\")\n",
    "print(len(old_val), len(new_val))\n",
    "old_val['Image']\n",
    "old_features = list(old_val.columns)[1:]\n",
    "# save the full data\n",
    "new_val.to_csv(os.path.join(PATH_TO_OUTPUT, 'full', 'val.csv'))\n",
    "# save d=500 data\n",
    "new_val[old_features+[i for i in range(500)]].to_csv(os.path.join(PATH_TO_OUTPUT, 'd=500', 'val.csv'))\n",
    "# save d=300 data\n",
    "new_val[old_features+[i for i in range(300)]].to_csv(os.path.join(PATH_TO_OUTPUT, 'd=300', 'val.csv'))\n",
    "# save d=250 data\n",
    "new_val[old_features+[i for i in range(250)]].to_csv(os.path.join(PATH_TO_OUTPUT, 'd=250', 'val.csv'))\n",
    "# save d=200 data\n",
    "new_val[old_features+[i for i in range(200)]].to_csv(os.path.join(PATH_TO_OUTPUT, 'd=200', 'val.csv'))\n",
    "# save d=100 data\n",
    "new_val[old_features+[i for i in range(100)]].to_csv(os.path.join(PATH_TO_OUTPUT, 'd=100', 'val.csv'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f4e2c7e",
   "metadata": {},
   "source": [
    "#### Save Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8eceda4",
   "metadata": {},
   "outputs": [],
   "source": [
    "old_test = pd.read_csv(PATH_TO_TEST)\n",
    "new_test = old_test.set_index('Image').join(spat.set_index('Image'), how=\"inner\")\n",
    "print(len(old_test), len(new_test))\n",
    "old_features = list(old_test.columns)\n",
    "old_features\n",
    "# save the full data\n",
    "new_test.to_csv(os.path.join(PATH_TO_OUTPUT, 'full', 'test.csv'))\n",
    "# save d=500 data\n",
    "new_test[old_features+[i for i in range(500)]].to_csv(os.path.join(PATH_TO_OUTPUT, 'd=500', 'test.csv'))\n",
    "# save d=300 data\n",
    "new_test[old_features+[i for i in range(300)]].to_csv(os.path.join(PATH_TO_OUTPUT, 'd=300', 'test.csv'))\n",
    "# save d=250 data\n",
    "new_test[old_features+[i for i in range(250)]].to_csv(os.path.join(PATH_TO_OUTPUT, 'd=250', 'test.csv'))\n",
    "# save d=200 data\n",
    "new_test[old_features+[i for i in range(200)]].to_csv(os.path.join(PATH_TO_OUTPUT, 'd=200', 'test.csv'))\\\n",
    "# save d=100 data\n",
    "new_test[old_features+[i for i in range(100)]].to_csv(os.path.join(PATH_TO_OUTPUT, 'd=100', 'test.csv'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}