synthetic_algorithms_comparison/step1_encoding_sampling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "63e34e18",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cfc70b57",
   "metadata": {},
   "source": [
    "**Used stratified_sampling data, see stratified_sampling_data_sampling.ipynb for details of data cleaning and preprocessing**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d5b75a52",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/_b/q92mgy711lz66vr3h2x0jllh0000gt/T/ipykernel_48867/3240321201.py:3: DtypeWarning: Columns (13) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  breast_survival = pd.read_csv(data_path + data_file, index_col=0)\n"
     ]
    }
   ],
   "source": [
    "#path_to_data is the folder for all input and output of SynTwin, replace path_to_data to your path\n",
    "#breast_survival_10-15_sample.csv (cleaned data) from step1 is stored in subfolder SEER Data/Breast survival 2010 -2015\n",
    "breast_survival = pd.read_csv(\"path_to_data/SEER Data/Breast survival 2010 -2015/breast_survival_10-15_sample.csv\", index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7c84edae",
   "metadata": {},
   "outputs": [],
   "source": [
    "#cleaning for label encoding\n",
    "survival_months_dict = {'SurvivalMonths': {'0072':72, '0059':59, '0081':81, '0065':65, '0088':88,\n",
    "       '0098':98, '0107':107, '0080':80, '0037':37, '0062':62, '0052':52, '0116':116, '0013':13,\n",
    "       '0054':54, '0049':49, '0056':56, '0108':108, '0091':91, '0070':70, '0111':111, '0090':90,\n",
    "       '0101':101, '0112':112, '0051':51, '0100':100, '0053':53, '0095':95, '0073':73, '0103':103,\n",
    "       '0022':22, '0074':74, '0110':110, '0033':33, '0031':31, '0005':5, '0086':86, '0060':60,\n",
    "       '0096':96, '0050':50, '0010':10, '0089':89, '0035':35, '0109':109, '0066':66, '0087':87,\n",
    "       '0067':67, '0058':58, '0082':82, '0105':105, '0093':93, '0057':57, '0048':48, '0079':79,\n",
    "       '0071':71, '0099':99, '0041':41, '0075':75, '0102':102, '0097':97, '0114':114, '0094':94,\n",
    "       '0078':78, '0076':76, '0104':104, '0077':77, '0083':83, '0113':113, '0061':61, '0115':115,\n",
    "       '0007':7, '0064':64, '0119':119, '0118':118, '0008':8, '0092':92, '0084':84, '0028':28,\n",
    "       '0018':18, '0068':68, '0117':117, '0063':63, '0085':85, '0055':55, '0046':46, '0006':6,\n",
    "       '0042':42, '0032':32, '0069':69, '0001':1, '0106':106, '0034':34, '0004':4, '0015':15,\n",
    "       '0003':3, '0021':21, '0019':19, '0047':47, '0012':12, '0027':27, '0016':16, '0024':24,\n",
    "       '0043':43, '0014':14, '0020':20, '0000':0, '0038':38, '0040':40, '0002':2, '0039':39,\n",
    "       '0009':9, '0030':30, '0023':23, '0045':45, '0011':11, '0026':26, '0044':44, '0029':29,\n",
    "       '0017':17, '0025':25, '0036':36}}\n",
    "breast_survival = breast_survival.replace(survival_months_dict)\n",
    "breast_survival['SurvivalMonths'] = pd.to_numeric(breast_survival['SurvivalMonths'], errors='coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c9e78d42",
   "metadata": {},
   "outputs": [],
   "source": [
    "#label encoding (clean survival month first, don't encode patientid)\n",
    "patientid = breast_survival['PatientID']\n",
    "others = breast_survival.drop(columns=['PatientID'])\n",
    "\n",
    "encoding_dict = defaultdict(LabelEncoder)\n",
    "others_encoded = others.apply(lambda x: encoding_dict[x.name].fit_transform(x))\n",
    "\n",
    "breast_survival_encoded = pd.merge(others_encoded, patientid, left_index=True, right_index=True)\n",
    "labelencoding_survival = others.apply(lambda x: dict(zip(encoding_dict[x.name].classes_, range(len(encoding_dict[x.name].classes_)))))\n",
    "\n",
    "with open(data_path+'/breast_survival_encoding_dict.pickle', 'wb') as f:\n",
    "    pickle.dump(encoding_dict, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fe23959c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#sampling\n",
    "breast_survival_patientid_lists = []\n",
    "\n",
    "for seed in range(1000):\n",
    "    breast_survival_subset = breast_survival_encoded.sample(n = 1000, replace=True, random_state=seed)\n",
    "    breast_survival_subset.shape\n",
    "    breast_survival_patientid_lists.extend(breast_survival_subset['PatientID'].tolist())\n",
    "    breast_survival_subset.to_csv(\"step2_synthetic_algorithms/CLGP/data/breast_survival_1000samples_\"+str(seed)+\".csv\")\n",
    "    breast_survival_subset.to_csv(\"step2_synthetic_algorithms/MC-MedGAN/data/breast_survival_1000samples_\"+str(seed)+\".csv\")\n",
    "    breast_survival_subset.to_csv(\"step2_synthetic_algorithms/MPoM/data/breast_survival_1000samples_\"+str(seed)+\".csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "feb810cf",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}