debug.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython import get_ipython"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No traceback available to show.\n"
     ]
    }
   ],
   "source": [
    "get_ipython().run_line_magic('tb', '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py\n",
      "['/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py', '@preproc.args']\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import argparse\n",
    "from pathlib import Path\n",
    "#Configuration file\n",
    "config_file = 'preproc.args'\n",
    "config_arg = '@'+config_file\n",
    "print(sys.argv[0])\n",
    "if config_arg not in sys.argv:\n",
    "    primary = sys.argv[0]\n",
    "    sys.argv = [primary,config_arg] \n",
    "print(sys.argv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/externals/joblib/__init__.py:15: DeprecationWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
      "  warnings.warn(msg, category=DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "@preproc.args\n",
      "argv type:  <class 'list'>\n",
      "argv:  ['/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py', '@preproc.args']\n",
      "{'CPU': 4,\n",
      " 'all_modalities': ['MR1'],\n",
      " 'image_format': 'NIFTI',\n",
      " 'image_masks': ['MASK'],\n",
      " 'image_shape': (256, 256, None),\n",
      " 'input_images': '/home/jovyan/data',\n",
      " 'input_type': 'Image',\n",
      " 'n_channels': 1,\n",
      " 'normalize': 1,\n",
      " 'output_dir': 'output/',\n",
      " 'output_file': 'output/data_test.h5',\n",
      " 'output_images': 0,\n",
      " 'output_imstats': 1,\n",
      " 'overwrite': 1,\n",
      " 'patch_shape': None,\n",
      " 'problem_type': 'Segmentation',\n",
      " 'training_modalities': ['MR1']}\n"
     ]
    }
   ],
   "source": [
    "#from  src.run_preprocess import *\n",
    "from src.run_debug import *\n",
    "\n",
    "file = '@preproc.args'\n",
    "\n",
    "print(file)\n",
    "config = main()\n",
    "pprint.pprint(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Image data shape:  (0, 1, 256, 256, None)\n",
      " Image modalities:  ['MR1']\n",
      "  No. of Subjects:  9\n"
     ]
    }
   ],
   "source": [
    "# Open the hdf5 file\n",
    "if config['overwrite'] or not os.path.exists(config[\"output_file\"]):\n",
    "    hdf5_file = tables.open_file(config[\"output_file\"], mode='w')\n",
    "    overwrite = 1\n",
    "else:\n",
    "    hdf5_file = tables.open_file(config[\"output_file\"], mode='r')\n",
    "\n",
    "config[\"hdf5_file\"] = hdf5_file\n",
    "\n",
    "niftis = nifti_loader(\n",
    "        config[\"hdf5_file\"],\n",
    "        config[\"input_images\"],\n",
    "        config[\"problem_type\"],\n",
    "        config[\"image_shape\"],\n",
    "        config[\"training_modalities\"],\n",
    "        config[\"image_masks\"]\n",
    "    )\n",
    "\n",
    "print(' Image data shape: ', niftis.image_data_shape)\n",
    "print(' Image modalities: ', niftis.image_modalities)\n",
    "print('  No. of Subjects: ', len(niftis.ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data files  9\n",
      "Subject ids  ['DUKET1dynportal0244', 'DUKET1dynpre0285', 'DUKET1nfs0415', 'DUKET2ssfse0021', 'DUKET2ssfse0033', 'UCSDT1GRE0000_3', 'UCSDT1GRE0142_1', 'UCSDT1HBP0088', 'UCSDT1mAP0042_b']\n",
      "Image data shape  (0, 1, 256, 256, None)\n",
      "Truth data shape  (0, 1, 256, 256, None)\n",
      "N channels  1\n"
     ]
    }
   ],
   "source": [
    "print('Data files ', len(niftis.data_files))\n",
    "print('Subject ids ', niftis.ids)\n",
    "print('Image data shape ', niftis.image_data_shape)\n",
    "print('Truth data shape ', niftis.truth_data_shape)\n",
    "print('N channels ', niftis.n_channels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "niftis.Noneindex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "/ (RootGroup) ''\n",
       "  children := ['imdata' (VLArray), 'affine' (EArray), 'imstats' (EArray), 'truth' (VLArray), 'volume' (EArray)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "niftis.hdf5.root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading: /home/jovyan/data/DUKET1dynportal0244/MR1.nii\n",
      "Reading: /home/jovyan/data/DUKET1dynportal0244/MASK.nii\n",
      "Reading rri: /home/jovyan/data/DUKET1dynportal0244/MR1.nii\n",
      "Reading rri: /home/jovyan/data/DUKET1dynportal0244/MASK.nii\n",
      "Reading: /home/jovyan/data/DUKET1dynpre0285/MR1.nii\n",
      "Reading: /home/jovyan/data/DUKET1dynpre0285/MASK.nii\n",
      "Reading rri: /home/jovyan/data/DUKET1dynpre0285/MR1.nii\n",
      "Reading rri: /home/jovyan/data/DUKET1dynpre0285/MASK.nii\n",
      "Reading: /home/jovyan/data/DUKET1nfs0415/MR1.nii\n",
      "Reading: /home/jovyan/data/DUKET1nfs0415/MASK.nii\n",
      "Reading rri: /home/jovyan/data/DUKET1nfs0415/MR1.nii\n",
      "Reading rri: /home/jovyan/data/DUKET1nfs0415/MASK.nii\n",
      "Reading: /home/jovyan/data/DUKET2ssfse0021/MR1.nii\n",
      "Reading: /home/jovyan/data/DUKET2ssfse0021/MASK.nii\n",
      "Reading rri: /home/jovyan/data/DUKET2ssfse0021/MR1.nii\n",
      "Reading rri: /home/jovyan/data/DUKET2ssfse0021/MASK.nii\n",
      "Reading: /home/jovyan/data/DUKET2ssfse0033/MR1.nii\n",
      "Reading: /home/jovyan/data/DUKET2ssfse0033/MASK.nii\n",
      "Reading rri: /home/jovyan/data/DUKET2ssfse0033/MR1.nii\n",
      "Reading rri: /home/jovyan/data/DUKET2ssfse0033/MASK.nii\n",
      "Reading: /home/jovyan/data/UCSDT1GRE0000_3/MR1.nii\n",
      "Reading: /home/jovyan/data/UCSDT1GRE0000_3/MASK.nii\n",
      "Reading rri: /home/jovyan/data/UCSDT1GRE0000_3/MR1.nii\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/nilearn/image/resampling.py:510: UserWarning: Casting data from int16 to float32\n",
      "  warnings.warn(\"Casting data from %s to %s\" % (data.dtype.name, aux))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading rri: /home/jovyan/data/UCSDT1GRE0000_3/MASK.nii\n",
      "Reading: /home/jovyan/data/UCSDT1GRE0142_1/MR1.nii\n",
      "Reading: /home/jovyan/data/UCSDT1GRE0142_1/MASK.nii\n",
      "Reading rri: /home/jovyan/data/UCSDT1GRE0142_1/MR1.nii\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/nilearn/image/resampling.py:510: UserWarning: Casting data from int16 to float32\n",
      "  warnings.warn(\"Casting data from %s to %s\" % (data.dtype.name, aux))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading rri: /home/jovyan/data/UCSDT1GRE0142_1/MASK.nii\n",
      "Reading: /home/jovyan/data/UCSDT1HBP0088/MR1.nii\n",
      "Reading: /home/jovyan/data/UCSDT1HBP0088/MASK.nii\n",
      "Reading rri: /home/jovyan/data/UCSDT1HBP0088/MR1.nii\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/nilearn/image/resampling.py:510: UserWarning: Casting data from int16 to float32\n",
      "  warnings.warn(\"Casting data from %s to %s\" % (data.dtype.name, aux))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading rri: /home/jovyan/data/UCSDT1HBP0088/MASK.nii\n",
      "Reading: /home/jovyan/data/UCSDT1mAP0042_b/MR1.nii\n",
      "Reading: /home/jovyan/data/UCSDT1mAP0042_b/MASK.nii\n",
      "Reading rri: /home/jovyan/data/UCSDT1mAP0042_b/MR1.nii\n",
      "Reading rri: /home/jovyan/data/UCSDT1mAP0042_b/MASK.nii\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "/imdata (VLArray(9,), shuffle, blosc(5)) ''\n",
       "  atom = Float32Atom(shape=(1, 256, 256), dflt=0.0)\n",
       "  byteorder = 'little'\n",
       "  nrows = 9\n",
       "  flavor = 'numpy'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "niftis.load_toHDF5_debug()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "niftis.hdf5_toImages('output')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "niftis.Noneindex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, 1, 256, 256, None)\n",
      "[1, 256, 256, 80]\n",
      "(1, 256, 256, 80)\n"
     ]
    }
   ],
   "source": [
    "print(niftis.image_data_shape)\n",
    "imshape = list(niftis.image_data_shape)\n",
    "imshape[niftis.Noneindex] = niftis.hdf5.root.imdata[0].shape[0]\n",
    "imshape = imshape[1:]\n",
    "print(imshape)\n",
    "imshape = tuple(imshape)\n",
    "print(imshape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(80, 1, 256, 256)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "niftis.hdf5.root.truth[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "niftis.image_data_shape = (0,1,256,256,32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tables.Filters(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_samples = len(niftis.ids)\n",
    "\n",
    "#image_storage1 = niftis.hdf5.create_earray(niftis.hdf5.root, 'imdata', tables.Float32Atom(), \n",
    "#                                          shape=(0,1,256,256,32), filters=filters, expectedrows=n_samples)\n",
    "image_storage2 = niftis.hdf5.create_vlarray(niftis.hdf5.root, 'imdata2', tables.Float32Atom(shape=()), filters=filters, expectedrows=n_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.unet3d.normalize import reslice_image_set\n",
    "for (i, imagePath) in enumerate(niftis.data_files):\n",
    "    print(i)\n",
    "    images, imstats = reslice_image_set(\n",
    "                in_files=imagePath, image_shape=(256,256,None), label_indices=len(imagePath)-1, crop=True, stats=True)\n",
    "    subject_data = [image.get_fdata() for image in images]\n",
    "    break\n",
    "    image_storage2.append(np.asarray(subject_data[:niftis.n_channels])[np.newaxis])\n",
    "    print(imstats)\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subject_data[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a=np.asarray([5,6])\n",
    "a.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.asarray(list(range(1,21))).reshape(2,2,5)\n",
    "print(a.shape)\n",
    "b = np.asarray(list(range(1,17))).reshape(2,2,4)\n",
    "print(b.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fileh.close()\n",
    "filters = tables.Filters(complevel=5, complib='blosc')\n",
    "n_samples=9\n",
    "exp_slices = 5 # expected slices\n",
    "fileh = tables.open_file('earrays.h5', mode='w')\n",
    "vlarray = fileh.create_vlarray(fileh.root, 'vlarray1', tables.Int32Atom(shape=(1,2,2)),\"ragged array of ints\",\n",
    "                              filters=filters)  #(1,2,2)\n",
    "\n",
    "\n",
    "vlind = fileh.create_earray(fileh.root, 'vlind', tables.Int32Atom(), shape=(0,2), filters=filters, expectedrows=n_samples)\n",
    "\n",
    "\n",
    "earray2 = fileh.create_earray(fileh.root, 'imdata_slices', tables.Int32Atom(), \n",
    "                                          shape=(0,1,2,2), filters=filters, expectedrows=n_samples*exp_slices)\n",
    "\n",
    "\n",
    "#earray = fileh.create_earray(fileh.root, 'imdata', tables.Int32Atom(), \n",
    "#                                          shape=(0,1,2,2,5), filters=filters, expectedrows=n_samples)\n",
    "start= 0\n",
    "stop = 0\n",
    "\n",
    "vlind.append(np.array([0,5])[np.newaxis])\n",
    "vlind.append(np.array([5,10])[np.newaxis])\n",
    "\n",
    "# Append some (variable length) rows:\n",
    "vlarray.append(a.reshape(5,1,2,2))\n",
    "vlarray.append(b.reshape(5,1,2,2))\n",
    "\n",
    "earray.append(a[np.newaxis][np.newaxis])\n",
    "earray.append(b[np.newaxis][np.newaxis])\n",
    "\n",
    "earray2.append(a.reshape(5,1,2,2))\n",
    "earray2.append(b.reshape(5,1,2,2))\n",
    "\n",
    "# Now, read it through an iterator:\n",
    "# print('-->', vlarray.title)\n",
    "# for x in vlarray:\n",
    "#     print('%s[%d]--> %s' % (vlarray.name, vlarray.nrow, x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"/vlarray1\" in fileh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "earray[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "earray.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "earray2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = np.stack(earray2[vlind[0,0]:vlind[0,1]],axis=0)\n",
    "y = y.reshape(1,2,2,5)\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "earray[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "earray[0].all() == y.all() == z.all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "z = vlarray[0].reshape(1,2,2,5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_data_shape = (0,1,2,2,None)\n",
    "truth_data_shape = (0,1,2,2,None)\n",
    "input_shape = (2,2,None)\n",
    "if None in image_data_shape:\n",
    "    Noneindex = input_shape.index(None)\n",
    "    imarray_shape = tuple([i for i in image_data_shape if i])\n",
    "    imtruth_shape = tuple([i for i in truth_data_shape if i])\n",
    "    print(Noneindex)\n",
    "    print(imarray_shape)\n",
    "    print(imtruth_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "v = 10\n",
    "a = 7\n",
    "if v:\n",
    "    a = 1\n",
    "else:\n",
    "    a = 2\n",
    "    \n",
    "print(a)    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config['output_file'] = '/data/something/something/name.hdf5'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_d = config['output_file'].split('/')[0]\n",
    "out_f = os.path.splitext(os.path.basename(config['output_file']))[0]\n",
    "out_f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.path.basename(config['output_file'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "file_extension": ".py",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "nbformat": 4,
 "nbformat_minor": 2
}