xnat_nifti_downloader.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XNAT nifti downloader\n",
    "Download all the dicoms for all subjects specified in a csv file.\n",
    "\n",
    "Please edit login.cfg with your credentials before executing this script.\n",
    "\n",
    "Before (re-)running this script, please clear output, shutdown and relaunch kernel, close down and reopen your browser, and then (re-)launch all the cells! Else the memory is not correctly freed (this is a bug in ipywidgets or jupyter notebook).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "# BEWARE: autoreload works on functions and on general code, but NOT on new class methods:\n",
    "# if you add or change the name of a method, you have to reload the kernel!\n",
    "# also it will fail if you use super() calls in the classes you change\n",
    "\n",
    "# Profilers:\n",
    "# http://pynash.org/2013/03/06/timing-and-profiling/\n",
    "# http://mortada.net/easily-profile-python-code-in-jupyter.html\n",
    "# use %lprun -m module func(*args, **kwargs)\n",
    "try:\n",
    "    %load_ext line_profiler\n",
    "    %load_ext memory_profiler\n",
    "    from fdict import fdict\n",
    "except ImportError as exc:\n",
    "    pass\n",
    "\n",
    "import json\n",
    "import pandas as pd\n",
    "import os\n",
    "import pyxnat\n",
    "from tqdm import tqdm_notebook\n",
    "\n",
    "# Setup some display options for pandas\n",
    "pd.set_option('max_columns', 400)\n",
    "pd.set_option('expand_frame_repr', False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Edit the filepath to the csv file with the subjects you want to download here\n",
    "csv_filepath = 'xnat_data_extract_rest-icu.csv'\n",
    "# Edit the subjects and projects column names\n",
    "subjectcol = 'subject.id'\n",
    "projectcol = 'project.id'\n",
    "# Edit the folder path where the NIFTI images will be saved\n",
    "nifti_path = os.path.join(os.getcwd(), 'niftis')\n",
    "#nifti_path='F:\\ctbi_rest_niftis'\n",
    "# Download each NIFTI file separately (True) or directly the whole experiment as a zip file (True) ? The latter is faster and you don't risk missing any file.\n",
    "dlmanualmode = True\n",
    "# Max retries to download nifti files before failing\n",
    "maxretries = 10\n",
    "# Restart from a specific subject?\n",
    "restart_from = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Auxiliary functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#### HELPER FUNCTIONS\n",
    "from copy import deepcopy\n",
    "from libs.xmlpp import get_pprint as xml_pprint\n",
    "def get_raw_xml(elements_list):\n",
    "    '''Get the source xml of a list of lxml elements or pyxnat objects'''\n",
    "    # Convert to a list of elements if it's a single element (to ease looping)\n",
    "    if not isinstance(elements_list, list):\n",
    "        elements_list = [elements_list]\n",
    "\n",
    "    out = ''\n",
    "    for i, element in enumerate(elements_list):\n",
    "        out += '\\n=== Element %i\\n' % i\n",
    "        # If this is an XML element\n",
    "        if isinstance(element, lxml.etree._Element):\n",
    "            # Make a copy of the element because we will modify it\n",
    "            e = deepcopy(element)\n",
    "            # Strip comments, else lxml does not know how to print the XML\n",
    "            lxml.etree.strip_tags(e, lxml.etree.Comment)\n",
    "            # Add the XML of this element to the output\n",
    "            out += xml_pprint(lxml.etree.tostring(e, pretty_print=True))\n",
    "            #print(lxml.etree.tostring(e, pretty_print=True)) #debug\n",
    "        # pyxnat object, we just fetch the xml from the server\n",
    "        if isinstance(element, pyxnat.core.resources.EObject):\n",
    "            out += element.get()\n",
    "        # Print differently if this is any other type\n",
    "        else:\n",
    "            out += repr(element)\n",
    "    return out\n",
    "\n",
    "def pprint_xml(obj):\n",
    "    print(xml_pprint(get_raw_xml(obj)))\n",
    "\n",
    "#### HELPER GLOBALS\n",
    "# XNAT namespace (to use with lxml xpath queries)\n",
    "xnatns = {'arc': 'http://nrg.wustl.edu/arc',\n",
    " 'cat': 'http://nrg.wustl.edu/catalog',\n",
    " 'ext': 'http://nrg.wustl.edu/ext',\n",
    " 'pipe': 'http://nrg.wustl.edu/pipe',\n",
    " 'prov': 'http://www.nbirn.net/prov',\n",
    " 'scr': 'http://nrg.wustl.edu/scr',\n",
    " 'val': 'http://nrg.wustl.edu/val',\n",
    " 'wrk': 'http://nrg.wustl.edu/workflow',\n",
    " 'xdat': 'http://nrg.wustl.edu/security',\n",
    " 'xnat': 'http://nrg.wustl.edu/xnat',\n",
    " 'xnat_a': 'http://nrg.wustl.edu/xnat_assessments',\n",
    " 'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Connect to XNAT server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def xnat_connect():\n",
    "    # Loading login infos\n",
    "    cfgpath = os.path.join(os.getcwd(), 'login.cfg')\n",
    "    with open(cfgpath) as f:\n",
    "        login_infos = json.load(f)\n",
    "\n",
    "    # Connect to XNAT db\n",
    "    central = pyxnat.Interface(server=\"http://tbixnat.incf.org:8080\", user=login_infos['username'], password=login_infos['password'], cachedir='/tmp')\n",
    "    # Add schemas (allows to use .attrs() to get list of attributes)\n",
    "    central.manage.schemas.add('xnat/xnat.xsd')\n",
    "\n",
    "    # Get list of all centers\n",
    "    centers = central.select.projects()\n",
    "    return central, centers\n",
    "\n",
    "# Select center (constraining to one center for the moment)\n",
    "# TODO: loop over all centers\n",
    "#cULgData_Liege_project = central.select.project('LIE')\n",
    "\n",
    "central, centers = xnat_connect()\n",
    "\n",
    "# Show all centers list\n",
    "print(centers.get())\n",
    "\n",
    "# Show structure of project\n",
    "central.inspect.structure()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load csv file as pandas dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(csv_filepath, sep=';', index_col=False, encoding='utf-8', escapechar='\\\\')\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download dicoms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Groupby project and subject (to avoid duplication of subject, we don't want to download twice the same)\n",
    "df_subjects = df.ix[:, (projectcol, subjectcol)].groupby((projectcol,subjectcol)).count().reset_index()\n",
    "df_subjects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import unicodedata\n",
    "def slugify(value, allow_unicode=False):\n",
    "    \"\"\"\n",
    "    Convert to ASCII if 'allow_unicode' is False. Convert spaces to hyphens.\n",
    "    Remove characters that aren't alphanumerics, underscores, or hyphens.\n",
    "    Convert to lowercase. Also strip leading and trailing whitespace.\n",
    "    From Django and this excellent SO answer: https://stackoverflow.com/a/295466/1121352\n",
    "    \"\"\"\n",
    "    value = str(value)\n",
    "    if allow_unicode:\n",
    "        value = unicodedata.normalize('NFKC', value)\n",
    "    else:\n",
    "        value = unicodedata.normalize('NFKD', unicode(value.encode('utf-8', 'ignore'))).encode('ascii', 'ignore').decode('ascii')\n",
    "    value = re.sub(r'[^\\w\\s-]', '', value).strip().lower()\n",
    "    return re.sub(r'[-\\s]+', '-', value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Prepare progress bar\n",
    "tbar = tqdm_notebook(total=len(df_subjects), desc='DOWNLD', unit='subject')\n",
    "\n",
    "# Main download loop\n",
    "for subjnb, row in enumerate(df_subjects.iterrows()):\n",
    "    # Loop for each project & subject in the provided csv\n",
    "    projectid = row[1][projectcol]\n",
    "    subjectid = row[1][subjectcol]\n",
    "    if not restart_from or subjnb > restart_from:\n",
    "        while True:  # infinite loop to retry in case of error\n",
    "            flagrestart = False\n",
    "            try:\n",
    "                experiments = central.select(\"/project/%s/subject/%s\" % (projectid, subjectid)).experiments()\n",
    "                # sometimes experiment/scans is empty, so we will restart until we can extract something\n",
    "                if experiments is None:\n",
    "                    flag_restart = True\n",
    "                    break\n",
    "                for experiment in experiments:\n",
    "                    # sometimes experiment/scans is empty, so we will restart until we can extract something\n",
    "                    if experiment is None:\n",
    "                        flag_restart = True\n",
    "                        break\n",
    "                    # For each experiment (acquisition sessions) of this subject\n",
    "                    experimentid = experiment.id()\n",
    "                    try:\n",
    "                        experimenttype = slugify(experiment.xpath('@visit_id')[0]).replace(' ', '-')\n",
    "                        if not experimenttype:\n",
    "                            experimenttype = 'blank'\n",
    "                    except IndexError as exc:\n",
    "                        experimenttype = 'none'\n",
    "                    scans = experiment.scans()\n",
    "                    # sometimes experiment/scans is empty, so we will restart until we can extract something\n",
    "                    if scans is None:\n",
    "                        flag_restart = True\n",
    "                        break\n",
    "                    for scan in scans:\n",
    "                        # sometimes experiment/scans is empty, so we will restart until we can extract something\n",
    "                        if scan is None:\n",
    "                            flag_restart = True\n",
    "                            break\n",
    "                        # For each scan of this experiment\n",
    "                        # Build subdirectory path where to save this nifti\n",
    "                        #print(scan.xpath('@UID')[0])  # do NOT use scan.attrs.get('@UID'), pyxnat v1.0 is not reliable when using that! Prefer to use xpath to get attributes.\n",
    "                        scantype = scan.xpath('@type')[0].strip()\n",
    "                        if not scantype:\n",
    "                            scantype = 'blank'\n",
    "                        scanid = scan.xpath('@ID')[0]\n",
    "                        try:\n",
    "                            scanuid = scan.xpath('@UID')[0]\n",
    "                        except IndexError as exc:\n",
    "                            scanuid = '0'\n",
    "                        # Build path to store this resource\n",
    "                        scanpath = '%s_%s_%s' % (scanid, scantype, scanuid)\n",
    "                        scanpath = slugify(scanpath)  # clean up path to always have only valid path characters\n",
    "                        # Build full path\n",
    "                        scanfullpath = os.path.join(nifti_path, projectid, subjectid, '%s_%s' % (experimentid, experimenttype), scanpath)\n",
    "                        if not os.path.exists(scanfullpath):\n",
    "                            # create directories recursively\n",
    "                            os.makedirs(scanfullpath)\n",
    "                        # Save meta-infos\n",
    "                        scan_metadata = scan.get()\n",
    "                        with open(os.path.join(scanfullpath, 'metadata.xml'), 'w') as f:\n",
    "                            f.write(scan_metadata)\n",
    "                        # Download the NIFTI files (manual mode: we download each scan separately, this allows finer grained control over naming etc)\n",
    "                        if dlmanualmode:\n",
    "                            r = scan.resource('NIFTI')\n",
    "                            for nfile in r.files():\n",
    "                                # TODO: remove from cachemanager as soon as it gets downloaded, else we will overbloat our cache for nothing (maybe it helps accelerate redownloads but well we don't care)\n",
    "                                for i in xrange(maxretries):\n",
    "                                    try:\n",
    "                                        nfile.get_copy(dest=os.path.join(scanfullpath, nfile.label()))\n",
    "                                        break\n",
    "                                    except Exception as exc:\n",
    "                                        continue\n",
    "                    # Download the NIFTI files (auto mode: download the whole archive of all scans for this experiment directly as a zip file)\n",
    "                    if not dlmanualmode:\n",
    "                        if experiment.scans().get():\n",
    "                            try:\n",
    "                                experiment.scans().download(os.path.join(nifti_path, projectid, subjectid, experimentid))\n",
    "                            except Exception as exc:\n",
    "                                if 'BadZipfile' in str(type(exc)) or 'not a zip file' in str(exc):\n",
    "                                    # Sometimes the experiment's scans contains no real file, and no real acquisition, then we need to skip\n",
    "                                    pass  # TODO: sometimes we get badzipfile because of connection abort or another bug during transfert, the difference is that the file is bigger than 0 or 1KB. We need to check for this case and retry download (else we end up with partial zip files for some experiments that are available normally)\n",
    "                                else:\n",
    "                                    raise\n",
    "            except Exception as exc:\n",
    "                if 'ConnectionError' in str(type(exc)) or 'XMLSyntaxError' in str(type(exc)):\n",
    "                    flagrestart = True\n",
    "                    pass\n",
    "                else:\n",
    "                    raise\n",
    "            # No exception? Then break of the infinite loop, go to next experiment\n",
    "            if not flagrestart:\n",
    "                break\n",
    "    # Update progress bar\n",
    "    tbar.update()\n",
    "\n",
    "print('All done!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}