comparison_test_python3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import subprocess as sb\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ensure_dir(directory):\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dockerhub_account='pinellolab'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_syn_dataset=\"./dataset/synthetic\"\n",
    "dir_real_dataset=\"./dataset/real\"\n",
    "\n",
    "dir_syn_results=\"./results/synthetic\"\n",
    "dir_real_results=\"./results/real\"\n",
    "\n",
    "ensure_dir(dir_syn_results)\n",
    "ensure_dir(dir_real_results)\n",
    "\n",
    "syn_datasets=[]\n",
    "real_datasets=[]\n",
    "\n",
    "metrics_syn=[\"correlation\", \"accuracy\" ,\"stability\"]\n",
    "metrics_real=[\"correlation\", \"accuracy\" ,\"stability\"]\n",
    "\n",
    "methods=[\"monocle2\",]\n",
    "\n",
    "FORCE_RERUN=False\n",
    "DEBUG=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test methods on synthetic datasets\n",
    "for dataset_dir in glob.glob(\"./dataset/synthetic/*\"):\n",
    "    if os.path.isdir(dataset_dir):\n",
    "        syn_datasets.append(dataset_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test methods on real datasets\n",
    "for dataset_dir in glob.glob(\"./dataset/real/*\"):\n",
    "    if os.path.isdir(dataset_dir):\n",
    "        real_datasets.append(dataset_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./dataset/synthetic/prosst_topology3_prosstt0']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "syn_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "real_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "[Current Dataset]: prosst_topology3_prosstt0\n",
      "\n",
      "  [Running Method]: monocle2 (pinellolab/hca_method_monocle2)\n",
      "\n",
      "docker run             -v ${PWD}:/data -w /data pinellolab/hca_method_monocle2 run_method             ./dataset/synthetic/prosst_topology3_prosstt0/input.txt ./results/synthetic/prosst_topology3_prosstt0/monocle2/output.txt\n",
      "    [Evaluating Metric]: correlation (pinellolab/hca_score_correlation)\n",
      "\n",
      "    docker run                     -v  ${PWD}:/hca -w /hca pinellolab/hca_score_correlation -o ./results/synthetic/prosst_topology3_prosstt0/monocle2/output.txt -t ./dataset/synthetic/prosst_topology3_prosstt0/truth.txt  -s ./results/synthetic/prosst_topology3_prosstt0/monocle2/hca_score_correlation.txt\n",
      "\n",
      "    [Evaluating Metric]: accuracy (pinellolab/hca_score_accuracy)\n",
      "\n",
      "    docker run                     -v  ${PWD}:/hca -w /hca pinellolab/hca_score_accuracy -o ./results/synthetic/prosst_topology3_prosstt0/monocle2/output.txt -t ./dataset/synthetic/prosst_topology3_prosstt0/truth.txt  -s ./results/synthetic/prosst_topology3_prosstt0/monocle2/hca_score_accuracy.txt\n",
      "\n",
      "    [Evaluating Metric]: stability (pinellolab/hca_score_stability)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for dataset_dir in syn_datasets:\n",
    "    dataset_name=os.path.basename(dataset_dir)\n",
    "        \n",
    "    print ('\\n\\n[Current Dataset]: %s\\n' % dataset_name)\n",
    "    method_input_filename=os.path.join(dataset_dir,'input.txt')\n",
    "    metric_input_filename=os.path.join(dataset_dir,'truth.txt')\n",
    "    \n",
    "    \n",
    "    if os.path.exists(method_input_filename) and \\\n",
    "        os.path.exists(metric_input_filename): \n",
    "    \n",
    "        for method_name in methods:\n",
    "            \n",
    "            docker_image_method='%s/hca_method_%s' % (dockerhub_account, method_name)\n",
    "            print ('  [Running Method]: %s (%s)\\n' %\n",
    "                (method_name, docker_image_method))\n",
    "\n",
    "            \n",
    "            output_dir= os.path.join(dir_syn_results,dataset_name,method_name)\n",
    "            ensure_dir(output_dir)\n",
    "\n",
    "\n",
    "            method_output_filename=os.path.join(output_dir ,'output.txt')\n",
    "            \n",
    "            \n",
    "            #print (docker_image_method,dataset_path, dataset_name,output_folder)\n",
    "\n",
    "            cmd_method_to_run='docker run \\\n",
    "            -v ${PWD}:/data -w /data %s run_method \\\n",
    "            %s %s'\\\n",
    "            % (docker_image_method,method_input_filename, method_output_filename )\n",
    "           \n",
    "            if DEBUG:\n",
    "                print ( cmd_method_to_run)\n",
    "            \n",
    "            if os.path.exists(method_output_filename) and not FORCE_RERUN:\n",
    "                print('    The method was already run on this dataset, skipping.\\n')\n",
    "                \n",
    "            else:\n",
    "                sb.check_output(cmd_method_to_run,shell=True)\n",
    "\n",
    "                \n",
    "\n",
    "            for metric_name in metrics_syn:\n",
    "                \n",
    "                docker_image_metric='%s/hca_score_%s' % (dockerhub_account, metric_name)\n",
    "                print('    [Evaluating Metric]: %s (%s)\\n' %( metric_name, docker_image_metric))\n",
    "\n",
    "                if metric_name != 'stability':                \n",
    "                    \n",
    "                    metric_output_filename=os.path.join(output_dir,'hca_score_%s.txt' % metric_name)\n",
    "                    cmd_metric_to_run='docker run \\\n",
    "                    -v  ${PWD}:/hca -w /hca %s -o %s -t %s  -s %s' \\\n",
    "                    %(docker_image_metric,method_output_filename, metric_input_filename, metric_output_filename)\n",
    "\n",
    "                    if DEBUG:\n",
    "                        print ('    '+cmd_metric_to_run+'\\n')\n",
    "                    \n",
    "                    if os.path.exists(metric_output_filename) and not FORCE_RERUN:\n",
    "                        print('      The metric was already run for this dataset method pair, skipping.\\n')\n",
    "                    \n",
    "                    else:\n",
    "                        sb.check_output(cmd_metric_to_run,shell=True)\n",
    "                else:\n",
    "                    pass\n",
    "           \n",
    "    else:\n",
    "        print ('Input dataset is missing the required files: input.txt and or truth.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "method_output_filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "method_input_filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!ls ./results/synthetic/prosst_topology3_prosstt0/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}