From 347b16044d81f1609b1091b1ca2c5543a4037777 Mon Sep 17 00:00:00 2001 From: Jalil Nourisa Date: Mon, 23 Sep 2024 17:15:01 +0200 Subject: [PATCH] local runs updated --- NN-grn-inference.ipynb | 1106 ----- runs.ipynb | 4349 +++++++++-------- scripts/run_robust_analys.sh | 21 +- scripts/sbatch/calculate_scores.sh | 5 +- src/api/comp_method.yaml | 2 +- src/methods/single_omics/grnboost2/script.py | 33 +- src/metrics/regression_2/main.py | 3 - src/metrics/regression_2/script_all.py | 53 - src/metrics/{regression_1 => }/script_all.py | 14 +- src/robustness_analysis/permute_grn/main.py | 67 + src/robustness_analysis/permute_grn/script.py | 65 +- src/robustness_analysis/script_all.py | 130 + src/utils/util.py | 18 + 13 files changed, 2688 insertions(+), 3178 deletions(-) delete mode 100644 NN-grn-inference.ipynb delete mode 100644 src/metrics/regression_2/script_all.py rename src/metrics/{regression_1 => }/script_all.py (80%) create mode 100644 src/robustness_analysis/permute_grn/main.py create mode 100644 src/robustness_analysis/script_all.py diff --git a/NN-grn-inference.ipynb b/NN-grn-inference.ipynb deleted file mode 100644 index 82b486dbb..000000000 --- a/NN-grn-inference.ipynb +++ /dev/null @@ -1,1106 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Multiomics GRN inference evaluation\n", - "## VAE-SEM model\n", - "### by Jalil Nourisa" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting anndata\n", - " Using cached anndata-0.10.9-py3-none-any.whl.metadata (6.9 kB)\n", - "Collecting array-api-compat!=1.5,>1.4 (from anndata)\n", - " Using cached array_api_compat-1.8-py3-none-any.whl.metadata (1.5 kB)\n", - "Requirement already satisfied: exceptiongroup in /home/jnourisa/miniconda3/envs/pytorch/lib/python3.9/site-packages (from anndata) (1.2.2)\n", - "Collecting h5py>=3.1 (from anndata)\n", - " Using cached h5py-3.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (2.5 kB)\n", - "Collecting natsort (from anndata)\n", - " Using cached natsort-8.4.0-py3-none-any.whl.metadata (21 kB)\n", - "Requirement already satisfied: numpy>=1.23 in /home/jnourisa/miniconda3/envs/pytorch/lib/python3.9/site-packages (from anndata) (2.0.1)\n", - "Requirement already satisfied: packaging>=20.0 in /home/jnourisa/miniconda3/envs/pytorch/lib/python3.9/site-packages (from anndata) (24.1)\n", - "Collecting pandas!=2.1.0rc0,!=2.1.2,>=1.4 (from anndata)\n", - " Using cached pandas-2.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (19 kB)\n", - "Collecting scipy>1.8 (from anndata)\n", - " Using cached scipy-1.13.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (60 kB)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /home/jnourisa/miniconda3/envs/pytorch/lib/python3.9/site-packages (from pandas!=2.1.0rc0,!=2.1.2,>=1.4->anndata) (2.9.0)\n", - "Collecting pytz>=2020.1 (from pandas!=2.1.0rc0,!=2.1.2,>=1.4->anndata)\n", - " Using cached pytz-2024.2-py2.py3-none-any.whl.metadata (22 kB)\n", - "Collecting tzdata>=2022.7 (from pandas!=2.1.0rc0,!=2.1.2,>=1.4->anndata)\n", - " Using cached tzdata-2024.1-py2.py3-none-any.whl.metadata (1.4 kB)\n", - "Requirement already satisfied: six>=1.5 in /home/jnourisa/miniconda3/envs/pytorch/lib/python3.9/site-packages (from python-dateutil>=2.8.2->pandas!=2.1.0rc0,!=2.1.2,>=1.4->anndata) (1.16.0)\n", - "Using cached anndata-0.10.9-py3-none-any.whl (128 kB)\n", - "Using cached array_api_compat-1.8-py3-none-any.whl (38 kB)\n", - "Using cached h5py-3.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.3 MB)\n", - "Using cached pandas-2.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.1 MB)\n", - "Using cached scipy-1.13.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (38.6 MB)\n", - "Using cached natsort-8.4.0-py3-none-any.whl (38 kB)\n", - "Using cached pytz-2024.2-py2.py3-none-any.whl (508 kB)\n", - "Using cached tzdata-2024.1-py2.py3-none-any.whl (345 kB)\n", - "Installing collected packages: pytz, array-api-compat, tzdata, scipy, natsort, h5py, pandas, anndata\n", - "Successfully installed anndata-0.10.9 array-api-compat-1.8 h5py-3.11.0 natsort-8.4.0 pandas-2.2.2 pytz-2024.2 scipy-1.13.1 tzdata-2024.1\n" - ] - } - ], - "source": [ - "!pip install anndata" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "execution": { - "iopub.execute_input": "2024-06-27T16:10:17.111822Z", - "iopub.status.busy": "2024-06-27T16:10:17.111469Z", - "iopub.status.idle": "2024-06-27T16:10:19.741614Z", - "shell.execute_reply": "2024-06-27T16:10:19.740860Z", - "shell.execute_reply.started": "2024-06-27T16:10:17.111796Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "using device: cuda\n", - "-----Seed Set!-----\n" - ] - } - ], - "source": [ - "import torch \n", - "# import anndata as ad\n", - "import pandas as pd\n", - "import numpy as np\n", - "import tqdm\n", - "# from sklearn.metrics import r2_score, roc_auc_score\n", - "# from sklearn.metrics import mean_squared_error \n", - "# from sklearn.decomposition import TruncatedSVD, KernelPCA \n", - "# from sklearn.ensemble import RandomForestRegressor\n", - "import os\n", - "import random \n", - "# import category_encoders \n", - "import torch.nn as nn \n", - "# import matplotlib.pyplot as plt\n", - "\n", - "os.environ['CUBLAS_WORKSPACE_CONFIG'] = ':4096:8'\n", - "\n", - "SEED = 0xCAFE\n", - "USE_GPU = True\n", - "if USE_GPU and torch.cuda.is_available():\n", - " print('using device: cuda')\n", - "else:\n", - " print('using device: cpu')\n", - " USE_GPU = False\n", - "\n", - "def seed_everything():\n", - " seed = 42\n", - " random.seed(seed)\n", - " os.environ['PYTHONHASHSEED'] = str(seed)\n", - " os.environ['TOKENIZERS_PARALLELISM'] = 'true'\n", - " np.random.seed(seed)\n", - " torch.manual_seed(seed)\n", - " if torch.cuda.is_available():\n", - " torch.cuda.manual_seed(seed)\n", - " torch.cuda.manual_seed_all(seed)\n", - " torch.backends.cudnn.deterministic = True\n", - " torch.backends.cudnn.benchmark = False\n", - " print('-----Seed Set!-----')\n", - "seed_everything()\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "adata_rna = ad.read_h5ad('../output/scRNA/adata_rna.h5ad')\n", - "gene_names = adata_rna.var_names\n", - "gene_name_dict = {gene_name: i for i, gene_name in enumerate(gene_names)}\n", - "n_genes = len(gene_names)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Process grn net" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "# # Download GRN\n", - "# # net = pd.read_csv(\"https://github.com/pablormier/omnipath-static/raw/main/op/collectri-26.09.2023.zip\")\n", - "# net = pd.read_csv(\"../output/benchmark/grn_models/scenicplus.csv\")\n", - "\n", - "# # Iterate over the edges (regulatory relationships)\n", - "# edge_idx = set()\n", - "# grn_weights = []\n", - "# for gene_a, gene_b, weight in zip(net['source'], net['target'], net['weight']):\n", - "# if (gene_a not in gene_name_dict) or (gene_b not in gene_name_dict):\n", - "# continue # Consider only gene names that are present in the training data\n", - "# i = gene_name_dict[gene_a] # Index of first gene\n", - "# j = gene_name_dict[gene_b] # Index of second gene\n", - "# grn_weights.append(weight)\n", - "# edge_idx.add((i, j))\n", - "# edge_idx = np.asarray(list(edge_idx), dtype=int)\n", - "\n", - "# # Convert list of edges into an adjacency matrix\n", - "# grn = np.zeros((len(gene_names), len(gene_names)))\n", - "# for i, weight in enumerate(grn_weights):\n", - "# weight = 1 if weight>=0 else -1\n", - "# grn[edge_idx[i, 0], edge_idx[i, 1]] = weight\n", - "\n", - "# # Remove rows and columns with no annotation\n", - "# grn_mask = np.logical_or(grn.sum(axis=0) > 0, grn.sum(axis=1) > 0)\n", - "# grn = grn[grn_mask, :][:, grn_mask]\n" - ] - }, - { - "cell_type": "code", - "execution_count": 321, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.00639388880957349" - ] - }, - "execution_count": 321, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# grn = pd.read_csv(\"../output/benchmark/grn_models/scenicplus.csv\", index_col=0)\n", - "grn = pd.read_csv(\"https://github.com/pablormier/omnipath-static/raw/main/op/collectri-26.09.2023.zip\")\n", - "# only keep those with both tf and gene included in gene_names\n", - "grn = grn[grn.source.isin(gene_names) & grn.target.isin(gene_names)].reset_index(drop=True)\n", - "# binarize\n", - "weights = [1 if weight>=0 else -1 for weight in grn.weight ] \n", - "grn.weight = weights\n", - "\n", - "GT = grn.pivot(index='source', columns='target', values='weight').fillna(0)\n", - " \n", - "GT = GT.abs() # binary \n", - "GT.abs().mean().mean()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "execution": { - "iopub.execute_input": "2024-06-27T16:43:43.020607Z", - "iopub.status.busy": "2024-06-27T16:43:43.020220Z", - "iopub.status.idle": "2024-06-27T16:43:43.023916Z", - "shell.execute_reply": "2024-06-27T16:43:43.023278Z", - "shell.execute_reply.started": "2024-06-27T16:43:43.020582Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "matrix([[0., 0., 0., ..., 0., 0., 2.],\n", - " [0., 1., 2., ..., 0., 0., 0.],\n", - " [0., 0., 0., ..., 0., 0., 0.],\n", - " ...,\n", - " [0., 0., 0., ..., 0., 1., 0.],\n", - " [0., 0., 3., ..., 0., 0., 1.],\n", - " [0., 0., 0., ..., 0., 0., 0.]])" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Y = adata_rna.X.todense() #TODO: modify the train code to deal with sparse\n", - "Y" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "execution": { - "iopub.execute_input": "2024-06-27T16:28:37.686198Z", - "iopub.status.busy": "2024-06-27T16:28:37.685772Z", - "iopub.status.idle": "2024-06-27T16:28:39.747151Z", - "shell.execute_reply": "2024-06-27T16:28:39.746275Z", - "shell.execute_reply.started": "2024-06-27T16:28:37.686168Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "class Dataset(torch.utils.data.Dataset):\n", - " def __init__(self, features, labels):\n", - " self.features = features \n", - " self.labels = labels \n", - " def __len__(self):\n", - " return len(self.features)\n", - " def __getitem__(self, idx):\n", - " return self.features[idx], self.labels[idx]\n", - "features = torch.tensor(Y, dtype=torch.float32) \n", - "labels = torch.tensor(Y, dtype=torch.float32)\n", - "\n", - "dataset = Dataset(features=features, labels=labels)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Encode data" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "execution": { - "iopub.execute_input": "2024-06-27T16:35:14.952828Z", - "iopub.status.busy": "2024-06-27T16:35:14.952450Z", - "iopub.status.idle": "2024-06-27T16:35:14.956529Z", - "shell.execute_reply": "2024-06-27T16:35:14.955864Z", - "shell.execute_reply.started": "2024-06-27T16:35:14.952800Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# class MultiOutputTargetEncoder:\n", - "# def __init__(self):\n", - "# self.encoders: List[category_encoders.leave_one_out.LeaveOneOutEncoder] = []\n", - " \n", - "# @staticmethod\n", - "# def new_encoder() -> category_encoders.leave_one_out.LeaveOneOutEncoder:\n", - "# return category_encoders.leave_one_out.LeaveOneOutEncoder(return_df=False)\n", - " \n", - "# def fit(self, X: np.ndarray, y: np.ndarray) -> None:\n", - "# self.encoders = []\n", - "# for j in tqdm.tqdm(range(y.shape[1]), desc='fit LOO encoders'):\n", - "# self.encoders.append(MultiOutputTargetEncoder.new_encoder())\n", - "# self.encoders[-1].fit(X, y[:, j])\n", - " \n", - "# def transform(self, X: np.ndarray) -> np.ndarray:\n", - "# Z = []\n", - "# for encoder in tqdm.tqdm(self.encoders, desc='transform LOO encoders'):\n", - "# y_hat = encoder.transform(X)\n", - "# Z.append(y_hat)\n", - "# Z = np.asarray(Z)\n", - "# return np.transpose(Z, (1, 0, 2))\n", - "# encoder = MultiOutputTargetEncoder()\n", - "\n", - "# encoder.fit(np.asarray([df_train.index.get_level_values(var) for var in features_X]).T, df_train.values)\n", - "\n", - "# X = encoder.transform(np.asarray([df_train.index.get_level_values(var) for var in features_X]).T)\n", - "# X_submit = encoder.transform(np.asarray([df_test.index.get_level_values(var) for var in features_X]).T)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### NN design" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "execution": { - "iopub.execute_input": "2024-06-27T16:44:18.031771Z", - "iopub.status.busy": "2024-06-27T16:44:18.031386Z", - "iopub.status.idle": "2024-06-27T16:44:18.037461Z", - "shell.execute_reply": "2024-06-27T16:44:18.036630Z", - "shell.execute_reply.started": "2024-06-27T16:44:18.031745Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "class NN_AE(torch.nn.Module): \n", - " '''\n", - " Simple AE design.\n", - " '''\n", - " def __init__(self, n_genes:int,n_nodes_hidden:int=120):\n", - " torch.nn.Module.__init__(self)\n", - " self.n_genes = n_genes\n", - " dropout_rate = .1\n", - " \n", - " self.mlp = nn.Sequential(\n", - " nn.Linear(n_genes, n_nodes_hidden),\n", - " nn.LeakyReLU(0.2),\n", - " nn.Dropout(dropout_rate), \n", - " \n", - " nn.Linear(n_nodes_hidden, 64),\n", - " nn.LeakyReLU(0.2),\n", - " nn.Dropout(dropout_rate), \n", - " \n", - " nn.Linear(64, 120),\n", - " nn.LeakyReLU(0.2),\n", - " nn.Dropout(dropout_rate), \n", - " \n", - " nn.Linear(n_nodes_hidden, n_genes)\n", - " )\n", - "\n", - " def forward(self, x: torch.Tensor):\n", - " \n", - " x = self.mlp(x)\n", - " \n", - " return x, None, None\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "execution": { - "iopub.execute_input": "2024-06-03T15:18:39.603992Z", - "iopub.status.busy": "2024-06-03T15:18:39.603535Z", - "iopub.status.idle": "2024-06-03T15:18:39.613692Z", - "shell.execute_reply": "2024-06-03T15:18:39.612813Z", - "shell.execute_reply.started": "2024-06-03T15:18:39.603952Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "# def background_noise( #we use dropouts instead of this for pseudobulked data\n", - "# *size: int,\n", - "# cutoff: float = 0.05,\n", - "# device: str = 'cuda',\n", - "# generator: torch.Generator = None) -> torch.Tensor:\n", - "# sign = 2 * torch.randint(0, 2, size, device=device) - 1\n", - "\n", - "# return sign * torch.log10(cutoff + torch.rand(*size, generator=generator, device=device) * (1. - cutoff))\n", - "# class Scaler(torch.nn.Module): #from antoine\n", - " \n", - "# def __init__(self, m: int) -> None:\n", - "# torch.nn.Module.__init__(self)\n", - "# self.m: int = m\n", - "# self.a: torch.Tensor = torch.nn.Parameter(torch.ones((1, self.m)))\n", - "# self.b: torch.Tensor = torch.nn.Parameter(torch.zeros((1, self.m)))\n", - " \n", - "# def forward(self, X: torch.Tensor) -> torch.Tensor:\n", - "# return self.a * X + self.b\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Train" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "execution": { - "iopub.execute_input": "2024-06-27T16:44:21.565567Z", - "iopub.status.busy": "2024-06-27T16:44:21.565166Z", - "iopub.status.idle": "2024-06-27T16:44:21.700323Z", - "shell.execute_reply": "2024-06-27T16:44:21.699132Z", - "shell.execute_reply.started": "2024-06-27T16:44:21.565537Z" - }, - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Rel loss: 0.425, R2:0.574: 100%|██████████| 10/10 [01:06<00:00, 6.67s/it]\n" - ] - } - ], - "source": [ - "from torch.utils.data import DataLoader\n", - "def train_nn(model, dataloader, n_epoch, loss_func):\n", - " model = model.to('cuda')\n", - " model.train()\n", - "\n", - " optimizer = torch.optim.Adam(model.parameters(), lr=1e-3, eps=1e-8)\n", - " \n", - " scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.9, \n", - " patience=5, threshold_mode='rel', threshold=0.0001, cooldown=5, min_lr=1e-10, eps=1e-8)\n", - "\n", - " pbar = tqdm.tqdm(range(n_epoch))\n", - " for i_epoch in pbar:\n", - " rel_loss_store = []\n", - " Y_pred_stack = []\n", - " Y_true_stack = []\n", - " for batch_idx, (feature_batch, label_batch) in enumerate(dataloader):\n", - " \n", - " optimizer.zero_grad()\n", - " data_batch = feature_batch.to('cuda') #TODO: this should be done once \n", - " label_batch = label_batch.to('cuda')\n", - " \n", - " if False: # add background noise\n", - " generator = torch.Generator(device='cuda').manual_seed(32)\n", - " label_batch = label_batch + 0.2 * background_noise(*label_batch.size(), generator=generator) #TODO: optimize the weight\n", - "\n", - " x_pred, mu, log_var = model(data_batch) #forward\n", - "\n", - " Y_true_stack.append(label_batch.cpu().detach().numpy())\n", - " Y_pred_stack.append(x_pred.cpu().detach().numpy())\n", - "\n", - " loss_x = loss_func(x_pred, label_batch)\n", - "\n", - " #loss_KL = - 0.5 * torch.sum(1.0 + log_var - mu.pow(2) - log_var.exp()) #TODO: fix this\n", - "\n", - " beta = 1\n", - " #loss = loss_x + beta*loss_KL\n", - " loss = loss_x \n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # baseline pred\n", - " Y_pred_mean = torch.mean(label_batch, axis=0)\n", - " loss_baseline = loss_func(label_batch, Y_pred_mean)\n", - " rel_loss = loss_x/loss_baseline\n", - " rel_loss_store.append(rel_loss.item())\n", - " # if i_epoch%10==0:\n", - " # # AUROC\n", - " # mask = ~np.eye(n_genes, dtype=bool)\n", - " # grn_pred = np.abs(model.A.cpu().data.numpy())\n", - " # print('AUROC', roc_auc_score(np.abs(grn_net[mask]), grn_pred[mask]))\n", - "\n", - " mean_rel_loss = np.mean(rel_loss_store)\n", - " scheduler.step(mean_rel_loss)\n", - "\n", - " y_pred = np.concatenate(Y_pred_stack, axis=0)\n", - " y_true = np.concatenate(Y_true_stack, axis=0)\n", - "\n", - " r2 = r2_score(y_true, y_pred, multioutput='variance_weighted')\n", - "\n", - " pbar.set_description(f'Rel loss: {mean_rel_loss:.3f}, R2:{r2:.3f}')\n", - " \n", - "\n", - " return model\n", - "\n", - "batch_size = int(Y.shape[0]/20)\n", - "n_epoch = 10\n", - "n_model = 1\n", - "num_workers = 4\n", - "shuffle_data = True\n", - "loss_func = lambda y_pred, y_true: torch.sum(torch.square(y_pred-y_true))\n", - "\n", - "models = []\n", - "# y_pred_submit_all = []\n", - "for i in range(n_model):\n", - " dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle_data, num_workers=num_workers)\n", - " model = NN_AE(n_genes)\n", - " model = train_nn(model, dataloader, n_epoch, loss_func)\n", - " \n", - " models.append(model)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Extract GRN" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "# Register hooks to capture activations for all layers\n", - "nodel = model.to('cpu')\n", - "model.eval()\n", - "layers = model.mlp\n", - "\n", - "activations = []\n", - "\n", - "def get_activation_hook(layer_index):\n", - " def hook(module, input, output):\n", - " activations[layer_index] = output.detach()\n", - " return hook\n", - "\n", - "# Initialize activations list\n", - "activations = [None] * len(layers)\n", - "\n", - "# Register hooks for all Linear layers\n", - "for idx, layer in enumerate(model.mlp):\n", - " layer.register_forward_hook(get_activation_hook(idx))\n", - "\n", - "# Forward pass to get activations\n", - "with torch.no_grad():\n", - " _ = model(features)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "tensor([-14.6386, -11.2354, -2.3267, 3.0302, -0.5415])\n", - "tensor([-2.9277, -2.2471, -0.4653, 3.0302, -0.1083])\n", - "tensor([-2.9277, -2.2471, -0.4653, 3.0302, -0.1083])\n" - ] - } - ], - "source": [ - "print(activations[0][0][0:5])\n", - "print(activations[1][0][0:5])\n", - "print(activations[2][0][0:5])" - ] - }, - { - "cell_type": "code", - "execution_count": 103, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(22778, 120)\n", - "(120, 64)\n", - "(64, 120)\n", - "(120, 22778)\n" - ] - } - ], - "source": [ - "# adjusted the weights based on the activaition degrees (due to activaition function)\n", - "weights_adj_all = []\n", - "n_layers = len(model.mlp)\n", - "for i in range(n_layers):\n", - " layer = model.mlp[i]\n", - " if isinstance(layer, nn.Linear):\n", - " weight = layer.weight.detach().numpy().T\n", - " if i < (n_layers-1):\n", - " # weights_adj = weight \n", - " activation_per_node = activations[i+1].mean(axis=0, keepdims=True).numpy() #activation function is always one ahead\n", - " weights_adj = weight * activation_per_node\n", - " else:\n", - " weights_adj = weight\n", - " weights_adj_all.append(weights_adj)\n", - " print(weights_adj.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 444, - "metadata": {}, - "outputs": [], - "source": [ - "# calculate regulatory interactions\n", - "net = weights_adj_all[0]\n", - "for weights in weights_adj_all[1:]:\n", - " net = net @ weights\n", - "net_df = pd.DataFrame(net, columns=gene_names, index=gene_names)" - ] - }, - { - "cell_type": "code", - "execution_count": 448, - "metadata": {}, - "outputs": [], - "source": [ - "# net_df = pd.read_csv(\"../output/benchmark/grn_models/scenicplus.csv\")\n", - "# net_df = net_df.pivot(index='source', columns='target', values='weight').fillna(0)\n", - "# net_df = net_df.abs()\n", - "# net_df\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Evaluate GRN" - ] - }, - { - "cell_type": "code", - "execution_count": 452, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "n links: 4370736\n", - "top_n=5, aucroc=0.5082225495511123, precision_mean=0.006695495302780784, matt_mean=0.0005570626029492135, f1_mean=0.001858002846117856\n", - "n links: 4370736\n", - "top_n=10, aucroc=0.5082225495511123, precision_mean=0.006695495302780784, matt_mean=0.0005570626029492135, f1_mean=0.001858002846117856\n", - "n links: 4370736\n" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[452], line 42\u001b[0m\n\u001b[1;32m 39\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m y_true\u001b[38;5;241m.\u001b[39mshape\u001b[38;5;241m==\u001b[39my_pred\u001b[38;5;241m.\u001b[39mshape\n\u001b[1;32m 41\u001b[0m \u001b[38;5;66;03m# metrics\u001b[39;00m\n\u001b[0;32m---> 42\u001b[0m aucroc \u001b[38;5;241m=\u001b[39m \u001b[43mroc_auc_score\u001b[49m\u001b[43m(\u001b[49m\u001b[43my_true\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_pred\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 43\u001b[0m \u001b[38;5;66;03m# precision, recall, tt = precision_recall_curve(y_true, y_pred)\u001b[39;00m\n\u001b[1;32m 44\u001b[0m \u001b[38;5;66;03m# aucpr = (precision * recall).mean()\u001b[39;00m\n\u001b[1;32m 45\u001b[0m precision_mean \u001b[38;5;241m=\u001b[39m average_precision_score(y_true, y_pred)\n", - "File \u001b[0;32m~/.local/lib/python3.10/site-packages/sklearn/utils/_param_validation.py:213\u001b[0m, in \u001b[0;36mvalidate_params..decorator..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 207\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 208\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m config_context(\n\u001b[1;32m 209\u001b[0m skip_parameter_validation\u001b[38;5;241m=\u001b[39m(\n\u001b[1;32m 210\u001b[0m prefer_skip_nested_validation \u001b[38;5;129;01mor\u001b[39;00m global_skip_validation\n\u001b[1;32m 211\u001b[0m )\n\u001b[1;32m 212\u001b[0m ):\n\u001b[0;32m--> 213\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 214\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m InvalidParameterError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 215\u001b[0m \u001b[38;5;66;03m# When the function is just a wrapper around an estimator, we allow\u001b[39;00m\n\u001b[1;32m 216\u001b[0m \u001b[38;5;66;03m# the function to delegate validation to the estimator, but we replace\u001b[39;00m\n\u001b[1;32m 217\u001b[0m \u001b[38;5;66;03m# the name of the estimator by the name of the function in the error\u001b[39;00m\n\u001b[1;32m 218\u001b[0m \u001b[38;5;66;03m# message to avoid confusion.\u001b[39;00m\n\u001b[1;32m 219\u001b[0m msg \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39msub(\n\u001b[1;32m 220\u001b[0m \u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mparameter of \u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mw+ must be\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 221\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mparameter of \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfunc\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__qualname__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m must be\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 222\u001b[0m \u001b[38;5;28mstr\u001b[39m(e),\n\u001b[1;32m 223\u001b[0m )\n", - "File \u001b[0;32m~/.local/lib/python3.10/site-packages/sklearn/metrics/_ranking.py:617\u001b[0m, in \u001b[0;36mroc_auc_score\u001b[0;34m(y_true, y_score, average, sample_weight, max_fpr, multi_class, labels)\u001b[0m\n\u001b[1;32m 408\u001b[0m \u001b[38;5;129m@validate_params\u001b[39m(\n\u001b[1;32m 409\u001b[0m {\n\u001b[1;32m 410\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124my_true\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124marray-like\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 428\u001b[0m labels\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 429\u001b[0m ):\n\u001b[1;32m 430\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Compute Area Under the Receiver Operating Characteristic Curve (ROC AUC) \\\u001b[39;00m\n\u001b[1;32m 431\u001b[0m \u001b[38;5;124;03m from prediction scores.\u001b[39;00m\n\u001b[1;32m 432\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 614\u001b[0m \u001b[38;5;124;03m array([0.81..., 0.84... , 0.93..., 0.87..., 0.94...])\u001b[39;00m\n\u001b[1;32m 615\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 617\u001b[0m y_type \u001b[38;5;241m=\u001b[39m \u001b[43mtype_of_target\u001b[49m\u001b[43m(\u001b[49m\u001b[43my_true\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minput_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43my_true\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 618\u001b[0m y_true \u001b[38;5;241m=\u001b[39m check_array(y_true, ensure_2d\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, dtype\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m 619\u001b[0m y_score \u001b[38;5;241m=\u001b[39m check_array(y_score, ensure_2d\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n", - "File \u001b[0;32m~/.local/lib/python3.10/site-packages/sklearn/utils/multiclass.py:395\u001b[0m, in \u001b[0;36mtype_of_target\u001b[0;34m(y, input_name)\u001b[0m\n\u001b[1;32m 393\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m issparse(first_row):\n\u001b[1;32m 394\u001b[0m first_row \u001b[38;5;241m=\u001b[39m first_row\u001b[38;5;241m.\u001b[39mdata\n\u001b[0;32m--> 395\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[43mxp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43munique_values\u001b[49m\u001b[43m(\u001b[49m\u001b[43my\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m2\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m (y\u001b[38;5;241m.\u001b[39mndim \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m2\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(first_row) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m):\n\u001b[1;32m 396\u001b[0m \u001b[38;5;66;03m# [1, 2, 3] or [[1., 2., 3]] or [[1, 2]]\u001b[39;00m\n\u001b[1;32m 397\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmulticlass\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m suffix\n\u001b[1;32m 398\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m~/.local/lib/python3.10/site-packages/sklearn/utils/_array_api.py:307\u001b[0m, in \u001b[0;36m_NumPyAPIWrapper.unique_values\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 306\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21munique_values\u001b[39m(\u001b[38;5;28mself\u001b[39m, x):\n\u001b[0;32m--> 307\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mnumpy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43munique\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/.local/lib/python3.10/site-packages/numpy/lib/arraysetops.py:274\u001b[0m, in \u001b[0;36munique\u001b[0;34m(ar, return_index, return_inverse, return_counts, axis, equal_nan)\u001b[0m\n\u001b[1;32m 272\u001b[0m ar \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39masanyarray(ar)\n\u001b[1;32m 273\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m axis \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 274\u001b[0m ret \u001b[38;5;241m=\u001b[39m \u001b[43m_unique1d\u001b[49m\u001b[43m(\u001b[49m\u001b[43mar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreturn_index\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreturn_inverse\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreturn_counts\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 275\u001b[0m \u001b[43m \u001b[49m\u001b[43mequal_nan\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mequal_nan\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 276\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _unpack_tuple(ret)\n\u001b[1;32m 278\u001b[0m \u001b[38;5;66;03m# axis was specified and not None\u001b[39;00m\n", - "File \u001b[0;32m~/.local/lib/python3.10/site-packages/numpy/lib/arraysetops.py:336\u001b[0m, in \u001b[0;36m_unique1d\u001b[0;34m(ar, return_index, return_inverse, return_counts, equal_nan)\u001b[0m\n\u001b[1;32m 334\u001b[0m aux \u001b[38;5;241m=\u001b[39m ar[perm]\n\u001b[1;32m 335\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 336\u001b[0m \u001b[43mar\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msort\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 337\u001b[0m aux \u001b[38;5;241m=\u001b[39m ar\n\u001b[1;32m 338\u001b[0m mask \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mempty(aux\u001b[38;5;241m.\u001b[39mshape, dtype\u001b[38;5;241m=\u001b[39mnp\u001b[38;5;241m.\u001b[39mbool_)\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], - "source": [ - "from sklearn.metrics import roc_auc_score, precision_recall_curve, auc, f1_score, matthews_corrcoef, average_precision_score\n", - "\n", - "def metric_curve(y_true, y_scores):\n", - " tt = np.linspace(0, y_scores.max(), 20)\n", - " matt_scores = []\n", - " f1_scores = []\n", - " # print(tt)\n", - " for t in tt[:-1]:\n", - " y_pred = y_scores>t\n", - " f1_scores.append(f1_score(y_true, y_pred))\n", - " matt_scores.append(matthews_corrcoef(y_true, y_pred))\n", - " \n", - " return matt_scores, f1_scores\n", - "\n", - "for top_n in [5, 10, 100]:\n", - " # choose the top n genes with highest regulators\n", - " n_regulators = GT.sum(axis=0)\n", - " # GT_subset = GT[n_regulators.nlargest(top_n).index]\n", - " GT_subset = GT.copy()\n", - "\n", - " # only those tfs and targets given in GT\n", - " def match_dfs(df1, df2):\n", - " df1 = df1.loc[df1.index.isin(df2.index), df1.columns.isin(df2.columns)]\n", - " df2 = df2.loc[df2.index.isin(df1.index), df2.columns.isin(df1.columns)]\n", - " return df1, df2\n", - " net_subset, GT_subset = match_dfs(net_df, GT_subset)\n", - " # reindex for compatibility\n", - " net_subset = net_subset.reindex(index=GT_subset.index, columns=GT_subset.columns)\n", - "\n", - " y_pred = net_subset.values.flatten()\n", - " y_true = GT_subset.values.flatten()\n", - "\n", - " # no negative reg\n", - " y_pred = np.abs(y_pred)\n", - "\n", - " print('n links:',len(y_pred))\n", - "\n", - " assert np.isnan(y_pred).any()==False\n", - " assert y_true.shape==y_pred.shape\n", - "\n", - " # metrics\n", - " aucroc = roc_auc_score(y_true, y_pred)\n", - " # precision, recall, tt = precision_recall_curve(y_true, y_pred)\n", - " # aucpr = (precision * recall).mean()\n", - " precision_mean = average_precision_score(y_true, y_pred)\n", - "\n", - " matt_scores, f1_scores = metric_curve(y_true, y_pred)\n", - " matt_mean = np.mean(matt_scores)\n", - " f1_mean = np.mean(f1_scores)\n", - "\n", - " print(f'{top_n=}, {aucroc=}, {precision_mean=}, {matt_mean=}, {f1_mean=}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "n links: 303\n", - "top_n=5, aucroc=0.5264077669902912, precision_mean=0.3669703794235179, matt_mean=0.10818906873658679, f1_mean=0.08888404098938463\n", - "n links: 707\n", - "top_n=10, aucroc=0.5187080867850099, precision_mean=0.3046555354494385, matt_mean=0.09383949611263599, f1_mean=0.07612230579002001\n", - "n links: 7272\n", - "top_n=100, aucroc=0.5232494638773373, precision_mean=0.1546841109780841, matt_mean=0.059091840774979656, f1_mean=0.036139223144212335\n", - "n links: 74235\n", - "top_n=1000, aucroc=0.532563048262141, precision_mean=0.05838751393542791, matt_mean=0.06267851369070392, f1_mean=0.042948548762372384\n", - "n links: 370670\n", - "top_n=5000, aucroc=0.5308883277329034, precision_mean=0.021350126982468423, matt_mean=0.045122637323227084, f1_mean=0.028030001732845523\n" - ] - }, - { - "cell_type": "code", - "execution_count": 109, - "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'aa' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[109], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43maa\u001b[49m\n", - "\u001b[0;31mNameError\u001b[0m: name 'aa' is not defined" - ] - } - ], - "source": [ - "aa" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from local_utils import plots\n", - "plots.plot_cumulative_density(reg_net.flatten())\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Saliency map approach" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# !pip install captum" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/root/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "from captum.attr import IntegratedGradients" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Initialize NN explainer\n", - "# baseline = torch.zeros((1, X.size()[1], X.size()[2]))\n", - "# if USE_GPU:\n", - "# baseline = baseline.cuda()\n", - "ig = IntegratedGradients(model)\n", - "\n", - "# Sort the columns of the goldstandard network based on the coverage of annotations\n", - "idx = np.argsort(np.sum(grn, axis=0))[::-1]\n", - "\n", - "all_z_target, all_z_pred = [], []\n", - "for k in range(5):\n", - " \n", - " # Select column `j` from the goldstandard GRN\n", - " j = int(idx[k])\n", - " z_target = grn[:, j]\n", - " z_target[j] = np.nan # Assume that a gene cannot regulate itself -> will be filtered out later\n", - " all_z_target.append(z_target)\n", - " \n", - " # Compute feature attribution vectors for many input points and average over the results\n", - " z_pred = np.zeros(len(gene_names))\n", - " for i in tqdm.tqdm(range(1000)):\n", - " # Generate random input based on the null distribution\n", - " input_ = background_noise(X.size()[1], X.size()[2], device=baseline.device)\n", - " \n", - " # Compute feature attribution\n", - " attributions = ig.attribute(input_, baseline, target=j, return_convergence_delta=False)\n", - " # An alternative is to use input samples from the training set:\n", - " # attributions = ig.attribute(X[i, ...], baseline, target=j, return_convergence_delta=False)\n", - " # By averaging across all samples from the same cell type, one can build a biological network\n", - " # that is cell type-specific.\n", - " \n", - " z_pred += np.mean(np.abs(np.squeeze(attributions.cpu().data.numpy())), axis=1)\n", - " all_z_pred.append(z_pred[grn_mask])\n", - " \n", - "# Compute performance metrics\n", - "all_z_target = np.concatenate(all_z_target, axis=0)\n", - "all_z_pred = np.concatenate(all_z_pred, axis=0)\n", - "mask = ~np.isnan(all_z_target) # Discard self-regulations\n", - "print(f'AUROC: {roc_auc_score(all_z_target[mask], all_z_pred[mask])}')\n", - "print(f'AUPR : {average_precision_score(all_z_target[mask], all_z_pred[mask])}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - " # predict\n", - "# model.eval()\n", - "# y_pred_submit, mu, log_var = model(torch.tensor(X_submit, dtype=torch.float32, device='cuda'))\n", - "# y_pred_submit = y_pred_submit.cpu().detach().numpy()\n", - "# y_pred_submit_all.append(y_pred_submit)\n", - "\n", - "# print('r2:', r2_score(df_test, y_pred_submit, multioutput='variance_weighted'))\n", - "# models.append(model)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2024-06-03T16:28:02.331255Z", - "iopub.status.busy": "2024-06-03T16:28:02.330801Z", - "iopub.status.idle": "2024-06-03T16:28:02.342089Z", - "shell.execute_reply": "2024-06-03T16:28:02.341161Z", - "shell.execute_reply.started": "2024-06-03T16:28:02.331213Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[1., 0., 0., ..., 0., 0., 0.],\n", - " [0., 1., 0., ..., 0., 0., 0.],\n", - " [0., 0., 1., ..., 0., 0., 0.],\n", - " ...,\n", - " [0., 0., 0., ..., 1., 0., 0.],\n", - " [0., 0., 0., ..., 0., 1., 0.],\n", - " [0., 0., 0., ..., 0., 0., 1.]], device='cuda:0')" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.A" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Predict" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.status.busy": "2024-06-03T15:09:34.795825Z", - "iopub.status.idle": "2024-06-03T15:09:34.801895Z", - "shell.execute_reply": "2024-06-03T15:09:34.801613Z", - "shell.execute_reply.started": "2024-06-03T15:09:34.801578Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "print('number of models:', len(y_pred_submit_all))\n", - "y_pred_submit_mean = np.mean(y_pred_submit_all, axis=0)\n", - "print('r2:', r2_score(df_test, y_pred_submit_mean, multioutput='variance_weighted'))\n", - "print('mse:', mean_squared_error(y_pred_submit_mean, df_test.values))\n", - "print('mrrmse:', mrrmse(y_pred_submit_mean, df_test.values))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.status.busy": "2024-06-03T15:09:34.809004Z", - "iopub.status.idle": "2024-06-03T15:09:34.813749Z", - "shell.execute_reply": "2024-06-03T15:09:34.813459Z", - "shell.execute_reply.started": "2024-06-03T15:09:34.813421Z" - } - }, - "outputs": [], - "source": [ - "r2: 0.24227989212421114\n", - "mse: 1.7873100056454099\n", - "mrrmse: 0.8062594076816958" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Baseline RF" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.status.busy": "2024-06-03T15:09:34.830931Z", - "iopub.status.idle": "2024-06-03T15:09:34.835443Z", - "shell.execute_reply": "2024-06-03T15:09:34.835119Z", - "shell.execute_reply.started": "2024-06-03T15:09:34.835076Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "def encode(df_train, df_test, feature_space):\n", - " # encode each factor\n", - " x_encoded_dict = {}\n", - " for feature in feature_space:\n", - " index = df_train.index.get_level_values(feature)\n", - " n_com = min([len(index.unique()), 50])\n", - " # var_x = TruncatedSVD(n_components=n_com, n_iter=12, random_state=random_state).fit_transform(df_train)\n", - " var_x = KernelPCA(n_components=n_com, kernel='linear', random_state=random_state).fit_transform(df_train)\n", - " x_encoded = pd.DataFrame(var_x, index=index).reset_index()\n", - " x_encoded = x_encoded.groupby(feature).mean()\n", - " x_encoded_dict[feature] = x_encoded\n", - " # create X and X_submit\n", - " X = []\n", - " X_submit = []\n", - " for i_feature, feature in enumerate(feature_space):\n", - " # encode train data\n", - " index = df_train.index.get_level_values(feature)\n", - " feature_encoded = np.asarray([x_encoded_dict[feature].loc[name].values for name in index])\n", - " if i_feature == 0:\n", - " X = feature_encoded\n", - " else: \n", - " X = np.concatenate([X, feature_encoded], axis=1)\n", - " \n", - " # encode test data\n", - " index = df_test.index.get_level_values(feature)\n", - " feature_encoded = np.asarray([x_encoded_dict[feature].loc[name].values for name in index])\n", - " if i_feature == 0:\n", - " X_submit = feature_encoded\n", - " else: \n", - " X_submit = np.concatenate([X_submit, feature_encoded], axis=1)\n", - " return X, X_submit\n", - "\n", - "\n", - "random_state = 32\n", - "n_components = 50\n", - "X_rf, X_submit_rf = encode(df_train, df_test, features_X)\n", - "emb_model = RandomForestRegressor(n_estimators=100, random_state=random_state)\n", - "reducer = TruncatedSVD(n_components=n_components, n_iter=12, random_state=random_state)\n", - "Y = reducer.fit_transform(df_train)\n", - "\n", - "emb_model.fit(X_rf, Y)\n", - "y_pred_submit = reducer.inverse_transform(emb_model.predict(X_submit_rf))\n", - "\n", - "print('r2:', r2_score(df_test, y_pred_submit, multioutput='variance_weighted'))\n", - "print('mse:', mean_squared_error(y_pred_submit, df_test.values))\n", - "print('mrrmse:', mrrmse(y_pred_submit, df_test.values))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.status.busy": "2024-06-03T15:09:34.839314Z", - "iopub.status.idle": "2024-06-03T15:09:34.844806Z", - "shell.execute_reply": "2024-06-03T15:09:34.844512Z", - "shell.execute_reply.started": "2024-06-03T15:09:34.844480Z" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "r2: 0.24227989212421114\n", - "mse: 1.7873100056454099\n", - "mrrmse: 0.8062594076816958" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "saturn (Python 3)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.19" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/runs.ipynb b/runs.ipynb index cd1c593f8..b3761d7cd 100644 --- a/runs.ipynb +++ b/runs.ipynb @@ -5,37 +5,6 @@ "metadata": {}, "source": [] }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": {}, - "outputs": [], - "source": [ - "# collectRI = pd.read_csv(\"https://github.com/pablormier/omnipath-static/raw/main/op/collectri-26.09.2023.zip\")\n", - "# collectRI.to_csv(f'resources/grn_models/d0_hvgs/collectri.csv')\n", - "# collectRI.to_csv(f'resources/grn_models/default/collectri.csv')" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "multiomics_atac_d0.h5ad multiomics_rna_d0.h5ad multiomics_rna.rds\n", - "multiomics_atac.h5ad\t multiomics_rna_d0_hvg.h5ad perturbation_data.h5ad\n", - "multiomics_atac.rds\t multiomics_rna.h5ad\n", - "multiomics_rna_0.h5ad\t multiomics_rna_hvg.h5ad\n" - ] - } - ], - "source": [ - "!ls resources/grn-benchmark//" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -45,17 +14,38 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "upload: resources/grn_models/default/negative_control.csv to s3://openproblems-data/resources/grn/grn_models/default/negative_control.csv\n", - "upload: resources/grn_models/d0_hvgs/negative_control.csv to s3://openproblems-data/resources/grn/grn_models/d0_hvgs/negative_control.csv\n", - "upload: resources/results/d0_hvgs/trace.txt to s3://openproblems-data/resources/grn/results/d0_hvgs/trace.txt\n", - "delete: s3://openproblems-data/resources/grn/results/d0_hvgs_baseline/trace.txt\n" + "upload: resources/results/robustness_analysis/corr/scores_corr.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/corr/scores_corr.csv\n", + "upload: resources/results/robustness_analysis/net-100-scores.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/net-100-scores.csv\n", + "upload: resources/results/robustness_analysis/sign-10-scores.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/sign-10-scores.csv\n", + "upload: resources/results/robustness_analysis/sign-100-scores.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/sign-100-scores.csv\n", + "upload: resources/results/robustness_analysis/net-50-scores.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/net-50-scores.csv\n", + "upload: resources/results/robustness_analysis/sign-50-scores.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/sign-50-scores.csv\n", + "upload: resources/results/robustness_analysis/sign-20-scores.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/sign-20-scores.csv\n", + "upload: resources/results/robustness_analysis/sign-0-scores.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/sign-0-scores.csv\n", + "upload: resources/results/robustness_analysis/tmp/genie3.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/tmp/genie3.csv\n", + "upload: resources/results/robustness_analysis/tmp/pearson_corr.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/tmp/pearson_corr.csv\n", + "upload: resources/results/robustness_analysis/tmp/negative_control.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/tmp/negative_control.csv\n", + "upload: resources/results/robustness_analysis/tmp/positive_control.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/tmp/positive_control.csv\n", + "delete: s3://openproblems-data/resources/grn/results/scores/layers/scgen_pearson-GB.csv\n", + "upload: resources/results/robustness_analysis/tmp/collectri.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/tmp/collectri.csv\n", + "upload: resources/results/robustness_analysis/tmp/scglue.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/tmp/scglue.csv\n", + "upload: resources/results/robustness_analysis/tmp/pearson_causal.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/tmp/pearson_causal.csv\n", + "upload: resources/results/robustness_analysis/tmp/ppcor.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/tmp/ppcor.csv\n", + "upload: resources/results/robustness_analysis/tmp/portia.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/tmp/portia.csv\n", + "upload: resources/results/scores/layers/seurat_lognorm-ridge.csv to s3://openproblems-data/resources/grn/results/scores/layers/seurat_lognorm-ridge.csv\n", + "upload: resources/results/robustness_analysis/corr/corr.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/corr/corr.csv\n", + "upload: resources/results/scores/layers/seurat_pearson-ridge.csv to s3://openproblems-data/resources/grn/results/scores/layers/seurat_pearson-ridge.csv\n", + "upload: resources/results/scores/scgen_pearson-GB.csv to s3://openproblems-data/resources/grn/results/scores/scgen_pearson-GB.csv\n", + "upload: resources/results/robustness_analysis/tmp/scenic.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/tmp/scenic.csv\n", + "upload: resources/results/robustness_analysis/tmp/grnboost2.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/tmp/grnboost2.csv\n", + "upload: resources/results/robustness_analysis/tmp/celloracle.csv to s3://openproblems-data/resources/grn/results/robustness_analysis/tmp/celloracle.csv\n" ] } ], @@ -76,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -131,7 +121,10 @@ " # df_all_n = (df_all-df_all.min(axis=0))/(df_all.max(axis=0)-df_all.min(axis=0))\n", " # df_all['Rank'] = df_all_n.mean(axis=1).rank(ascending=False).astype(int)\n", " df_all = pd.concat([df_reg1, df_reg2], axis=1)\n", - " return df_all" + " return df_all\n", + "\n", + "\n", + "methods = [ 'collectri', 'negative_control', 'positive_control', 'pearson_corr', 'pearson_causal', 'portia', 'ppcor', 'genie3', 'grnboost2', 'scenic', 'scglue', 'celloracle']" ] }, { @@ -187,17 +180,26 @@ "metadata": {}, "outputs": [], "source": [ - "methods = [ 'collectri', 'negative_control', 'positive_control', 'pearson_corr', 'pearson_causal', 'portia', 'ppcor', 'genie3', 'grnboost2', 'scenic', 'scglue', 'celloracle']" + "# consensus: run this after updating method list\n", + "# !python src/metrics/regression_2/consensus/script.py #inside the method names and dir\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Submitted batch job 7746215\n" + ] + } + ], "source": [ - "# consensus: run this after updating method list\n", - "# !python src/metrics/regression_2/consensus/script.py #inside the method names and dir\n" + "# run calculating scores\n", + "!sbatch scripts/sbatch/calculate_scores.sh #includes both reg1 and 2. #inside the script, set the reg_type, read and write dirs, and methods" ] }, { @@ -209,19 +211,315 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 16, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Submitted batch job 7746169\n" - ] + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
 S1S2static-theta-0.0static-theta-0.5rank
collectri0.0204930.0268580.7192520.68098610
negative_control-0.013036-0.0131560.5437550.68329012
positive_control0.5107300.7031130.8119190.7055272
pearson_corr0.2088730.4423420.8277720.6840627
pearson_causal0.3684270.6044990.8702870.6998224
portia0.1389370.2124340.7058150.6861838
ppcor0.0155420.0325680.7356340.67534211
genie30.3781560.5052600.8694200.7144473
grnboost20.4001490.4741120.8122130.7398531
scenic0.1531750.1860010.8345980.7315486
scglue0.0704320.2057680.6478110.6870269
celloracle0.2546490.3630530.8048830.7207365
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "!sbatch scripts/sbatch/calculate_scores.sh #includes both reg1 and 2. #inside the script, set the reg_type, read and write dirs, and methods" + "df_scores_GB = pd.read_csv(\"resources/results/scores/hvg_GB.csv\", index_col=0)\n", + "df_all_n = (df_scores_GB-df_scores_GB.min(axis=0))/(df_scores_GB.max(axis=0)-df_scores_GB.min(axis=0))\n", + "df_scores_GB['rank'] = df_all_n.mean(axis=1).rank(ascending=False).astype(int)\n", + "df_scores_GB.style.background_gradient()" ] }, { @@ -233,271 +531,322 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
 S1S2static-theta-0.0static-theta-0.5S1S2static-theta-0.0static-theta-0.5rank
collectri-0.100238-0.2111820.4893160.514896collectri-0.100238-0.2111820.4893160.51489612
negative_control-0.043795-0.0455610.4590800.505002negative_control-0.043795-0.0455610.4590800.50500211
positive_control0.4891470.6771550.6554070.574608positive_control0.4891470.6771550.6554070.5746082
pearson_corr0.2386640.5146120.5295020.524232pearson_corr0.2386640.5146120.5295020.5242327
pearson_causal0.3552560.5787530.7413280.560490pearson_causal0.3552560.5787530.7413280.5604904
portia0.1489410.2272480.4512560.518048portia0.1489410.2272480.4512560.5180488
ppcor0.0228460.0941070.3966800.509874ppcor0.0228460.0941070.3966800.50987410
genie30.3726410.4903570.7540730.576580genie30.3726410.4903570.7540730.5765803
grnboost20.3810320.4598600.7818520.609075grnboost20.3810320.4598600.7818520.6090751
scenic0.1475530.2146940.6008390.574294scenic0.1475530.2146940.6008390.5742946
scglue0.0783090.2388590.4486170.527076scglue0.0783090.2388590.4486170.5270769
celloracle0.2168970.3114510.6395560.580147celloracle0.2168970.3114510.6395560.5801475
\n" ], "text/plain": [ - "" + "" ] }, - "execution_count": 16, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df_scores = pd.read_csv('resources/results/scores/ridge.csv', index_col=0)\n", + "df_scores = pd.read_csv('resources/results/scores/hvg_ridge.csv', index_col=0)\n", + "df_all_n = (df_scores-df_scores.min(axis=0))/(df_scores.max(axis=0)-df_scores.min(axis=0))\n", + "df_scores['rank'] = df_all_n.mean(axis=1).rank(ascending=False).astype(int)\n", "df_scores.style.background_gradient()" ] }, @@ -517,24 +866,24 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/vol/tmp/users/jnourisa/ipykernel_1858346/2555768217.py:19: FutureWarning: The 'delim_whitespace' keyword in pd.read_csv is deprecated and will be removed in a future version. Use ``sep='\\s+'`` instead\n", + "/vol/tmp/users/jnourisa/ipykernel_721136/2424713973.py:19: FutureWarning: The 'delim_whitespace' keyword in pd.read_csv is deprecated and will be removed in a future version. Use ``sep='\\s+'`` instead\n", " df = pd.read_csv(io.StringIO(output), delim_whitespace=True)\n", - "/vol/tmp/users/jnourisa/ipykernel_1858346/2555768217.py:19: FutureWarning: The 'delim_whitespace' keyword in pd.read_csv is deprecated and will be removed in a future version. Use ``sep='\\s+'`` instead\n", + "/vol/tmp/users/jnourisa/ipykernel_721136/2424713973.py:19: FutureWarning: The 'delim_whitespace' keyword in pd.read_csv is deprecated and will be removed in a future version. Use ``sep='\\s+'`` instead\n", " df = pd.read_csv(io.StringIO(output), delim_whitespace=True)\n", - "/vol/tmp/users/jnourisa/ipykernel_1858346/2555768217.py:19: FutureWarning: The 'delim_whitespace' keyword in pd.read_csv is deprecated and will be removed in a future version. Use ``sep='\\s+'`` instead\n", + "/vol/tmp/users/jnourisa/ipykernel_721136/2424713973.py:19: FutureWarning: The 'delim_whitespace' keyword in pd.read_csv is deprecated and will be removed in a future version. Use ``sep='\\s+'`` instead\n", " df = pd.read_csv(io.StringIO(output), delim_whitespace=True)\n", - "/vol/tmp/users/jnourisa/ipykernel_1858346/2555768217.py:19: FutureWarning: The 'delim_whitespace' keyword in pd.read_csv is deprecated and will be removed in a future version. Use ``sep='\\s+'`` instead\n", + "/vol/tmp/users/jnourisa/ipykernel_721136/2424713973.py:19: FutureWarning: The 'delim_whitespace' keyword in pd.read_csv is deprecated and will be removed in a future version. Use ``sep='\\s+'`` instead\n", " df = pd.read_csv(io.StringIO(output), delim_whitespace=True)\n", - "/vol/tmp/users/jnourisa/ipykernel_1858346/2555768217.py:19: FutureWarning: The 'delim_whitespace' keyword in pd.read_csv is deprecated and will be removed in a future version. Use ``sep='\\s+'`` instead\n", + "/vol/tmp/users/jnourisa/ipykernel_721136/2424713973.py:19: FutureWarning: The 'delim_whitespace' keyword in pd.read_csv is deprecated and will be removed in a future version. Use ``sep='\\s+'`` instead\n", " df = pd.read_csv(io.StringIO(output), delim_whitespace=True)\n", - "/vol/tmp/users/jnourisa/ipykernel_1858346/2555768217.py:19: FutureWarning: The 'delim_whitespace' keyword in pd.read_csv is deprecated and will be removed in a future version. Use ``sep='\\s+'`` instead\n", + "/vol/tmp/users/jnourisa/ipykernel_721136/2424713973.py:19: FutureWarning: The 'delim_whitespace' keyword in pd.read_csv is deprecated and will be removed in a future version. Use ``sep='\\s+'`` instead\n", " df = pd.read_csv(io.StringIO(output), delim_whitespace=True)\n" ] }, @@ -571,13 +920,13 @@ " \n", " \n", " portia\n", - " 7742241.bat+\n", + " 7744548.bat+\n", " batch\n", " 20\n", - " 0.110556\n", + " 0.153611\n", " COMPLETED\n", - " 46.943497\n", - " 47.474384\n", + " 5.854904\n", + " 6.284901\n", " \n", " \n", " grnboost2\n", @@ -635,7 +984,7 @@ ], "text/plain": [ " JobID JobName AllocCPUS Elapsed State MaxRSS \\\n", - "portia 7742241.bat+ batch 20 0.110556 COMPLETED 46.943497 \n", + "portia 7744548.bat+ batch 20 0.153611 COMPLETED 5.854904 \n", "grnboost2 7742249.bat+ batch 20 1.568056 COMPLETED 3.067471 \n", "scenic 7742283.bat+ batch 20 1.908056 COMPLETED 30.356461 \n", "genie3 7742285.bat+ batch 20 16.682500 COMPLETED 13.105103 \n", @@ -643,7 +992,7 @@ "scglue 7742343.bat+ batch 20 4.380278 FAILED 29.917423 \n", "\n", " MaxVMSize \n", - "portia 47.474384 \n", + "portia 6.284901 \n", "grnboost2 3.563801 \n", "scenic 32.573463 \n", "genie3 13.563530 \n", @@ -651,7 +1000,7 @@ "scglue 35.933720 " ] }, - "execution_count": 6, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -698,6 +1047,7 @@ "# Convert MaxRSS and MaxVMSize from KB to GB\n", "df_local['MaxRSS'] = df_local['MaxRSS'] / (1024 ** 2) # Convert KB to GB\n", "df_local['MaxVMSize'] = df_local['MaxVMSize'] / (1024 ** 2) # Convert KB to GB\n", + "df_local.to_csv('resources/results/trace/local_hvg.csv')\n", "df_local" ] }, @@ -710,16 +1060,16 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 30, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/vol/tmp/users/jnourisa/ipykernel_1858346/2779943256.py:4: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n", + "/vol/tmp/users/jnourisa/ipykernel_721136/2779943256.py:4: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n", " trace = trace.groupby('model').apply(lambda df: df.sort_values(by='duration', ascending=False).iloc[0])[['%cpu', 'peak_rss', 'peak_vmem', 'rchar', 'wchar', 'duration']]\n", - "/vol/tmp/users/jnourisa/ipykernel_1858346/2779943256.py:4: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n", + "/vol/tmp/users/jnourisa/ipykernel_721136/2779943256.py:4: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n", " trace = trace.groupby('model').apply(lambda df: df.sort_values(by='duration', ascending=False).iloc[0])[['%cpu', 'peak_rss', 'peak_vmem', 'rchar', 'wchar', 'duration']]\n" ] }, @@ -811,7 +1161,7 @@ "celloracle 799.6 14.9000 35.4 18.0000 86.1000 1.472222" ] }, - "execution_count": 7, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } @@ -854,21 +1204,22 @@ " else:\n", " trace[col] = trace[col].apply(format_ram)\n", " return trace \n", + "if False: # already processed\n", + " models = ['pearson_causal', 'pearson_corr', 'positive_control']\n", + " base_dir = 'resources/results/d0_hvgs_baseline'\n", + " trace = pd.read_csv(f'{base_dir}/trace.txt', sep='\\t')\n", "\n", - "models = ['pearson_causal', 'pearson_corr', 'positive_control']\n", - "base_dir = 'resources/results/d0_hvgs_baseline'\n", - "trace = pd.read_csv(f'{base_dir}/trace.txt', sep='\\t')\n", - "\n", - "trace_baselines = process_trace(trace)\n", + " trace_baselines = process_trace(trace)\n", "\n", - "models = ['celloracle']\n", - "base_dir = 'resources/results/celloracle_d0_hvgs'\n", - "trace = pd.read_csv(f'{base_dir}/trace.txt', sep='\\t')\n", - "trace_models = process_trace(trace)\n", + " models = ['celloracle']\n", + " base_dir = 'resources/results/celloracle_d0_hvgs'\n", + " trace = pd.read_csv(f'{base_dir}/trace.txt', sep='\\t')\n", + " trace_models = process_trace(trace)\n", "\n", "\n", - "trace_seqera = pd.concat([trace_baselines, trace_models], axis=0)\n", - "trace_seqera" + " trace_seqera = pd.concat([trace_baselines, trace_models], axis=0)\n", + " trace_seqera.to_csv('resources/results/trace/trace_seqera_hvg.csv')\n", + " trace_seqera" ] }, { @@ -880,7 +1231,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 32, "metadata": {}, "outputs": [ { @@ -931,8 +1282,8 @@ " \n", " \n", " portia\n", - " 46.943497\n", - " 0.110556\n", + " 5.854904\n", + " 0.153611\n", " \n", " \n", " grnboost2\n", @@ -969,7 +1320,7 @@ "pearson_corr 0.975100 0.072500\n", "positive_control 4.900000 0.075278\n", "celloracle 14.900000 1.472222\n", - "portia 46.943497 0.110556\n", + "portia 5.854904 0.153611\n", "grnboost2 3.067471 1.568056\n", "scenic 30.356461 1.908056\n", "genie3 13.105103 16.682500\n", @@ -977,7 +1328,7 @@ "scglue 29.917423 4.380278" ] }, - "execution_count": 8, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } @@ -999,6 +1350,15 @@ "df_res" ] }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "df_res.to_csv('resources/results/trace/trace_hvg.csv')" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -1008,7 +1368,17 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "df_res = pd.read_csv('resources/results/trace/trace_hvg.csv', index_col=0)\n", + "df_scores = pd.read_csv('resources/results/scores/hvg_ridge.csv', index_col=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 39, "metadata": {}, "outputs": [ { @@ -1033,8 +1403,8 @@ " \n", " \n", " method_name\n", - " ex(False)_tf(-1)\n", - " ex(True)_tf(-1)\n", + " S1\n", + " S2\n", " static-theta-0.0\n", " static-theta-0.5\n", " overall_score\n", @@ -1050,7 +1420,7 @@ " 0.000000\n", " 0.240505\n", " 0.095068\n", - " 0.077646\n", + " 0.083893\n", " 0.000000\n", " 0.000000\n", " \n", @@ -1061,7 +1431,7 @@ " 0.000000\n", " 0.162005\n", " 0.000000\n", - " 0.032401\n", + " 0.040501\n", " 0.000000\n", " 0.000000\n", " \n", @@ -1072,7 +1442,7 @@ " 1.000000\n", " 0.671716\n", " 0.668815\n", - " 0.860974\n", + " 0.835133\n", " 4.900000\n", " 0.075278\n", " \n", @@ -1083,7 +1453,7 @@ " 0.759962\n", " 0.344838\n", " 0.184772\n", - " 0.453523\n", + " 0.444373\n", " 0.975100\n", " 0.072500\n", " \n", @@ -1094,7 +1464,7 @@ " 0.854683\n", " 0.894789\n", " 0.533161\n", - " 0.774527\n", + " 0.752228\n", " 0.974700\n", " 0.064167\n", " \n", @@ -1105,9 +1475,9 @@ " 0.335592\n", " 0.141693\n", " 0.125352\n", - " 0.226639\n", - " 46.943497\n", - " 0.110556\n", + " 0.226782\n", + " 5.854904\n", + " 0.153611\n", " \n", " \n", " 6\n", @@ -1116,7 +1486,7 @@ " 0.138974\n", " 0.000000\n", " 0.046814\n", - " 0.050776\n", + " 0.058123\n", " 3.909119\n", " 0.556667\n", " \n", @@ -1127,7 +1497,7 @@ " 0.724142\n", " 0.927879\n", " 0.687766\n", - " 0.798691\n", + " 0.775401\n", " 13.105103\n", " 16.682500\n", " \n", @@ -1138,18 +1508,18 @@ " 0.679105\n", " 1.000000\n", " 1.000000\n", - " 0.891616\n", + " 0.864520\n", " 3.067471\n", " 1.568056\n", " \n", " \n", " 9\n", " scenic\n", - " 0.295812\n", - " 0.305056\n", + " 0.301653\n", + " 0.317053\n", " 0.530045\n", " 0.665799\n", - " 0.458534\n", + " 0.453637\n", " 30.356461\n", " 1.908056\n", " \n", @@ -1160,7 +1530,7 @@ " 0.352738\n", " 0.134840\n", " 0.212101\n", - " 0.214294\n", + " 0.214943\n", " 29.917423\n", " 4.380278\n", " \n", @@ -1171,7 +1541,7 @@ " 0.459941\n", " 0.630565\n", " 0.722041\n", - " 0.578251\n", + " 0.563992\n", " 14.900000\n", " 1.472222\n", " \n", @@ -1180,36 +1550,36 @@ "" ], "text/plain": [ - " method_name ex(False)_tf(-1) ex(True)_tf(-1) static-theta-0.0 \\\n", - "0 collectri 0.000000 0.000000 0.240505 \n", - "1 negative_control 0.000000 0.000000 0.162005 \n", - "2 positive_control 1.000000 1.000000 0.671716 \n", - "3 pearson_corr 0.487919 0.759962 0.344838 \n", - "4 pearson_causal 0.726277 0.854683 0.894789 \n", - "5 portia 0.304491 0.335592 0.141693 \n", - "6 ppcor 0.046705 0.138974 0.000000 \n", - "7 genie3 0.761818 0.724142 0.927879 \n", - "8 grnboost2 0.778974 0.679105 1.000000 \n", - "9 scenic 0.295812 0.305056 0.530045 \n", - "10 scglue 0.160093 0.352738 0.134840 \n", - "11 celloracle 0.443420 0.459941 0.630565 \n", + " method_name S1 S2 static-theta-0.0 static-theta-0.5 \\\n", + "0 collectri 0.000000 0.000000 0.240505 0.095068 \n", + "1 negative_control 0.000000 0.000000 0.162005 0.000000 \n", + "2 positive_control 1.000000 1.000000 0.671716 0.668815 \n", + "3 pearson_corr 0.487919 0.759962 0.344838 0.184772 \n", + "4 pearson_causal 0.726277 0.854683 0.894789 0.533161 \n", + "5 portia 0.304491 0.335592 0.141693 0.125352 \n", + "6 ppcor 0.046705 0.138974 0.000000 0.046814 \n", + "7 genie3 0.761818 0.724142 0.927879 0.687766 \n", + "8 grnboost2 0.778974 0.679105 1.000000 1.000000 \n", + "9 scenic 0.301653 0.317053 0.530045 0.665799 \n", + "10 scglue 0.160093 0.352738 0.134840 0.212101 \n", + "11 celloracle 0.443420 0.459941 0.630565 0.722041 \n", "\n", - " static-theta-0.5 overall_score Peak memory (GB) Duration (hour) \n", - "0 0.095068 0.077646 0.000000 0.000000 \n", - "1 0.000000 0.032401 0.000000 0.000000 \n", - "2 0.668815 0.860974 4.900000 0.075278 \n", - "3 0.184772 0.453523 0.975100 0.072500 \n", - "4 0.533161 0.774527 0.974700 0.064167 \n", - "5 0.125352 0.226639 46.943497 0.110556 \n", - "6 0.046814 0.050776 3.909119 0.556667 \n", - "7 0.687766 0.798691 13.105103 16.682500 \n", - "8 1.000000 0.891616 3.067471 1.568056 \n", - "9 0.665799 0.458534 30.356461 1.908056 \n", - "10 0.212101 0.214294 29.917423 4.380278 \n", - "11 0.722041 0.578251 14.900000 1.472222 " + " overall_score Peak memory (GB) Duration (hour) \n", + "0 0.083893 0.000000 0.000000 \n", + "1 0.040501 0.000000 0.000000 \n", + "2 0.835133 4.900000 0.075278 \n", + "3 0.444373 0.975100 0.072500 \n", + "4 0.752228 0.974700 0.064167 \n", + "5 0.226782 5.854904 0.153611 \n", + "6 0.058123 3.909119 0.556667 \n", + "7 0.775401 13.105103 16.682500 \n", + "8 0.864520 3.067471 1.568056 \n", + "9 0.453637 30.356461 1.908056 \n", + "10 0.214943 29.917423 4.380278 \n", + "11 0.563992 14.900000 1.472222 " ] }, - "execution_count": 41, + "execution_count": 39, "metadata": {}, "output_type": "execute_result" } @@ -1239,7 +1609,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 40, "metadata": {}, "outputs": [ { @@ -1263,7 +1633,7 @@ "\u001b[36m──\u001b[39m \u001b[1mColumn specification\u001b[22m \u001b[36m────────────────────────────────────────────────────────\u001b[39m\n", "\u001b[1mDelimiter:\u001b[22m \"\\t\"\n", "\u001b[31mchr\u001b[39m (1): method_name\n", - "\u001b[32mdbl\u001b[39m (12): ...1, ex(False)_tf(-1), ex(True)_tf(-1), static-theta-0.0, static-...\n", + "\u001b[32mdbl\u001b[39m (12): ...1, S1, S2, static-theta-0.0, static-theta-0.5, overall_score, P...\n", "\n", "\u001b[36mℹ\u001b[39m Use `spec()` to retrieve the full column specification for this data.\n", "\u001b[36mℹ\u001b[39m Specify the column types or set `show_col_types = FALSE` to quiet this message.\n", @@ -1272,8 +1642,8 @@ " \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \n", "\u001b[90m 1\u001b[39m method_name \u001b[31mNA\u001b[39m \u001b[90m\"\u001b[39mName\u001b[90m\"\u001b[39m meth… text \u001b[31mNA\u001b[39m \u001b[90m\u001b[39m\n", "\u001b[90m 2\u001b[39m overall_score overall_score \u001b[90m\"\u001b[39mScore\u001b[90m\"\u001b[39m over… bar overall \u001b[90m\u001b[39m\n", - "\u001b[90m 3\u001b[39m ex(False)_tf(-1) ex(False)_tf(-1) \u001b[90m\"\u001b[39mS11\u001b[90m\"\u001b[39m metr… funk… metric… \u001b[90m\u001b[39m\n", - "\u001b[90m 4\u001b[39m ex(True)_tf(-1) ex(True)_tf(-1) \u001b[90m\"\u001b[39mS12\u001b[90m\"\u001b[39m metr… funk… metric… \u001b[90m\u001b[39m\n", + "\u001b[90m 3\u001b[39m S1 S1 \u001b[90m\"\u001b[39mS1\u001b[90m\"\u001b[39m metr… funk… metric… \u001b[90m\u001b[39m\n", + "\u001b[90m 4\u001b[39m S2 S2 \u001b[90m\"\u001b[39mS2\u001b[90m\"\u001b[39m metr… funk… metric… \u001b[90m\u001b[39m\n", "\u001b[90m 5\u001b[39m static-theta-0.0 static-theta-0.0 \u001b[90m\"\u001b[39mTheta (m… metr… funk… metric… \u001b[90m\u001b[39m\n", "\u001b[90m 6\u001b[39m static-theta-0.5 static-theta-0.5 \u001b[90m\"\u001b[39mTheta (m… metr… funk… metric… \u001b[90m\u001b[39m\n", "\u001b[90m 7\u001b[39m memory_log \u001b[31mNA\u001b[39m \u001b[90m\"\u001b[39mPeak mem… reso… rect resour… \u001b[90m\u001b[39m\n", @@ -1315,1705 +1685,1883 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Metacell" + "# All layers" ] }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "lognorm-ridge.csv scgen_lognorm-ridge.csv scgen_pearson-ridge.csv\n", + "pearson-ridge.csv scgen_pearson-GB.csv\n" + ] + } + ], "source": [ - "# if par['metacell']:\n", - "# print('metacell')\n", - "# def create_meta_cells(df, n_cells=15):\n", - "# meta_x = []\n", - "# for i in range(0, df.shape[0], n_cells):\n", - "# meta_x.append(df.iloc[i:i+n_cells, :].sum(axis=0).values)\n", - "# df = pd.DataFrame(meta_x, columns=df.columns)\n", - "# return df\n", - " \n", - "# adata_df = pd.DataFrame(multiomics_rna.X.todense(), columns=multiomics_rna.var_names)\n", - "# adata_df['cell_type'] = multiomics_rna.obs['cell_type'].values\n", - "# adata_df['donor_id'] = multiomics_rna.obs['donor_id'].values\n", - "# df = adata_df.groupby(['cell_type','donor_id']).apply(lambda df: create_meta_cells(df))\n", - "# X = df.values\n", - "# var = pd.DataFrame(index=df.columns)\n", - "# obs = df.reset_index()[['cell_type','donor_id']]\n", - "# multiomics_rna = ad.AnnData(X=X, obs=obs, var=var)" + "!ls resources/results/scores/layers/" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 37, "metadata": {}, - "source": [ - "# Robustness analysis" - ] + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
 collectrinegative_controlpositive_controlpearson_corrpearson_causalportiappcorgenie3grnboost2scenicscgluecelloracle
lognorm-0.050548-0.0432870.1133140.0539650.0755620.0148560.0030780.0732060.0916190.0688100.0273150.068165
pearson-0.095474-0.0433350.3952730.1931890.2851710.1131850.0166860.2959200.2991600.1172410.0618480.171422
seurat_lognorm-0.052159-0.0432500.1303730.0608950.0855540.0174660.0037230.0817030.1051080.0792170.0315970.078648
seurat_pearson-0.095343-0.0425590.4125510.2012680.2897540.1081110.0166150.2936780.3018320.1227650.0648900.181489
scgen_lognorm-0.059849-0.0445210.2384230.1062240.1567740.0554870.0087950.1623190.2197270.1478120.0605720.150424
scgen_pearson-0.100238-0.0437950.4891470.2386640.3552560.1489410.0228460.3726410.3810320.1475530.0783090.216897
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "base_dir = 'resources/results/scores/layers'\n", + "layers = ['lognorm','pearson', 'seurat_lognorm', 'seurat_pearson', 'scgen_lognorm', 'scgen_pearson']\n", + "reg_type = 'ridge'\n", + "reg1_metric = 'S1'\n", + "reg2_metric = 'static-theta-0.5'\n", + "for i, layer in enumerate(layers):\n", + " df = pd.read_csv(f'{base_dir}/{layer}-{reg_type}.csv',index_col=0)\n", + " df_reg1 = df.loc[:, [reg1_metric]].rename(columns={reg1_metric:layer})\n", + " df_reg2 = df.loc[:, [reg2_metric]].rename(columns={reg2_metric:layer})\n", + " \n", + " if i == 0:\n", + " reg1_scores_layers = df_reg1\n", + " reg2_scores_layers = df_reg2\n", + " else:\n", + " reg1_scores_layers = pd.concat([reg1_scores_layers, df_reg1], axis=1)\n", + " reg2_scores_layers = pd.concat([reg2_scores_layers, df_reg2], axis=1)\n", + " \n", + "reg1_scores_layers = reg1_scores_layers.T\n", + "reg2_scores_layers = reg2_scores_layers.T\n", + "reg1_scores_layers.style.background_gradient()\n" + ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
 collectrinegative_controlpositive_controlpearson_corrpearson_causalportiappcorgenie3grnboost2scenicscgluecelloracle
lognorm0.1835660.1822930.2992290.2122370.2645120.1934870.1682370.2867180.3308190.2740800.2094470.299154
pearson0.2164190.2110020.3044170.2340600.2837380.2194560.1984340.3152550.3624390.3058300.2345930.319509
seurat_lognorm0.2513670.2528510.3582310.2821290.3295190.2628460.2348150.3516520.4017130.3480310.2763370.368954
seurat_pearson0.3228590.3208500.3988690.3333230.3793640.3222270.3016610.4108800.4646830.4125150.3387160.426141
scgen_lognorm0.5447190.5444290.6094610.5568630.5842340.5520320.5373380.5978710.6343710.6068460.5562640.618535
scgen_pearson0.5148960.5050020.5746080.5242320.5604900.5180480.5098740.5765800.6090750.5742940.5270760.580147
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "reg2_scores_layers.style.background_gradient()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Robustness analysis" + ] + }, + { + "cell_type": "code", + "execution_count": 30, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "temporaryFolder: /tmp/viash_hub_repo10268443552668753296 uri: https://github.com/openproblems-bio/openproblems-v2.git\n", - "Cloning into '.'...\n", - "checkout out: List(git, checkout, origin/main_build, --, .) 0 \n", - "\u001b[37mExporting run_robustness_analysis (workflows) =nextflow=> /mnt/c/Users/nourisa/Documents/testProjs/ongoing/task_grn_inference/target/nextflow/workflows/run_robustness_analysis\u001b[0m\n", - "\u001b[33mNot all configs built successfully\u001b[0m\n", - "\u001b[33m 39 configs were disabled\u001b[0m\n", - "\u001b[32m 1/1 configs built successfully\u001b[0m\n" + "{'reg_type': 'ridge', 'write_dir': 'resources/results/robustness_analysis', 'perturbation_data': 'resources/grn-benchmark/perturbation_data.h5ad', 'cell_type_specific': False, 'normalize': False, 'multiomics_rna': 'resources/grn-benchmark/multiomics_rna_d0_hvg.h5ad', 'tf_all': 'resources/prior/tf_all.csv', 'max_n_links': 50000, 'apply_tf': False, 'subsample': -2, 'verbose': 2, 'binarize': True, 'num_workers': 20, 'consensus': 'resources/prior/consensus-num-regulators.json', 'static_only': True, 'clip_scores': True, 'layer': 'scgen_pearson', 'prediction': 'resources/results/robustness_analysis/corr/corr_causal.csv', 'causal': False}\n", + "Read data\n", + "Random subsetting\n", + "Traceback (most recent call last):\n", + " File \"/home/jnourisa/projs/ongoing/task_grn_inference/src/robustness_analysis/script_all.py\", line 126, in \n", + " net = create_corr_net(par)\n", + " File \"/home/jnourisa/projs/ongoing/task_grn_inference/src/utils/util.py\", line 97, in create_corr_net\n", + " grn = corr_net(X, gene_names, par, tf_all, par['causal'])\n", + " File \"/home/jnourisa/projs/ongoing/task_grn_inference/src/utils/util.py\", line 49, in corr_net\n", + " net = net.sample(len(tf_all), axis=1, random_state=par['seed'])\n", + "KeyError: 'seed'\n" ] } ], "source": [ - "# !viash ns build --setup cb --parallel\n", - "!viash ns build --setup cb -q run_robustness_analysis\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Permute net" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "!bash scripts/run_robust_analys.sh \"net\"" + "!python src/robustness_analysis/script_all.py" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Causal" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "# causal \n", + "scores_corr = pd.read_csv(\"resources/results/robustness_analysis/corr/scores_corr.csv\", index_col=0)\n", + "scores_causal = pd.read_csv(\"resources/results/robustness_analysis/corr/scores_causal.csv\", index_col=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
S1S2static-theta-0.0static-theta-0.5
00.1294380.1461220.3959470.531279
\n", + "
" + ], + "text/plain": [ + " S1 S2 static-theta-0.0 static-theta-0.5\n", + "0 0.129438 0.146122 0.395947 0.531279" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scores_causal" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
S1S2static-theta-0.0static-theta-0.5
00.3084040.5309000.6962510.543549
00.2656140.4758090.5235040.541446
00.2909100.4916930.6788480.545381
00.2798640.4889060.5947990.533744
00.2871340.5050980.7639910.550709
...............
00.2757640.5036440.4639090.535066
00.2792470.4980710.6095570.534484
00.2713500.4689980.7215200.538930
00.2771220.4841590.6527540.537736
00.2993300.5371390.7103750.540515
\n", + "

100 rows × 4 columns

\n", + "
" + ], + "text/plain": [ + " S1 S2 static-theta-0.0 static-theta-0.5\n", + "0 0.308404 0.530900 0.696251 0.543549\n", + "0 0.265614 0.475809 0.523504 0.541446\n", + "0 0.290910 0.491693 0.678848 0.545381\n", + "0 0.279864 0.488906 0.594799 0.533744\n", + "0 0.287134 0.505098 0.763991 0.550709\n", + ".. ... ... ... ...\n", + "0 0.275764 0.503644 0.463909 0.535066\n", + "0 0.279247 0.498071 0.609557 0.534484\n", + "0 0.271350 0.468998 0.721520 0.538930\n", + "0 0.277122 0.484159 0.652754 0.537736\n", + "0 0.299330 0.537139 0.710375 0.540515\n", + "\n", + "[100 rows x 4 columns]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scores_corr" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_net/state.yaml to resources/results/robust_analy_reg2_net/state.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_net/scores.yaml to resources/results/robust_analy_reg2_net/scores.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_net/metric_configs.yaml to resources/results/robust_analy_reg2_net/metric_configs.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_net/trace.txt to resources/results/robust_analy_reg2_net/trace.txt\n" + "[False True False False False False False False False True False False\n", + " False False False True False False False False False False False False\n", + " False False False False False False False False False False False True\n", + " False False False False False False False False True True False True\n", + " False False False False False True False False False False False False\n", + " False False False False False False True True True False True False\n", + " False False False False False False False False False True False True\n", + " False False True True False False False False False False True False\n", + " False False False False]\n", + "[False True True True True True True True True True False False\n", + " True True True True False False True True True False True True\n", + " True True True False True True False True False True False True\n", + " True False True True False True False True True True False True\n", + " True True True True False True False True False False True True\n", + " True True False True True True True True True True True True\n", + " True True True False True True True True False True True True\n", + " False True True True True False False False True True True True\n", + " True True True False]\n", + "[ True True True True False False True False True True True True\n", + " True True False True True False True True True False False True\n", + " True False False False True True True True True True True True\n", + " True False True True False True True False True True True True\n", + " True True False True True True True True True True True True\n", + " True True True False True True False True True True True True\n", + " True True False True False True False True False True False True\n", + " True False True True True True True True True False True True\n", + " True False True True]\n", + "[False False False True False False False False True False True False\n", + " False False False False False False False False False False False False\n", + " False False False True True False False False False False False True\n", + " False False False False False True False False True False True False\n", + " False False False True False False True True False False False False\n", + " False False False False True False False True False False True False\n", + " False True False False False False False False False False False False\n", + " False False True True False True True False False False False False\n", + " True False False False]\n" ] } ], "source": [ - "!aws s3 sync s3://openproblems-data/resources/grn/results/robust_analy_reg2_net ./resources/results/robust_analy_reg2_net" + "for col in scores_corr.columns:\n", + " print(scores_corr[col].values run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "\u001b[3A\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[5A\n", - "executor > local (1)\u001b[K\n", - "[5f/aa4675] process > run_robustness_analysis:run... [ 0%] 0 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (6)\u001b[K\n", - "[f3/689356] process > run_robustness_analysis:run... [ 0%] 0 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (7)\u001b[K\n", - "[5f/aa4675] process > run_robustness_analysis:run... [ 3%] 1 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 1\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (9)\u001b[K\n", - "[bc/144211] process > run_robustness_analysis:run... [ 10%] 3 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 3\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (10)\u001b[K\n", - "[b2/e135c4] process > run_robustness_analysis:run... [ 13%] 4 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 4\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (10)\u001b[K\n", - "[b2/e135c4] process > run_robustness_analysis:run... [ 13%] 4 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 4\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (11)\u001b[K\n", - "[41/99eca2] process > run_robustness_analysis:run... [ 16%] 5 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 5\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (13)\u001b[K\n", - "[c2/0fe413] process > run_robustness_analysis:run... [ 23%] 7 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 6\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (13)\u001b[K\n", - "[c2/0fe413] process > run_robustness_analysis:run... [ 23%] 7 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 7\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (14)\u001b[K\n", - "[e1/58ddaa] process > run_robustness_analysis:run... [ 26%] 8 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 8\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (15)\u001b[K\n", - "[7b/b975d3] process > run_robustness_analysis:run... [ 30%] 9 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 8\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (15)\u001b[K\n", - "[7b/b975d3] process > run_robustness_analysis:run... [ 30%] 9 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 9\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (16)\u001b[K\n", - "[64/f3e1eb] process > run_robustness_analysis:run... [ 33%] 10 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 10\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (18)\u001b[K\n", - "[a9/8325fb] process > run_robustness_analysis:run... [ 40%] 12 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 11\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (19)\u001b[K\n", - "[d4/d30a10] process > run_robustness_analysis:run... [ 43%] 13 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 12\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (19)\u001b[K\n", - "[d4/d30a10] process > run_robustness_analysis:run... [ 43%] 13 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 13\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (20)\u001b[K\n", - "[48/b949bd] process > run_robustness_analysis:run... [ 46%] 14 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 14\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (21)\u001b[K\n", - "[40/2c9923] process > run_robustness_analysis:run... [ 50%] 15 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 15\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (22)\u001b[K\n", - "[45/952ee5] process > run_robustness_analysis:run... [ 53%] 16 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 16\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (23)\u001b[K\n", - "[57/dece12] process > run_robustness_analysis:run... [ 56%] 17 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 17\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (24)\u001b[K\n", - "[8b/7868a3] process > run_robustness_analysis:run... [ 60%] 18 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 18\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (25)\u001b[K\n", - "[a9/8325fb] process > run_robustness_analysis:run... [ 63%] 19 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 19\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (25)\u001b[K\n", - "[a9/8325fb] process > run_robustness_analysis:run... [ 63%] 19 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 19\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (26)\u001b[K\n", - "[3e/bd85b3] process > run_robustness_analysis:run... [ 66%] 20 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 20\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (27)\u001b[K\n", - "[58/cbdb13] process > run_robustness_analysis:run... [ 70%] 21 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 21\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (28)\u001b[K\n", - "[55/ec24c0] process > run_robustness_analysis:run... [ 73%] 22 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 22\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (29)\u001b[K\n", - "[cd/555e42] process > run_robustness_analysis:run... [ 76%] 23 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 23\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (30)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [ 80%] 24 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 24\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (31)\u001b[K\n", - "[09/ecd838] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[4f/726118] process > run_robustness_analysis:run... [ 0%] 0 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (32)\u001b[K\n", - "[3e/bd85b3] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[c7/828453] process > run_robustness_analysis:run... [ 0%] 0 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (33)\u001b[K\n", - "[58/cbdb13] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[7f/f16766] process > run_robustness_analysis:run... [ 0%] 0 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (33)\u001b[K\n", - "[58/cbdb13] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[7f/f16766] process > run_robustness_analysis:run... [ 0%] 0 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (34)\u001b[K\n", - "[55/ec24c0] process > run_robustness_analysis:run... [ 93%] 28 of 30\u001b[K\n", - "[80/dcac1b] process > run_robustness_analysis:run... [ 0%] 0 of 28\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (35)\u001b[K\n", - "[cd/555e42] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[a5/a9db21] process > run_robustness_analysis:run... [ 0%] 0 of 28\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (35)\u001b[K\n", - "[cd/555e42] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[a5/a9db21] process > run_robustness_analysis:run... [ 0%] 0 of 29\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (36)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e9/90d374] process > run_robustness_analysis:run... [ 0%] 0 of 29\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (36)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e9/90d374] process > run_robustness_analysis:run... [ 0%] 0 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (37)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7f/f16766] process > run_robustness_analysis:run... [ 3%] 1 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 1\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (37)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/726118] process > run_robustness_analysis:run... [ 6%] 2 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 1\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (38)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7e/fced9a] process > run_robustness_analysis:run... [ 6%] 2 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 2\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (39)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[a5/a9db21] process > run_robustness_analysis:run... [ 10%] 3 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 3\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (40)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7f/6484f9] process > run_robustness_analysis:run... [ 13%] 4 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 4\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (40)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7f/6484f9] process > run_robustness_analysis:run... [ 13%] 4 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 4\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (41)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[02/0a1ff4] process > run_robustness_analysis:run... [ 16%] 5 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 5\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (42)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[80/dcac1b] process > run_robustness_analysis:run... [ 20%] 6 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 6\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (43)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7f/355350] process > run_robustness_analysis:run... [ 23%] 7 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 6\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (43)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7f/355350] process > run_robustness_analysis:run... [ 23%] 7 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 7\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (44)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[02/0a1ff4] process > run_robustness_analysis:run... [ 26%] 8 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 8\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (44)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[02/0a1ff4] process > run_robustness_analysis:run... [ 26%] 8 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 8\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (45)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[c0/7f3f8c] process > run_robustness_analysis:run... [ 30%] 9 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 9\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (46)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7e/fced9a] process > run_robustness_analysis:run... [ 33%] 10 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 10\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (46)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7e/fced9a] process > run_robustness_analysis:run... [ 33%] 10 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 10\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (47)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[c0/7f3f8c] process > run_robustness_analysis:run... [ 36%] 11 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 11\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (48)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e2/5dd3e9] process > run_robustness_analysis:run... [ 40%] 12 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 11\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (48)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e2/5dd3e9] process > run_robustness_analysis:run... [ 40%] 12 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 12\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (49)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[06/b5bead] process > run_robustness_analysis:run... [ 43%] 13 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 12\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (49)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[06/b5bead] process > run_robustness_analysis:run... [ 43%] 13 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 13\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (50)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[29/84b62b] process > run_robustness_analysis:run... [ 46%] 14 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 14\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (50)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[11/a529f0] process > run_robustness_analysis:run... [ 50%] 15 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 14\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (51)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e4/2df648] process > run_robustness_analysis:run... [ 50%] 15 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 15\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (52)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[71/62b2be] process > run_robustness_analysis:run... [ 53%] 16 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 15\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (52)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[71/62b2be] process > run_robustness_analysis:run... [ 53%] 16 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 16\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (53)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 56%] 17 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 17\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (54)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ce/14a13f] process > run_robustness_analysis:run... [ 60%] 18 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 18\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (55)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e2/5dd3e9] process > run_robustness_analysis:run... [ 63%] 19 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 19\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (55)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e2/5dd3e9] process > run_robustness_analysis:run... [ 63%] 19 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 19\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (56)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[71/62b2be] process > run_robustness_analysis:run... [ 66%] 20 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 19\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (56)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[71/62b2be] process > run_robustness_analysis:run... [ 66%] 20 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 20\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (57)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ee/2134b6] process > run_robustness_analysis:run... [ 70%] 21 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 20\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (57)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ee/2134b6] process > run_robustness_analysis:run... [ 70%] 21 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 21\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (58)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/e10dd4] process > run_robustness_analysis:run... [ 73%] 22 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 22\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (58)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/e10dd4] process > run_robustness_analysis:run... [ 73%] 22 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 22\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (59)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/1ffa8d] process > run_robustness_analysis:run... [ 76%] 23 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 22\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (59)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/1ffa8d] process > run_robustness_analysis:run... [ 76%] 23 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 23\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (60)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ce/14a13f] process > run_robustness_analysis:run... [ 80%] 24 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 24\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (61)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[db/6af869] process > run_robustness_analysis:run... [ 0%] 0 of 24\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (61)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[db/6af869] process > run_robustness_analysis:run... [ 0%] 0 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (62)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[97/46ce3e] process > run_robustness_analysis:run... [ 4%] 1 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (62)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[97/46ce3e] process > run_robustness_analysis:run... [ 8%] 2 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (63)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[13/dd0594] process > run_robustness_analysis:run... [ 8%] 2 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (64)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[cc/a14f0b] process > run_robustness_analysis:run... [ 12%] 3 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (65)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[dd/df4db4] process > run_robustness_analysis:run... [ 16%] 4 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (65)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[dd/df4db4] process > run_robustness_analysis:run... [ 16%] 4 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (66)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[1e/d146ba] process > run_robustness_analysis:run... [ 20%] 5 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (66)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[1e/d146ba] process > run_robustness_analysis:run... [ 20%] 5 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (67)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[88/7b4b56] process > run_robustness_analysis:run... [ 24%] 6 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (67)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[88/7b4b56] process > run_robustness_analysis:run... [ 24%] 6 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (68)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[24/082c3f] process > run_robustness_analysis:run... [ 28%] 7 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (68)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[24/082c3f] process > run_robustness_analysis:run... [ 28%] 7 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (69)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[f2/cf0248] process > run_robustness_analysis:run... [ 32%] 8 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (70)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[be/464085] process > run_robustness_analysis:run... [ 36%] 9 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (71)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[f5/03f97a] process > run_robustness_analysis:run... [ 40%] 10 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (71)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[f5/03f97a] process > run_robustness_analysis:run... [ 40%] 10 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (72)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[78/138a40] process > run_robustness_analysis:run... [ 44%] 11 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (73)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[d7/1ed34a] process > run_robustness_analysis:run... [ 48%] 12 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (74)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[09/a046e9] process > run_robustness_analysis:run... [ 52%] 13 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (75)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[c1/99784d] process > run_robustness_analysis:run... [ 56%] 14 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (75)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[c1/99784d] process > run_robustness_analysis:run... [ 56%] 14 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (76)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[3d/6f5339] process > run_robustness_analysis:run... [ 60%] 15 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (77)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[cd/7ad7f5] process > run_robustness_analysis:run... [ 64%] 16 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (78)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[34/a92115] process > run_robustness_analysis:run... [ 68%] 17 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (79)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[00/e4e2ee] process > run_robustness_analysis:run... [ 72%] 18 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (80)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[f5/b4e24a] process > run_robustness_analysis:run... [ 76%] 19 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (81)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[87/3bc33c] process > run_robustness_analysis:run... [ 80%] 20 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (81)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[87/3bc33c] process > run_robustness_analysis:run... [ 80%] 20 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (82)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[c8/fae13c] process > run_robustness_analysis:run... [ 84%] 21 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (83)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[e0/204b2c] process > run_robustness_analysis:run... [ 88%] 22 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (83)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[e0/204b2c] process > run_robustness_analysis:run... [ 88%] 22 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (84)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[f8/b7bf95] process > run_robustness_analysis:run... [ 92%] 23 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (85)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/1ffa8d] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[bb/db3f8e] process > run_robustness_analysis:run... [ 92%] 23 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (86)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/e10dd4] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[40/90153e] process > run_robustness_analysis:run... [ 88%] 23 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (86)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/e10dd4] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[40/90153e] process > run_robustness_analysis:run... [ 85%] 23 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (87)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/e10dd4] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[07/730549] process > run_robustness_analysis:run... [ 88%] 24 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (87)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/e10dd4] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[bb/db3f8e] process > run_robustness_analysis:run... [ 92%] 25 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (87)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/e10dd4] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[40/90153e] process > run_robustness_analysis:run... [ 96%] 26 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (87)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/e10dd4] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[07/730549] process > run_robustness_analysis:run... [100%] 27 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (88)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[10/f830da] process > run_robustness_analysis:run... [ 93%] 28 of 30\u001b[K\n", - "[02/400b59] process > run_robustness_analysis:run... [ 96%] 27 of 28\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (88)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[10/f830da] process > run_robustness_analysis:run... [ 93%] 28 of 30\u001b[K\n", - "[02/400b59] process > run_robustness_analysis:run... [100%] 28 of 28\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (89)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[1a/9c2b76] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[5e/8cbd1a] process > run_robustness_analysis:run... [ 96%] 28 of 29\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (89)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[1a/9c2b76] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[5e/8cbd1a] process > run_robustness_analysis:run... [ 96%] 28 of 29\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (89)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7d/6ca956] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/8cbd1a] process > run_robustness_analysis:run... [ 96%] 28 of 29\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (90)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7d/6ca956] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e6/69e750] process > run_robustness_analysis:run... [ 93%] 28 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (90)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7d/6ca956] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/8cbd1a] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (90)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7d/6ca956] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e6/69e750] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (91)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7d/6ca956] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e6/69e750] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[71/51e091] process > run_robustness_analysis:pub... [ 0%] 0 of 1\u001b[K\n", - "\u001b[6A\n", - "executor > local (91)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7d/6ca956] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e6/69e750] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[71/51e091] process > run_robustness_analysis:pub... [100%] 1 of 1 ✔\u001b[K\n", - "\u001b[32;1mCompleted at: 25-Aug-2024 22:35:49\n", - "Duration : 9m 39s\n", - "CPU hours : 1.7\n", - "Succeeded : 91\n", - "\u001b[22;39m\u001b[K\n", - "\n" - ] + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
 collectrinegative_controlpositive_controlpearson_corrpearson_causalportiappcorgenie3grnboost2scenicscgluecelloracle
0-0.100238-0.0437950.4891470.2386640.3552560.1489410.0228460.3726410.3810320.1473660.0783090.216897
10-0.123141-0.0435300.4340330.2132320.2882800.1031490.0133730.3268760.3387790.1410920.0693040.197389
20-0.128874-0.0444970.4099200.1954310.2673210.0661330.0041310.2764390.2999010.1250480.0618170.175780
50-0.145682-0.0393710.3046780.1404340.180339-0.023024-0.0059500.1315160.2028500.0718230.0324130.103936
100-0.164579-0.045165-0.061065-0.005408-0.182804-0.114472-0.034709-0.341788-0.152451-0.021402-0.012279-0.057854
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 46, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "!bash scripts/run_robust_analys.sh \"weight\"" + "# net \n", + "noise_type = 'net'\n", + "base_dir = 'resources/results/robustness_analysis'\n", + "degrees = [0, 10, 20, 50, 100]\n", + "reg1_metric = 'S1'\n", + "reg2_metric = 'static-theta-0.5'\n", + "for i, degree in enumerate(degrees):\n", + " df = pd.read_csv(f'{base_dir}/{noise_type}-{degree}-scores.csv',index_col=0)\n", + " df_reg1 = df.loc[:, [reg1_metric]].rename(columns={reg1_metric:degree})\n", + " df_reg2 = df.loc[:, [reg2_metric]].rename(columns={reg2_metric:degree})\n", + " \n", + " if i == 0:\n", + " reg1_scores_layers = df_reg1\n", + " reg2_scores_layers = df_reg2\n", + " else:\n", + " reg1_scores_layers = pd.concat([reg1_scores_layers, df_reg1], axis=1)\n", + " reg2_scores_layers = pd.concat([reg2_scores_layers, df_reg2], axis=1)\n", + " \n", + "reg1_scores_layers = reg1_scores_layers.T\n", + "reg2_scores_layers = reg2_scores_layers.T\n", + "reg1_scores_layers.style.background_gradient()\n" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 48, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_weight/scores.yaml to resources/results/robust_analy_reg2_weight/scores.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_weight/state.yaml to resources/results/robust_analy_reg2_weight/state.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_weight/metric_configs.yaml to resources/results/robust_analy_reg2_weight/metric_configs.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_weight/trace.txt to resources/results/robust_analy_reg2_weight/trace.txt\n" - ] + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
 collectrinegative_controlpositive_controlpearson_corrpearson_causalportiappcorgenie3grnboost2scenicscgluecelloracle
00.5148960.5050020.5746080.5242320.5604900.5180480.5098740.5765800.6090750.5742940.5270760.580147
100.5154060.5056910.5718820.5230670.5555720.5243930.5160360.5708210.6017590.5659420.5308010.573236
200.5111720.5046630.5559540.5261950.5524490.5159560.5152920.5699760.5964150.5616190.5268210.563653
500.5118030.4958540.5529350.5247440.5347640.5160560.5107110.5566610.5819200.5483280.5183660.542993
1000.5055400.5033720.5310050.5137290.5230020.5067600.5132870.5048320.5145850.5040930.5064430.513158
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 48, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "!aws s3 sync s3://openproblems-data/resources/grn/results/robust_analy_reg2_weight ./resources/results/robust_analy_reg2_weight" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Negative vs positive controls" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "!bash scripts/run_pc_vs_nc.sh \"pc_vs_nc\"" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "!aws s3 sync s3://openproblems-data/resources/grn/results/robust_analy_reg2_pc_vs_nc ./resources/results/robust_analy_reg2_pc_vs_nc" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## regulatory sign" + "reg2_scores_layers.style.background_gradient()" ] }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 3, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "sign\n", - "\u001b[33mNextflow 24.04.4 is available - Please consider updating your version to it\u001b[m\n", - "N E X T F L O W ~ version 23.10.4\n", - "Launching `target/nextflow/workflows/run_robustness_analysis/main.nf` [soggy_sinoussi] DSL2 - revision: d55e1b9116\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "\u001b[2A\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "\u001b[3A\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[5A\n", - "executor > local (2)\u001b[K\n", - "[2f/3f5479] process > run_robustness_analysis:run... [ 0%] 0 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (6)\u001b[K\n", - "[f9/652d7d] process > run_robustness_analysis:run... [ 0%] 0 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (7)\u001b[K\n", - "[d7/bd6de3] process > run_robustness_analysis:run... [ 3%] 1 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 1\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (9)\u001b[K\n", - "[61/b7d70f] process > run_robustness_analysis:run... [ 10%] 3 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 2\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (10)\u001b[K\n", - "[86/319727] process > run_robustness_analysis:run... [ 13%] 4 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 4\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (11)\u001b[K\n", - "[ad/5bccfc] process > run_robustness_analysis:run... [ 16%] 5 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 5\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (12)\u001b[K\n", - "[55/295e22] process > run_robustness_analysis:run... [ 20%] 6 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 5\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (12)\u001b[K\n", - "[55/295e22] process > run_robustness_analysis:run... [ 20%] 6 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 6\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (14)\u001b[K\n", - "[60/1b8a95] process > run_robustness_analysis:run... [ 26%] 8 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 8\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (15)\u001b[K\n", - "[a4/0f774f] process > run_robustness_analysis:run... [ 30%] 9 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 9\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (16)\u001b[K\n", - "[a4/bf1a59] process > run_robustness_analysis:run... [ 33%] 10 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 10\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (17)\u001b[K\n", - "[f0/3f59fb] process > run_robustness_analysis:run... [ 36%] 11 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 11\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (18)\u001b[K\n", - "[2d/97cfa6] process > run_robustness_analysis:run... [ 40%] 12 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 12\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (19)\u001b[K\n", - "[82/e5f963] process > run_robustness_analysis:run... [ 43%] 13 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 13\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (20)\u001b[K\n", - "[16/08d5fc] process > run_robustness_analysis:run... [ 46%] 14 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 14\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (21)\u001b[K\n", - "[17/a55008] process > run_robustness_analysis:run... [ 50%] 15 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 15\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (23)\u001b[K\n", - "[aa/63fbb5] process > run_robustness_analysis:run... [ 56%] 17 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 16\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (24)\u001b[K\n", - "[f0/3f59fb] process > run_robustness_analysis:run... [ 60%] 18 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 18\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (25)\u001b[K\n", - "[83/2cd2c1] process > run_robustness_analysis:run... [ 63%] 19 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 18\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (25)\u001b[K\n", - "[83/2cd2c1] process > run_robustness_analysis:run... [ 63%] 19 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 19\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (26)\u001b[K\n", - "[25/668fba] process > run_robustness_analysis:run... [ 66%] 20 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 20\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (27)\u001b[K\n", - "[c9/c54c3a] process > run_robustness_analysis:run... [ 70%] 21 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 21\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (28)\u001b[K\n", - "[45/4ba6a2] process > run_robustness_analysis:run... [ 73%] 22 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 22\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (28)\u001b[K\n", - "[45/4ba6a2] process > run_robustness_analysis:run... [ 73%] 22 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 22\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (29)\u001b[K\n", - "[2a/6e29ce] process > run_robustness_analysis:run... [ 76%] 23 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 23\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (30)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [ 80%] 24 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 24\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (31)\u001b[K\n", - "[83/2cd2c1] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[a5/757052] process > run_robustness_analysis:run... [ 0%] 0 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (32)\u001b[K\n", - "[25/668fba] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[0e/0accdb] process > run_robustness_analysis:run... [ 0%] 0 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (33)\u001b[K\n", - "[c9/c54c3a] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[f6/a2c7dd] process > run_robustness_analysis:run... [ 0%] 0 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (34)\u001b[K\n", - "[45/4ba6a2] process > run_robustness_analysis:run... [ 93%] 28 of 30\u001b[K\n", - "[81/2ff6fc] process > run_robustness_analysis:run... [ 0%] 0 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (34)\u001b[K\n", - "[45/4ba6a2] process > run_robustness_analysis:run... [ 93%] 28 of 30\u001b[K\n", - "[81/2ff6fc] process > run_robustness_analysis:run... [ 0%] 0 of 28\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (35)\u001b[K\n", - "[2a/6e29ce] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[e5/dd1da2] process > run_robustness_analysis:run... [ 0%] 0 of 29\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (36)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[c9/cc55af] process > run_robustness_analysis:run... [ 0%] 0 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (36)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[c9/cc55af] process > run_robustness_analysis:run... [ 0%] 0 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (37)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[55/b2845d] process > run_robustness_analysis:run... [ 3%] 1 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 1\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (38)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[0e/0accdb] process > run_robustness_analysis:run... [ 6%] 2 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 1\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (38)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[0e/0accdb] process > run_robustness_analysis:run... [ 6%] 2 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 2\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (39)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[81/2ff6fc] process > run_robustness_analysis:run... [ 10%] 3 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 2\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (39)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[81/2ff6fc] process > run_robustness_analysis:run... [ 10%] 3 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 3\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (40)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e5/dd1da2] process > run_robustness_analysis:run... [ 13%] 4 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 3\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (40)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e5/dd1da2] process > run_robustness_analysis:run... [ 13%] 4 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 4\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (41)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[17/4cc270] process > run_robustness_analysis:run... [ 16%] 5 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 4\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (41)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[17/4cc270] process > run_robustness_analysis:run... [ 16%] 5 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 5\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (42)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[c9/cc55af] process > run_robustness_analysis:run... [ 20%] 6 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 6\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (43)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4b/034b41] process > run_robustness_analysis:run... [ 23%] 7 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 7\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (44)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7a/54021d] process > run_robustness_analysis:run... [ 26%] 8 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 8\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (45)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[f6/a2c7dd] process > run_robustness_analysis:run... [ 30%] 9 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 9\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (46)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[dc/a6fcac] process > run_robustness_analysis:run... [ 33%] 10 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 10\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (47)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[d9/fb7083] process > run_robustness_analysis:run... [ 36%] 11 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 11\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (47)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[d9/fb7083] process > run_robustness_analysis:run... [ 36%] 11 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 11\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (48)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7b/5eddfc] process > run_robustness_analysis:run... [ 40%] 12 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 12\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (48)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7b/5eddfc] process > run_robustness_analysis:run... [ 40%] 12 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 12\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (49)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[80/28e78a] process > run_robustness_analysis:run... [ 43%] 13 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 12\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (49)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[80/28e78a] process > run_robustness_analysis:run... [ 43%] 13 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 13\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (50)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[6b/9b2bad] process > run_robustness_analysis:run... [ 46%] 14 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 14\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (51)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[1b/00118d] process > run_robustness_analysis:run... [ 50%] 15 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 15\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (52)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[69/3cbc30] process > run_robustness_analysis:run... [ 53%] 16 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 15\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (52)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[69/3cbc30] process > run_robustness_analysis:run... [ 53%] 16 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 16\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (53)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[28/41b50f] process > run_robustness_analysis:run... [ 56%] 17 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 17\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (53)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[28/41b50f] process > run_robustness_analysis:run... [ 56%] 17 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 17\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (54)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 60%] 18 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 17\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (54)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 60%] 18 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 18\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (55)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5a/c20a99] process > run_robustness_analysis:run... [ 63%] 19 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 19\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (56)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/b50eb7] process > run_robustness_analysis:run... [ 66%] 20 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 20\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (57)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 70%] 21 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 21\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (58)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5a/c20a99] process > run_robustness_analysis:run... [ 73%] 22 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 22\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (59)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[69/3cbc30] process > run_robustness_analysis:run... [ 76%] 23 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 23\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (60)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[b0/1cd463] process > run_robustness_analysis:run... [ 80%] 24 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 24\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (61)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[60/c4fb7e] process > run_robustness_analysis:run... [ 0%] 0 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (61)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[60/c4fb7e] process > run_robustness_analysis:run... [ 0%] 0 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (62)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[60/c4fb7e] process > run_robustness_analysis:run... [ 4%] 1 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (63)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[d8/3dccf5] process > run_robustness_analysis:run... [ 8%] 2 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (64)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[3a/c82a35] process > run_robustness_analysis:run... [ 12%] 3 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (65)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[fe/12f35c] process > run_robustness_analysis:run... [ 16%] 4 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (65)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[fe/12f35c] process > run_robustness_analysis:run... [ 16%] 4 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (66)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[fe/12f35c] process > run_robustness_analysis:run... [ 20%] 5 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (67)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[59/b22c05] process > run_robustness_analysis:run... [ 24%] 6 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (68)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[b5/f43d8a] process > run_robustness_analysis:run... [ 28%] 7 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (69)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[d7/23f6c7] process > run_robustness_analysis:run... [ 32%] 8 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (69)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[d7/23f6c7] process > run_robustness_analysis:run... [ 32%] 8 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (70)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[c0/e1ade3] process > run_robustness_analysis:run... [ 36%] 9 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (70)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[c0/e1ade3] process > run_robustness_analysis:run... [ 36%] 9 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (71)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[b6/e860a7] process > run_robustness_analysis:run... [ 40%] 10 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (71)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[b6/e860a7] process > run_robustness_analysis:run... [ 40%] 10 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (72)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[47/70e155] process > run_robustness_analysis:run... [ 44%] 11 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (72)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[47/70e155] process > run_robustness_analysis:run... [ 44%] 11 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (73)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[47/70e155] process > run_robustness_analysis:run... [ 48%] 12 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (73)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[47/70e155] process > run_robustness_analysis:run... [ 48%] 12 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (74)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[6d/11513e] process > run_robustness_analysis:run... [ 46%] 12 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (75)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[da/de5833] process > run_robustness_analysis:run... [ 50%] 13 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (76)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[78/0ba11b] process > run_robustness_analysis:run... [ 53%] 14 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (77)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[88/5c09d9] process > run_robustness_analysis:run... [ 57%] 15 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (78)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[22/57f530] process > run_robustness_analysis:run... [ 61%] 16 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (79)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[4d/ed1ab5] process > run_robustness_analysis:run... [ 65%] 17 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (79)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[4d/ed1ab5] process > run_robustness_analysis:run... [ 65%] 17 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (80)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[22/57f530] process > run_robustness_analysis:run... [ 69%] 18 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (81)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[51/3cf21c] process > run_robustness_analysis:run... [ 73%] 19 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (82)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[c5/8c073d] process > run_robustness_analysis:run... [ 76%] 20 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (83)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[8e/6c35e8] process > run_robustness_analysis:run... [ 80%] 21 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (84)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[c9/214a53] process > run_robustness_analysis:run... [ 84%] 22 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (85)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[20/538eef] process > run_robustness_analysis:run... [ 88%] 23 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (86)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/5e2fe1] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[ee/5dbe09] process > run_robustness_analysis:run... [ 85%] 23 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (86)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/5e2fe1] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[ee/5dbe09] process > run_robustness_analysis:run... [ 85%] 23 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (87)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/5e2fe1] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[db/cf8d69] process > run_robustness_analysis:run... [ 88%] 24 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (87)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/5e2fe1] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[20/538eef] process > run_robustness_analysis:run... [ 92%] 25 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (87)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/5e2fe1] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[ee/5dbe09] process > run_robustness_analysis:run... [ 96%] 26 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (87)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/5e2fe1] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[db/cf8d69] process > run_robustness_analysis:run... [100%] 27 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (88)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[94/a96236] process > run_robustness_analysis:run... [ 93%] 28 of 30\u001b[K\n", - "[62/4e8c10] process > run_robustness_analysis:run... [ 96%] 27 of 28\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (88)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[b0/1cd463] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[62/4e8c10] process > run_robustness_analysis:run... [ 96%] 27 of 28\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (89)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[b0/1cd463] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[ea/f3975f] process > run_robustness_analysis:run... [ 93%] 27 of 29\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (90)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/b50eb7] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[68/cffa95] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (90)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/b50eb7] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[62/4e8c10] process > run_robustness_analysis:run... [ 93%] 28 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (90)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/b50eb7] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ea/f3975f] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (90)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/b50eb7] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[68/cffa95] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (91)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/b50eb7] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[68/cffa95] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[27/96b063] process > run_robustness_analysis:pub... [ 0%] 0 of 1\u001b[K\n", - "\u001b[6A\n", - "executor > local (91)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/b50eb7] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[68/cffa95] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[27/96b063] process > run_robustness_analysis:pub... [100%] 1 of 1 ✔\u001b[K\n", - "\u001b[32;1mCompleted at: 26-Aug-2024 10:56:46\n", - "Duration : 9m 30s\n", - "CPU hours : 1.7\n", - "Succeeded : 91\n", - "\u001b[22;39m\u001b[K\n", - "\n" - ] + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
 collectrinegative_controlpositive_controlpearson_corrpearson_causalportiappcorgenie3grnboost2scenicscgluecelloracle
0-0.100238-0.0437950.4891470.2386640.3552560.1489410.0228460.3726410.3810320.1475530.0783090.216897
10-0.107958-0.0421990.4332460.1706120.2973540.1034480.0185510.3092430.3204760.1058500.0515650.148519
20-0.129616-0.0422580.3831690.1095020.2592010.064159-0.0018100.2030460.2295620.0674770.0269450.080443
50-0.154785-0.040425-0.091824-0.000377-0.188567-0.090789-0.013036-0.114169-0.122291-0.025912-0.010141-0.135087
100-0.100238-0.0437950.4891470.2386640.3552560.1489410.0228460.3726410.3810320.1475530.0783090.216897
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "!bash scripts/run_robust_analys.sh \"sign\"" + "noise_type = 'sign'\n", + "base_dir = 'resources/results/robustness_analysis'\n", + "degrees = [0, 10, 20, 50, 100]\n", + "reg1_metric = 'S1'\n", + "reg2_metric = 'static-theta-0.5'\n", + "for i, degree in enumerate(degrees):\n", + " df = pd.read_csv(f'{base_dir}/{noise_type}-{degree}-scores.csv',index_col=0)\n", + " df_reg1 = df.loc[:, [reg1_metric]].rename(columns={reg1_metric:degree})\n", + " df_reg2 = df.loc[:, [reg2_metric]].rename(columns={reg2_metric:degree})\n", + " \n", + " if i == 0:\n", + " reg1_scores_layers = df_reg1\n", + " reg2_scores_layers = df_reg2\n", + " else:\n", + " reg1_scores_layers = pd.concat([reg1_scores_layers, df_reg1], axis=1)\n", + " reg2_scores_layers = pd.concat([reg2_scores_layers, df_reg2], axis=1)\n", + " \n", + "reg1_scores_layers = reg1_scores_layers.T\n", + "reg2_scores_layers = reg2_scores_layers.T\n", + "reg1_scores_layers.style.background_gradient()" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 4, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_sign/trace.txt to resources/results/robust_analy_reg2_sign/trace.txt\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_sign/state.yaml to resources/results/robust_analy_reg2_sign/state.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_sign/scores.yaml to resources/results/robust_analy_reg2_sign/scores.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_sign/metric_configs.yaml to resources/results/robust_analy_reg2_sign/metric_configs.yaml\n" - ] + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
 collectrinegative_controlpositive_controlpearson_corrpearson_causalportiappcorgenie3grnboost2scenicscgluecelloracle
00.5148960.5050020.5746080.5242320.5604900.5180480.5098740.5765800.6090750.5742940.5270760.580147
100.5141130.5047080.5746080.5242320.5604900.5180480.5098740.5765800.6090750.5742940.5270760.580147
200.5138570.5058990.5746080.5242320.5604900.5180480.5098740.5765800.6090750.5742940.5270760.580147
500.5120820.5063970.5746080.5242320.5604900.5180480.5098740.5765800.6090750.5742940.5270760.580147
1000.5083490.5082020.5746080.5242320.5604900.5180480.5098740.5765800.6090750.5742940.5270760.580147
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "!aws s3 sync s3://openproblems-data/resources/grn/results/robust_analy_reg2_sign ./resources/results/robust_analy_reg2_sign" + "reg2_scores_layers.style.background_gradient()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Corr vs causal" + "# repo" ] }, { @@ -3022,88 +3570,23 @@ "metadata": {}, "outputs": [], "source": [ - "!bash scripts/run_robust_analys_causal.sh " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "download: s3://openproblems-data/resources/grn/results/robust_analy_causal_1/scores.yaml to resources/results/robust_analy_causal_1/scores.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_causal_1/state.yaml to resources/results/robust_analy_causal_1/state.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_causal_1/trace.txt to resources/results/robust_analy_causal_1/trace.txt\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_causal_1/metric_configs.yaml to resources/results/robust_analy_causal_1/metric_configs.yaml\n" - ] - } - ], - "source": [ - "!aws s3 sync s3://openproblems-data/resources/grn/results/robust_analy_causal_1 ./resources/results/robust_analy_causal_1" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[37mRead data\u001b[0m\n", - "\u001b[37m\u001b[0m\n", - "\u001b[37mNoramlize data\u001b[0m\n", - "\u001b[37mCreate corr net\u001b[0m\n", - "\u001b[37mProcessing groups: 0%| | 0/4 [00:00 pd.DataFrame: - # Set global seed for reproducibility purposes random_state = SEED np.random.seed(random_state) @@ -282,8 +281,6 @@ def main(par: Dict[str, Any]) -> pd.DataFrame: n_genes = len(gene_names) groups = LabelEncoder().fit_transform(perturbation_data.obs.plate_name) - - grn = load_grn(par['prediction'], gene_names, par) # Load and standardize perturbation data diff --git a/src/metrics/regression_2/script_all.py b/src/metrics/regression_2/script_all.py deleted file mode 100644 index 76173788e..000000000 --- a/src/metrics/regression_2/script_all.py +++ /dev/null @@ -1,53 +0,0 @@ -import pandas as pd -import anndata as ad -import sys -import numpy as np -import os - -## VIASH START -par = { - 'reg_type': 'ridge', - 'read_dir': "resources/grn_models/d0_hvgs", - 'write_dir': "resources/results/d0_hvgs_ridge", - 'methods': [ 'collectri', 'negative_control', 'positive_control', 'pearson_corr', 'pearson_causal', 'portia', 'ppcor', 'genie3', 'grnboost2', 'scenic', 'scglue', 'celloracle'], - - - "perturbation_data": "resources/grn-benchmark/perturbation_data.h5ad", - "tf_all": "resources/prior/tf_all.csv", - "min_tf": False, - "max_n_links": 50000, - "apply_tf": "true", - 'layer': 'scgen_pearson', - 'subsample': -2, - 'num_workers': 4, - 'verbose': 1, - 'binarize': True, - 'num_workers': 20, - 'consensus': 'resources/prior/consensus-num-regulators.json', - 'static_only': True, - 'clip_scores': True -} -# VIASH END -meta = { - "resources_dir": 'src/metrics/regression_2/', - "util": 'src/utils' -} -sys.path.append(meta["resources_dir"]) -sys.path.append(meta["util"]) -from main import main - - -os.makedirs(par['write_dir'], exist_ok=True) - - -for i, method in enumerate(par['methods']): - par['prediction'] = f"{par['read_dir']}/{method}.csv" - prediction = main(par) - prediction.index = [method] - if i==0: - df_all = prediction - else: - df_all = pd.concat([df_all, prediction]) - df_all.to_csv(f"{par['write_dir']}/reg2.csv") - print(df_all) - diff --git a/src/metrics/regression_1/script_all.py b/src/metrics/script_all.py similarity index 80% rename from src/metrics/regression_1/script_all.py rename to src/metrics/script_all.py index d25c5faa1..289d70235 100644 --- a/src/metrics/regression_1/script_all.py +++ b/src/metrics/script_all.py @@ -10,11 +10,13 @@ 'read_dir': "resources/grn_models/d0_hvgs", 'write_dir': "resources/results/scores", 'methods': [ 'collectri', 'negative_control', 'positive_control', 'pearson_corr', 'pearson_causal', 'portia', 'ppcor', 'genie3', 'grnboost2', 'scenic', 'scglue', 'celloracle'], - 'layers': ['lognorm', 'pearson', 'scgen_lognorm', 'scgen_pearson'], + # 'layers': ['lognorm', 'pearson', 'seurat_lognorm', 'seurat_pearson', 'scgen_lognorm', 'scgen_pearson'], + 'layers': ['seurat_lognorm', 'seurat_pearson'], + + # 'layers': ['scgen_pearson'], "perturbation_data": "resources/grn-benchmark/perturbation_data.h5ad", "tf_all": "resources/prior/tf_all.csv", - "min_tf": False, "max_n_links": 50000, "apply_tf": "true", 'subsample': -2, @@ -44,12 +46,12 @@ reg1 = main(par) from regression_2.main import main reg2 = main(par) - prediction = pd.concat([reg1, reg2], axis=1) - prediction.index = [method] + score = pd.concat([reg1, reg2], axis=1) + score.index = [method] if i==0: - df_all = prediction + df_all = score else: - df_all = pd.concat([df_all, prediction]) + df_all = pd.concat([df_all, score]) df_all.to_csv(f"{par['write_dir']}/{layer}-{par['reg_type']}.csv") print(df_all) diff --git a/src/robustness_analysis/permute_grn/main.py b/src/robustness_analysis/permute_grn/main.py new file mode 100644 index 000000000..0206c87cd --- /dev/null +++ b/src/robustness_analysis/permute_grn/main.py @@ -0,0 +1,67 @@ + +import os +import pandas as pd +import numpy as np +def main(par): + degree = par['degree']/100 + type = par['noise_type'] + + + prediction = pd.read_csv(par['prediction']) + + + if type == 'weight': # add noise to weight + assert 'weight' in prediction.columns + print('Add noise to weight') + std_dev = prediction['weight'].std() + noise = np.random.normal(loc=0, scale=degree * std_dev, size=prediction['weight'].shape) + prediction['weight'] += noise + + elif type == 'net': # shuffle source-target matrix + print('Permute links') + + # 1. Pivot the GRN with target as index and source as columns + pivot_df = prediction.pivot(index='target', columns='source', values='weight') + + # Fill NaNs with 0 or a value of your choice + pivot_df.fillna(0, inplace=True) + + # 2. Randomly choose degree% of the matrix to shuffle + matrix_flattened = pivot_df.values.flatten() + n_elements = len(matrix_flattened) + n_shuffle = int(n_elements * degree) + + # Randomly select 20% of the matrix elements' indices + shuffle_indices = np.random.choice(n_elements, n_shuffle, replace=False) + + # Get the values that will be shuffled + shuffle_values = matrix_flattened[shuffle_indices] + + # 3. Shuffle the selected values + np.random.shuffle(shuffle_values) + + # Assign the shuffled values back to the selected positions + matrix_flattened[shuffle_indices] = shuffle_values + + # Reshape the flattened array back into the matrix + pivot_df_shuffled = pd.DataFrame(matrix_flattened.reshape(pivot_df.shape), + index=pivot_df.index, + columns=pivot_df.columns) + + flat_df = pivot_df_shuffled.reset_index() + + # Melt the DataFrame to turn it back into long-form (source-target-weight) + prediction = flat_df.melt(id_vars='target', var_name='source', value_name='weight') + prediction = prediction[prediction['weight'] !=0 ].reset_index(drop=True) + elif type == 'sign': # change the regulatory sign + num_rows = len(prediction) + num_to_modify = int(num_rows * degree) + # 2. Randomly select indices to modify + random_indices = np.random.choice(prediction.index, size=num_to_modify, replace=False) + # 3. Change the sign of the selected rows + prediction.loc[random_indices, 'weight'] *= -1 + elif type == 'binary': # change the regulatory sign + prediction['weight'] = np.where(prediction['weight'] > 0, 1, -1) + else: + raise ValueError(f'Wrong type ({type}) for adding noise') + return prediction diff --git a/src/robustness_analysis/permute_grn/script.py b/src/robustness_analysis/permute_grn/script.py index da6664ea8..9bce7b750 100644 --- a/src/robustness_analysis/permute_grn/script.py +++ b/src/robustness_analysis/permute_grn/script.py @@ -9,71 +9,8 @@ 'degree': 20, 'noise_type': 'links' } - ## VIASH END - -degree = par['degree']/100 -type = par['noise_type'] - - -prediction = pd.read_csv(par['prediction']) - - -if type == 'weight': # add noise to weight - assert 'weight' in prediction.columns - print('Add noise to weight') - std_dev = prediction['weight'].std() - noise = np.random.normal(loc=0, scale=degree * std_dev, size=prediction['weight'].shape) - prediction['weight'] += noise - -elif type == 'net': # shuffle source-target matrix - print('Permute links') - - # 1. Pivot the GRN with target as index and source as columns - pivot_df = prediction.pivot(index='target', columns='source', values='weight') - - # Fill NaNs with 0 or a value of your choice - pivot_df.fillna(0, inplace=True) - - # 2. Randomly choose degree% of the matrix to shuffle - matrix_flattened = pivot_df.values.flatten() - n_elements = len(matrix_flattened) - n_shuffle = int(n_elements * degree) - - # Randomly select 20% of the matrix elements' indices - shuffle_indices = np.random.choice(n_elements, n_shuffle, replace=False) - - # Get the values that will be shuffled - shuffle_values = matrix_flattened[shuffle_indices] - - # 3. Shuffle the selected values - np.random.shuffle(shuffle_values) - - # Assign the shuffled values back to the selected positions - matrix_flattened[shuffle_indices] = shuffle_values - - # Reshape the flattened array back into the matrix - pivot_df_shuffled = pd.DataFrame(matrix_flattened.reshape(pivot_df.shape), - index=pivot_df.index, - columns=pivot_df.columns) - - flat_df = pivot_df_shuffled.reset_index() - - # Melt the DataFrame to turn it back into long-form (source-target-weight) - prediction = flat_df.melt(id_vars='target', var_name='source', value_name='weight') - prediction = prediction[prediction['weight'] !=0 ].reset_index(drop=True) -elif type == 'sign': # change the regulatory sign - num_rows = len(prediction) - num_to_modify = int(num_rows * degree) - # 2. Randomly select indices to modify - random_indices = np.random.choice(prediction.index, size=num_to_modify, replace=False) - # 3. Change the sign of the selected rows - prediction.loc[random_indices, 'weight'] *= -1 -elif type == 'binary': # change the regulatory sign - prediction['weight'] = np.where(prediction['weight'] > 0, 1, -1) -else: - raise ValueError(f'Wrong type ({type}) for adding noise') - print('Output noised GRN') +prediction = main(par) prediction.to_csv(par['prediction_n']) diff --git a/src/robustness_analysis/script_all.py b/src/robustness_analysis/script_all.py new file mode 100644 index 000000000..84a31732a --- /dev/null +++ b/src/robustness_analysis/script_all.py @@ -0,0 +1,130 @@ +import pandas as pd +import anndata as ad +import sys +import numpy as np +import os +import random + +par = { + 'reg_type': 'ridge', + 'read_dir': "resources/grn_models/d0_hvgs", + 'write_dir': "resources/results/robustness_analysis", + 'degrees': [0, 10, 20, 50, 100], + 'noise_types': ["net", "sign"], + 'methods': [ 'collectri', 'negative_control', 'positive_control', 'pearson_corr', 'pearson_causal', 'portia', 'ppcor', 'genie3', 'grnboost2', 'scenic', 'scglue', 'celloracle'], + + + "perturbation_data": "resources/grn-benchmark/perturbation_data.h5ad", + "tf_all": "resources/prior/tf_all.csv", + "max_n_links": 50000, + "apply_tf": "true", + 'subsample': -2, + 'num_workers': 4, + 'verbose': 3, + 'binarize': True, + 'num_workers': 20, + 'consensus': 'resources/prior/consensus-num-regulators.json', + 'static_only': True, + 'clip_scores': True, + 'layer': 'scgen_pearson', +} + +meta = { + "resources_dir": 'src/', + "util": 'src/utils' +} +sys.path.append(meta["resources_dir"]) +sys.path.append(meta["util"]) + +os.makedirs(par['write_dir'], exist_ok=True) +os.makedirs(f"{par['write_dir']}/tmp/", exist_ok=True) + + +def run_reg(par): + from metrics.regression_1.main import main + reg1 = main(par) + return reg1 + from metrics.regression_2.main import main + reg2 = main(par) + score = pd.concat([reg1, reg2], axis=1) + return score + +#------ noise types and degrees ------# +if False: + for noise_type in par['noise_types']: # run for each noise type (net, sign, weight) + for degree in par['degrees']: # run for each degree + for i, method in enumerate(par['methods']): # run for each method + par['prediction'] = f"{par['read_dir']}/{method}.csv" + par['prediction_n'] = f"{par['write_dir']}/tmp/{method}.csv" + par['degree'] = degree + par['noise_type'] = noise_type + + # permute + from robustness_analysis.permute_grn.main import main + prediction_n = main(par) + prediction_n.to_csv(par['prediction_n']) + + # run regs + par['prediction'] = par['prediction_n'] + score = run_reg(par) + score.index = [method] + if i==0: + df_all = score + else: + df_all = pd.concat([df_all, score]) + df_all.to_csv(f"{par['write_dir']}/{noise_type}-{degree}-scores.csv") + print(df_all) + +#------ causal vs corr ------# +if True: + from util import create_corr_net + par = { + 'reg_type': 'ridge', + 'write_dir': "resources/results/robustness_analysis", + ## base corr + "perturbation_data": "resources/grn-benchmark/perturbation_data.h5ad", + 'cell_type_specific': False, + 'normalize': False, + ## metric + 'multiomics_rna': 'resources/grn-benchmark/multiomics_rna_d0_hvg.h5ad', + "tf_all": "resources/prior/tf_all.csv", + "max_n_links": 50000, + "apply_tf": False, #this has to be false + 'subsample': -2, + 'verbose': 2, + 'binarize': True, + 'num_workers': 20, + 'consensus': 'resources/prior/consensus-num-regulators.json', + 'static_only': True, + 'clip_scores': True, + 'layer': 'scgen_pearson', + } + + # # run for corr + # os.makedirs(f"{par['write_dir']}/corr/", exist_ok=True) + # par['causal'] = False + # for i in range(100): + # par['causal'] + # par['prediction'] = f"{par['write_dir']}/corr/corr.csv" + # par['seed'] = i + # random.seed(par['seed']) + # print('seed :', par['seed']) + + # net = create_corr_net(par) + # net.to_csv(par['prediction']) + # score = run_reg(par) + # if i == 0: + # scores_corr = score + # else: + # scores_corr = pd.concat([score, scores_corr], axis=0) + # print(scores_corr) + # scores_corr.to_csv(f"{par['write_dir']}/corr/scores_corr.csv") + + # run for causal corr + par['prediction'] = f"{par['write_dir']}/corr/corr_causal.csv" + par['causal'] = True + net = create_corr_net(par) + net.to_csv(par['prediction']) + score = run_reg(par) + print(score) + score.to_csv(f"{par['write_dir']}/corr/scores_causal.csv") \ No newline at end of file diff --git a/src/utils/util.py b/src/utils/util.py index 9339483ce..62e069e74 100644 --- a/src/utils/util.py +++ b/src/utils/util.py @@ -14,7 +14,23 @@ def verbose_tqdm(iterable, desc, level, verbose_level): else: return iterable # Return the iterable without a progress bar +def basic_qc(adata, min_genes_per_cell = 200, max_genes_per_cell = 5000, max_mt_frac = 0.05, min_cells_per_gene = 10): + mt = adata.var_names.str.startswith('MT-') + # 1. stats + total_counts = adata.X.sum(axis=1) + n_genes_by_counts = (adata.X > 0).sum(axis=1) + mt_frac = adata[:, mt].X.sum(axis=1) / total_counts + + # 2. Filter cells + mask_cells= (n_genes_by_counts > min_genes_per_cell)& \ + (n_genes_by_counts < max_genes_per_cell)&\ + (mt_frac < max_mt_frac) + # 3. Filter genes + n_cells = (adata.X!=0).sum(axis=0) + mask_genes = n_cells>min_cells_per_gene + + return adata[mask_cells, mask_genes] def process_links(net, par): net = net[net.source!=net.target] net_sorted = net.reindex(net['weight'].abs().sort_values(ascending=False).index) @@ -26,8 +42,10 @@ def corr_net(X, gene_names, par, tf_all, causal=False): net = np.dot(X.T, X) / X.shape[0] net = pd.DataFrame(net, index=gene_names, columns=gene_names) if causal: + print('Causal subsetting') net = net[tf_all] else: + print('Random subsetting') net = net.sample(len(tf_all), axis=1, random_state=par['seed']) net = net.reset_index() index_name = net.columns[0]