find_correlations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from influence_info.influencers.experiments.pairwise_applicator import *\n",
    "\n",
    "dataset = \"IMDB\"\n",
    "\n",
    "import os\n",
    "os.makedirs(f\"figures/{dataset}\", exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "values, training_idx, validation_idx = get_all_values(\n",
    "    f\"outputs/{dataset}/basic/transformer_text_classifier\",\n",
    "    ([\"influence_function/*\"] if \"full\" not in dataset else []) +\n",
    "    [\n",
    "        # \"tracein_softmax\",\n",
    "        \"influence_function_softmax/*\",\n",
    "        \"representer_points_with_sec\",\n",
    "        \"nearest_neighbors/*\",\n",
    "        # \"nearest_neighbors_sep/*\",\n",
    "    ],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "values[5][2].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "predictions = [\n",
    "    json.loads(line)\n",
    "    for line in open(f\"outputs/{dataset}/basic/transformer_text_classifier/predictions.test.jsonl\")\n",
    "]\n",
    "predictions = {x[\"idx\"]: x for x in predictions}\n",
    "validation_predicted_labels = [predictions[x][\"predicted_labels\"] for x in validation_idx]\n",
    "\n",
    "predictions = [\n",
    "    json.loads(line)\n",
    "    for line in open(f\"outputs/{dataset}/basic/transformer_text_classifier/predictions.train.jsonl\")\n",
    "]\n",
    "predictions = {x[\"idx\"]: x for x in predictions}\n",
    "training_predicted_labels = [predictions[x][\"predicted_labels\"] for x in training_idx]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "for i, val in enumerate(values):\n",
    "    if not (\n",
    "        val[0] == \"nearest_neighbors\" or val[0] == \"influence_function\" or val[0] == \"nearest_neighbors_sep\"\n",
    "    ):\n",
    "        values[i] = (val[0], val[1], val[2][np.arange(val[2].shape[0]), :, validation_predicted_labels])\n",
    "        print(values[i][2].shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import spearmanr\n",
    "\n",
    "applicator = mean_pairwise_applicator(lambda x, y: spearmanr(x.flatten(), y.flatten()).correlation)\n",
    "\n",
    "top_n_func = lambda x, y, n: len(set(np.argsort(x)[-n:]) & set(np.argsort(y)[-n:])) / float(n)\n",
    "top_n_applicator = lambda n: pairwise_applicator(lambda x, y: top_n_func(x, y, n), np.array)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vals = [5, 10, 20, 30, 50, 80, 100, 200, 400, 500] if \"full\" not in dataset else [10, 50]\n",
    "ys = []\n",
    "for n in vals:\n",
    "    triplets = pairwise_experiment(values, top_n_applicator(n))\n",
    "    ys.append(triplets[2][2])\n",
    "    if n == 10:\n",
    "        top10 = triplets\n",
    "    if n == 50:\n",
    "        top50 = triplets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ys = np.array(ys)\n",
    "ys = pd.DataFrame(ys)\n",
    "ys.index = vals\n",
    "ys = ys.melt(ignore_index=False).reset_index()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "if \"full\" not in dataset :\n",
    "    ax = sns.lineplot(data=ys, x=\"index\", y=\"value\")\n",
    "    ax.set_xlabel(\"N\")\n",
    "    ax.set_ylabel(\"proportion of common elements\")\n",
    "    ax.get_figure().savefig(f\"figures/{dataset}/top_N.pdf\", bbox_inches=\"tight\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = plt.figure(figsize=(8, 5))\n",
    "ax = plot_pairwise_matrix([(x, y, np.mean(v)) for x, y, v in top10])\n",
    "plt.show()\n",
    "ax.get_figure().savefig(f\"figures/{dataset}/top_10.pdf\", bbox_inches=\"tight\")\n",
    "\n",
    "f = plt.figure(figsize=(8, 5))\n",
    "ax = plot_pairwise_matrix([(x, y, np.mean(v)) for x, y, v in top50])\n",
    "plt.show()\n",
    "ax.get_figure().savefig(f\"figures/{dataset}/top_50.pdf\", bbox_inches=\"tight\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "triplets = pairwise_experiment(values, applicator)\n",
    "f = plt.figure(figsize=(8, 5))\n",
    "ax = plot_pairwise_matrix(triplets)\n",
    "ax.get_figure().savefig(f\"figures/{dataset}/corr.pdf\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if \"full\" not in dataset :\n",
    "    no_sent_bert_triplets = removefilter(\n",
    "        triplets,\n",
    "        [\n",
    "            ('nearest_neighbors_sep', 'sim_func:cos'),\n",
    "            ('nearest_neighbors_sep', 'sim_func:dot'),\n",
    "            ('nearest_neighbors_sep', 'sim_func:euc')\n",
    "        ],\n",
    "    )\n",
    "    ax = plot_pairwise_matrix(no_sent_bert_triplets)\n",
    "    ax.get_figure().savefig(f\"figures/{dataset}/corr_wo_sentbert.pdf\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions[\"train_0\"][\"logits\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5-final"
  },
  "orig_nbformat": 2
 },
 "nbformat": 4,
 "nbformat_minor": 2
}