source/inseq_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "CUDA SETUP: Required library version not found: libsbitsandbytes_cpu.so. Maybe you need to compile it from source?\n",
      "CUDA SETUP: Defaulting to libbitsandbytes_cpu.so...\n",
      "dlopen(/Users/barberademol/Desktop/ik-nlp-tutorials/venv/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cpu.so, 0x0006): tried: '/Users/barberademol/Desktop/ik-nlp-tutorials/venv/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cpu.so' (not a mach-o file), '/System/Volumes/Preboot/Cryptexes/OS/Users/barberademol/Desktop/ik-nlp-tutorials/venv/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cpu.so' (no such file), '/Users/barberademol/Desktop/ik-nlp-tutorials/venv/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cpu.so' (not a mach-o file)\n",
      "CUDA SETUP: Required library version not found: libsbitsandbytes_cpu.so. Maybe you need to compile it from source?\n",
      "CUDA SETUP: Defaulting to libbitsandbytes_cpu.so...\n",
      "dlopen(/Users/barberademol/Desktop/ik-nlp-tutorials/venv/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cpu.so, 0x0006): tried: '/Users/barberademol/Desktop/ik-nlp-tutorials/venv/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cpu.so' (not a mach-o file), '/System/Volumes/Preboot/Cryptexes/OS/Users/barberademol/Desktop/ik-nlp-tutorials/venv/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cpu.so' (no such file), '/Users/barberademol/Desktop/ik-nlp-tutorials/venv/lib/python3.8/site-packages/bitsandbytes/libbitsandbytes_cpu.so' (not a mach-o file)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/barberademol/Desktop/ik-nlp-tutorials/venv/lib/python3.8/site-packages/bitsandbytes/cextension.py:31: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers and GPU quantization are unavailable.\n",
      "  warn(\"The installed version of bitsandbytes was compiled without GPU support. \"\n",
      "/Users/barberademol/Desktop/ik-nlp-tutorials/venv/lib/python3.8/site-packages/torchmetrics/text/bert.py:179: UserWarning: The argument `model_name_or_path` was not specified while it is required when the default `transformers` model is used. It will use the default recommended model - 'roberta-large'.\n",
      "  warn(\n"
     ]
    }
   ],
   "source": [
    "# load the explanation model to Inseq\n",
    "import inseq\n",
    "\n",
    "from t5_lit_module import LitT5\n",
    "\n",
    "# Load model from checkpoint\n",
    "model_explanation = LitT5.load_from_checkpoint(\n",
    "    checkpoint_path=\"checkpoints/model_full.ckpt\",\n",
    ")\n",
    "\n",
    "inseq_model_explanation = inseq.load_model(model_explanation.model, \"saliency\")\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Investigate how the output is generated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/barberademol/Desktop/ik-nlp-tutorials/venv/lib/python3.8/site-packages/transformers/generation/utils.py:1288: UserWarning: Using `max_length`'s default (20) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Attributing with saliency...: 100%|██████████| 14/14 [00:06<00:00,  1.87it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<br/><b>0th instance:</b><br/>\n",
       "<html>\n",
       "<div id=\"usieyiccwpusnzjovqwp_viz_container\">\n",
       "    <div id=\"usieyiccwpusnzjovqwp_content\" style=\"padding:15px;border-style:solid;margin:5px;\">\n",
       "        <div id = \"usieyiccwpusnzjovqwp_saliency_plot_container\" class=\"usieyiccwpusnzjovqwp_viz_container\" style=\"display:block\">\n",
       "            \n",
       "<div id=\"qahftrxckafnafqofpva_saliency_plot\" class=\"qahftrxckafnafqofpva_viz_content\">\n",
       "    <div style=\"margin:5px;font-family:sans-serif;font-weight:bold;\">\n",
       "        <span style=\"font-size: 20px;\">Source Saliency Heatmap</span>\n",
       "        <br>\n",
       "        x: Generated tokens, y: Attributed tokens\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" cellpadding=\"5\" cellspacing=\"5\"\n",
       "    style=\"overflow-x:scroll;display:block;\">\n",
       "    <tr><th></th>\n",
       "<th>▁A</th><th>▁big</th><th>▁dog</th><th>▁and</th><th>▁</th><th>a</th><th>▁smaller</th><th>▁dog</th><th>▁are</th><th>▁two</th><th>▁dogs</th><th>.</th><th>&lt;/s&gt;</th></tr><tr><th>▁</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10924935630817992)\">0.041</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.029</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.029</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.028</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.025</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.027</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.02</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.023</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.032</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.019</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.03</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1171321053673995)\">0.043</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14866310160427795)\">0.054</th></tr><tr><th>premise</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.009</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.007</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.008</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.007</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.009</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th></tr><tr><th>:</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.029</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.022</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.024</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.021</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.018</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.019</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.017</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.021</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.02</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.019</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.017</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.019</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.016</th></tr><tr><th>▁A</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.2826698356110118)\">0.099</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1171321053673995)\">0.041</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10136660724896006)\">0.038</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.029</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.028</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.032</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.023</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.022</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.024</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.023</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.02</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.016</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th></tr><tr><th>▁big</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.23537334125569415)\">0.083</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.55068330362448)\">0.193</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.29843533372945136)\">0.107</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.17231134878193693)\">0.061</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.20384234501881549)\">0.072</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.16442859972271742)\">0.059</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.20384234501881549)\">0.074</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1171321053673995)\">0.042</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10136660724896006)\">0.037</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.12501485442661908)\">0.045</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.12501485442661908)\">0.045</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.029</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.015</th></tr><tr><th>▁dog</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1171321053673995)\">0.042</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14866310160427795)\">0.054</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.2747870865517925)\">0.097</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14866310160427795)\">0.053</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.031</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.09348385818974037)\">0.035</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.025</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.17231134878193693)\">0.062</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10136660724896006)\">0.037</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.032</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10136660724896006)\">0.037</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.021</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.018</th></tr><tr><th>▁and</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.17231134878193693)\">0.06</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.12501485442661908)\">0.044</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.16442859972271742)\">0.06</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.24325609031491383)\">0.088</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.20384234501881549)\">0.074</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.22749059219647458)\">0.08</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14866310160427795)\">0.054</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.17231134878193693)\">0.061</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1959595959595959)\">0.069</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.16442859972271742)\">0.059</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14866310160427795)\">0.052</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10924935630817992)\">0.041</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.031</th></tr><tr><th>▁</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.019</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.018</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.025</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.03</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10924935630817992)\">0.039</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1171321053673995)\">0.044</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.03</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.025</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.022</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.022</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.017</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.02</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.017</th></tr><tr><th>a</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.09348385818974037)\">0.034</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.022</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.033</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1171321053673995)\">0.041</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.2117250940780353)\">0.076</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.24325609031491383)\">0.086</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.12501485442661908)\">0.046</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.032</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.028</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.025</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.027</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.024</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.017</th></tr><tr><th>▁smaller</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14866310160427795)\">0.053</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1959595959595959)\">0.069</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.2747870865517925)\">0.098</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.24325609031491383)\">0.086</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.6137452960982372)\">0.216</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.44032481679540497)\">0.154</th><th style=\"background:rgba(255.0, 13.0, 87.0, 1.0)\">0.351</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.22749059219647458)\">0.081</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.13289760348583876)\">0.048</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.13289760348583876)\">0.047</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10924935630817992)\">0.04</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10924935630817992)\">0.039</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.021</th></tr><tr><th>▁dog</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10924935630817992)\">0.04</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.12501485442661908)\">0.045</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.16442859972271742)\">0.058</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14866310160427795)\">0.053</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1171321053673995)\">0.043</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.12501485442661908)\">0.046</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10136660724896006)\">0.038</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.36149732620320857)\">0.128</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.12501485442661908)\">0.047</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.032</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1171321053673995)\">0.044</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14078035254505847)\">0.05</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.17231134878193693)\">0.061</th></tr><tr><th>▁fight</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.20384234501881549)\">0.073</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1880768469003763)\">0.068</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14078035254505847)\">0.051</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.24325609031491383)\">0.087</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14866310160427795)\">0.053</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.15654585066349747)\">0.055</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.023</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.20384234501881549)\">0.073</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.22749059219647458)\">0.081</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.17231134878193693)\">0.063</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1959595959595959)\">0.071</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.33784907902554967)\">0.12</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.4245593186769657)\">0.149</th></tr><tr><th>▁over</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10136660724896006)\">0.036</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.031</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.023</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.032</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.033</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.027</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.014</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10136660724896006)\">0.036</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.032</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.02</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10136660724896006)\">0.037</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14078035254505847)\">0.052</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1959595959595959)\">0.071</th></tr><tr><th>▁</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.009</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.009</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.014</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.02</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.018</th></tr><tr><th>a</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.015</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.015</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.014</th></tr><tr><th>▁piece</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.014</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.009</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.008</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.008</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.014</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.011</th></tr><tr><th>▁of</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.014</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.015</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.017</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.017</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.014</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.02</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.018</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.018</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.018</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.017</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.017</th></tr><tr><th>▁fabric</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.021</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.026</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.015</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.014</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10924935630817992)\">0.04</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.019</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.022</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.026</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.017</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.015</th></tr><tr><th>.</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.13289760348583876)\">0.048</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14866310160427795)\">0.055</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1171321053673995)\">0.042</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.16442859972271742)\">0.058</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14866310160427795)\">0.054</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.13289760348583876)\">0.048</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.23537334125569415)\">0.084</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.12501485442661908)\">0.045</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14866310160427795)\">0.052</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.15654585066349747)\">0.057</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.13289760348583876)\">0.049</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.17231134878193693)\">0.061</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.15654585066349747)\">0.057</th></tr><tr><th>▁hypothesis</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.007</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.008</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.006</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.007</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.006</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.008</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.014</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.015</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.014</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th></tr><tr><th>:</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.02</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.015</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.016</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.021</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.014</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.009</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.014</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.027</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.03</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.026</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.026</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.023</th></tr><tr><th>▁Two</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1171321053673995)\">0.041</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.09348385818974037)\">0.033</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.03</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.16442859972271742)\">0.06</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.029</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10136660724896006)\">0.037</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.018</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.031</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.3299663299663301)\">0.116</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.4087938205585263)\">0.144</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.17231134878193693)\">0.062</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10136660724896006)\">0.037</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.029</th></tr><tr><th>▁dogs</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10136660724896006)\">0.036</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.09348385818974037)\">0.033</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1171321053673995)\">0.043</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.12501485442661908)\">0.045</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.019</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.026</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.016</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.12501485442661908)\">0.046</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.15654585066349747)\">0.057</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.17231134878193693)\">0.062</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.3536145771439889)\">0.125</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10924935630817992)\">0.039</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.09348385818974037)\">0.033</th></tr><tr><th>▁fighting</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1880768469003763)\">0.066</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.13289760348583876)\">0.049</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.09348385818974037)\">0.033</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10136660724896006)\">0.036</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.031</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10924935630817992)\">0.039</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.031</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1880768469003763)\">0.068</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.2590215884333532)\">0.091</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.20384234501881549)\">0.074</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.3772628243216479)\">0.133</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.39302832244008706)\">0.139</th></tr><tr><th>▁over</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.09348385818974037)\">0.034</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.022</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.02</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.02</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.018</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.021</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.022</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.09348385818974037)\">0.033</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.024</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.09348385818974037)\">0.036</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.12501485442661908)\">0.046</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.17231134878193693)\">0.061</th></tr><tr><th>▁</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.009</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.008</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.009</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.009</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.008</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.009</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.008</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th></tr><tr><th>a</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.009</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.017</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.014</th></tr><tr><th>▁piece</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.008</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.008</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.007</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.009</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.007</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.014</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th></tr><tr><th>▁of</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.016</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.015</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.014</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.015</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.019</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.015</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.019</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.016</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.015</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.014</th></tr><tr><th>▁fabric</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.017</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.021</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.031</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.016</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.021</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.025</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.016</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th></tr><tr><th>&lt;/s&gt;</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.009</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.013</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.021</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.009</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.009</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.015</th></tr></table>\n",
       "</div>\n",
       "\n",
       "        </div>\n",
       "    </div>\n",
       "</div>\n",
       "</html>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "inseq_model_explanation.attribute(\n",
    "    \"premise: A big dog and a smaller dog fight over a piece of fabric. hypothesis: Two dogs fighting over a piece of fabric\"\n",
    ").show()\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Directly comparing two forced outputs"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can use PairAggregator to compare the feature importance of two different models that perform the explanation task to identify the strengths and weaknesses of each model and potentially improve their performance.\n",
    "\n",
    "Note: Outputs must have the same length. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Attributing with saliency...: 100%|██████████| 14/14 [00:19<00:00,  1.53s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<br/><b>0th instance:</b><br/>\n",
       "<html>\n",
       "<div id=\"usieyiccwpusnzjovqwp_viz_container\">\n",
       "    <div id=\"usieyiccwpusnzjovqwp_content\" style=\"padding:15px;border-style:solid;margin:5px;\">\n",
       "        <div id = \"usieyiccwpusnzjovqwp_saliency_plot_container\" class=\"usieyiccwpusnzjovqwp_viz_container\" style=\"display:block\">\n",
       "            \n",
       "<div id=\"qahftrxckafnafqofpva_saliency_plot\" class=\"qahftrxckafnafqofpva_viz_content\">\n",
       "    <div style=\"margin:5px;font-family:sans-serif;font-weight:bold;\">\n",
       "        <span style=\"font-size: 20px;\">Source Saliency Heatmap</span>\n",
       "        <br>\n",
       "        x: Generated tokens, y: Attributed tokens\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" cellpadding=\"5\" cellspacing=\"5\"\n",
       "    style=\"overflow-x:scroll;display:block;\">\n",
       "    <tr><th></th>\n",
       "<th>▁A</th><th>▁big▁dog</th><th>▁and</th><th>▁</th><th>a</th><th>▁smaller</th><th>▁dog</th><th>▁are</th><th>▁two → ▁three</th><th>▁dogs</th><th>.</th><th>&lt;/s&gt;</th></tr><tr><th>▁</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.003</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.03042186571598325)\">-0.004</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.06195286195286191)\">-0.008</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.022539116656763607)\">-0.003</th></tr><tr><th>premise</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.003</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th></tr><tr><th>:</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.022539116656763607)\">-0.003</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.001</th></tr><tr><th>▁A</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.002</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th></tr><tr><th>▁big</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.003</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.002</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th></tr><tr><th>▁dog</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.001</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.001</th></tr><tr><th>▁and</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.006</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.014656367597544028)\">-0.003</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.03830461477520309)\">-0.005</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.002</th></tr><tr><th>▁</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.002</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.014656367597544028)\">-0.002</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.001</th></tr><tr><th>a</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.005</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.014656367597544028)\">-0.002</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.022539116656763607)\">-0.003</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.001</th></tr><tr><th>▁smaller</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.01</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.06195286195286191)\">-0.008</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.001</th></tr><tr><th>▁dog</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.001</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.03042186571598325)\">-0.004</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.10136660724896014)\">-0.012</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.03830461477520309)\">-0.005</th></tr><tr><th>▁fight</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.06195286195286191)\">-0.008</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.1723113487819369)\">-0.02</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.2432560903149138)\">-0.028</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.08560110913052085)\">-0.01</th></tr><tr><th>▁over</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.002</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.10136660724896014)\">-0.012</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.09348385818974048)\">-0.011</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.04618736383442258)\">-0.005</th></tr><tr><th>▁</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.002</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.001</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.03042186571598325)\">-0.004</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.001</th></tr><tr><th>a</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.001</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.014656367597544028)\">-0.002</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.001</th></tr><tr><th>▁piece</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.001</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.001</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.001</th></tr><tr><th>▁of</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.014656367597544028)\">-0.002</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.022539116656763607)\">-0.003</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.001</th></tr><tr><th>▁fabric</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.03042186571598325)\">-0.004</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.03830461477520309)\">-0.005</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.014656367597544028)\">-0.003</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th></tr><tr><th>.</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.15654585066349747)\">0.018</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.022539116656763607)\">-0.003</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.10924935630817977)\">-0.013</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.022539116656763607)\">-0.003</th></tr><tr><th>▁hypothesis</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.002</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.014656367597544028)\">-0.002</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th></tr><tr><th>:</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.005</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.022539116656763607)\">-0.003</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.03042186571598325)\">-0.004</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.001</th></tr><tr><th>▁Two</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.9605862547039017)\">0.109</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.09348385818974037)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.006</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>▁dogs</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.004</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.16442859972271748)\">-0.019</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.003</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.001</th></tr><tr><th>▁fighting</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.10136660724896014)\">-0.012</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.14078035254505836)\">-0.017</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.23537334125569417)\">-0.027</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.06983561101208147)\">-0.008</th></tr><tr><th>▁over</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.003</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.09348385818974048)\">-0.011</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.06983561101208147)\">-0.009</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.03830461477520309)\">-0.005</th></tr><tr><th>▁</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.002</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.014656367597544028)\">-0.002</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th></tr><tr><th>a</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.002</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.002</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.001</th></tr><tr><th>▁piece</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th></tr><tr><th>▁of</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.001</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.022539116656763607)\">-0.003</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.014656367597544028)\">-0.002</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.001</th></tr><tr><th>▁fabric</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.022539116656763607)\">-0.003</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.06195286195286191)\">-0.007</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.014656367597544028)\">-0.002</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th></tr><tr><th>&lt;/s&gt;</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.002</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.014656367597544028)\">-0.002</th><th style=\"background:rgba(54.70588235294111, 122.49411764705886, 213.40784313725496, 0.0)\">-0.001</th></tr><tr style=\"outline: thin solid\"><th><b>probability</b></th><th>-0.0</th><th>-0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th><b>-0.878</b></th><th>-0.013</th><th>-0.177</th><th>-0.061</th></table>\n",
       "</div>\n",
       "\n",
       "        </div>\n",
       "    </div>\n",
       "</div>\n",
       "</html>\n",
       "<br/><b>0th instance:</b><br/>\n",
       "<html>\n",
       "<div id=\"zqorvufaigfywirkxlgg_viz_container\">\n",
       "    <div id=\"zqorvufaigfywirkxlgg_content\" style=\"padding:15px;border-style:solid;margin:5px;\">\n",
       "        <div id = \"zqorvufaigfywirkxlgg_saliency_plot_container\" class=\"zqorvufaigfywirkxlgg_viz_container\" style=\"display:block\">\n",
       "            \n",
       "<div id=\"sbfhcgchqjjfgyqpesej_saliency_plot\" class=\"sbfhcgchqjjfgyqpesej_viz_content\">\n",
       "    <div style=\"margin:5px;font-family:sans-serif;font-weight:bold;\">\n",
       "        <span style=\"font-size: 20px;\">Target Saliency Heatmap</span>\n",
       "        <br>\n",
       "        x: Generated tokens, y: Attributed tokens\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" cellpadding=\"5\" cellspacing=\"5\"\n",
       "    style=\"overflow-x:scroll;display:block;\">\n",
       "    <tr><th></th>\n",
       "<th>▁A</th><th>▁big▁dog</th><th>▁and</th><th>▁</th><th>a</th><th>▁smaller</th><th>▁dog</th><th>▁are</th><th>▁two → ▁three</th><th>▁dogs</th><th>.</th><th>&lt;/s&gt;</th></tr><tr><th>▁A</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.08560110913052085)\">-0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.002</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.002</th></tr><tr><th>▁big▁dog</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.10136660724896014)\">-0.012</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.07771836007130117)\">-0.009</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.03042186571598325)\">-0.004</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>▁and</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.14866310160427798)\">-0.018</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.06983561101208147)\">-0.008</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.03042186571598325)\">-0.004</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th></tr><tr><th>▁</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.10136660724896014)\">-0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.005</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.003</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th></tr><tr><th>a</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.08560110913052085)\">-0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.005</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.003</th></tr><tr><th>▁smaller</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.14078035254505836)\">-0.017</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.007</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.003</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.007</th></tr><tr><th>▁dog</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.16442859972271748)\">-0.019</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.04618736383442258)\">-0.006</th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.006773618538324436)\">-0.001</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th></tr><tr><th>▁are</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(30.0, 136.0, 229.0, 0.3772628243216478)\">-0.043</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.15654585066349747)\">0.018</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.15654585066349747)\">0.018</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.007</th></tr><tr><th>▁two → ▁three</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.8265795206971679)\">0.094</th><th style=\"background:rgba(255.0, 13.0, 87.0, 1.0)\">0.113</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.22749059219647458)\">0.027</th></tr><tr><th>▁dogs</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.15654585066349747)\">0.018</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.004</th></tr><tr><th>.</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.008</th></tr><tr><th>&lt;/s&gt;</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th></tr><tr style=\"outline: thin solid\"><th><b>probability</b></th><th>-0.0</th><th>-0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th><b>-0.878</b></th><th>-0.013</th><th>-0.177</th><th>-0.061</th></table>\n",
       "</div>\n",
       "\n",
       "        </div>\n",
       "    </div>\n",
       "</div>\n",
       "</html>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from inseq.data.aggregator import AggregatorPipeline, ContiguousSpanAggregator, SequenceAttributionAggregator, PairAggregator\n",
    "\n",
    "# Perform the attribution with forced decoding. Return convergence deltas, probabilities and target attributions.\n",
    "out = inseq_model_explanation.attribute(\n",
    "    [\n",
    "        \"premise: A big dog and a smaller dog fight over a piece of fabric. hypothesis: Two dogs fighting over a piece of fabric\",\n",
    "        \"premise: A big dog and a smaller dog fight over a piece of fabric. hypothesis: Two dogs fighting over a piece of fabric\",\n",
    "    ],\n",
    "    [\n",
    "        \"A big dog and a smaller dog are two dogs.\",\n",
    "        \"A big dog and a smaller dog are three dogs.\",\n",
    "    ],\n",
    "    attribute_target=True,\n",
    "    step_scores=[\"probability\"],\n",
    "    include_eos_baseline=False,\n",
    ")\n",
    "\n",
    "# Aggregation pipeline composed by two steps:\n",
    "# 1. Aggregate contiguous tokens across all attribution dimensions\n",
    "# 2. Aggregate the last dimension of the neuron-level attribution to make it token-level\n",
    "squeezesum = AggregatorPipeline([ContiguousSpanAggregator, SequenceAttributionAggregator])\n",
    "\n",
    "# aggregrate\n",
    "actual_output = out.sequence_attributions[0].aggregate(aggregator=squeezesum, target_spans=(1, 3))\n",
    "forced_output = out.sequence_attributions[1].aggregate(aggregator=squeezesum, target_spans=(1, 3))\n",
    "\n",
    "# Take the diff of the scores of the two attribution and show it\n",
    "actual_output.show(aggregator=PairAggregator, paired_attr=forced_output)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compare actual output with another possible output"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Why our model generated its output sequence rather than another one that we consider to be more likely. Inseq uses a feature attribution algorithm to identify the most important features for predicting the original label and the contrastive label. The difference in feature importance between the original and contrastive examples is then used to attribute importance to the features for predicting the original label. By using custom attribution targets in this process, the feature importance can be attributed specifically to the features that are relevant for predicting the target labels of interest. This can provide more targeted and meaningful insights into the model's decision-making process."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from inseq.attr.step_functions import probability_fn\n",
    "\n",
    "# Simplified implementation of inseq.attr.step_functions.contrast_prob_diff_fn\n",
    "# Works only for encoder-decoder models!\n",
    "def example_prob_diff_fn(\n",
    "    # Default arguments in attribution_model.forward\n",
    "    attribution_model,\n",
    "    forward_output,\n",
    "    encoder_input_embeds,\n",
    "    encoder_attention_mask,\n",
    "    decoder_input_ids,\n",
    "    decoder_attention_mask,\n",
    "    target_ids,\n",
    "    # Extra arguments for our use case\n",
    "    contrast_ids,\n",
    "    contrast_attention_mask,\n",
    "    # We use kwargs to collect unused default arguments\n",
    "    **kwargs,\n",
    "):\n",
    "    \"\"\"Custom attribution function returning the difference between next step probability for\n",
    "    candidate generation vs. a contrastive alternative, answering the question \"Which features\n",
    "    were salient in deciding to pick the selected token rather than its contrastive alternative?\"\n",
    "\n",
    "    Extra args:\n",
    "        contrast_ids: Tensor containing the ids of the contrastive input to be compared to the\n",
    "            regular one.\n",
    "        contrast_attention_mask: Tensor containing the attention mask for the contrastive input\n",
    "    \"\"\"\n",
    "    # We truncate contrastive ids and their attention map to the current generation step\n",
    "    contrast_decoder_input_ids = contrast_ids[:, : decoder_input_ids.shape[1]].to(attribution_model.device)\n",
    "    contrast_decoder_attention_mask = contrast_attention_mask[:, : decoder_attention_mask.shape[1]].to(\n",
    "        attribution_model.device\n",
    "    )\n",
    "    # We select the next contrastive token as target\n",
    "    contrast_target_ids = contrast_ids[:, decoder_input_ids.shape[1]].to(attribution_model.device)\n",
    "    # Forward pass with the same model used for the main generation, but using contrastive inputs instead\n",
    "    contrast_output = attribution_model.model(\n",
    "        inputs_embeds=encoder_input_embeds,\n",
    "        attention_mask=encoder_attention_mask,\n",
    "        decoder_input_ids=contrast_decoder_input_ids,\n",
    "        decoder_attention_mask=contrast_decoder_attention_mask,\n",
    "    )\n",
    "    # Return the prob difference as target for attribution\n",
    "    model_probs = probability_fn(attribution_model, forward_output, target_ids)\n",
    "    contrast_probs = probability_fn(attribution_model, contrast_output, contrast_target_ids)\n",
    "    return model_probs - contrast_probs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Attributing with saliency...: 100%|██████████| 14/14 [00:21<00:00,  1.69s/it]\n",
      "/Users/barberademol/Desktop/ik-nlp-tutorials/venv/lib/python3.8/site-packages/inseq/data/attribution.py:275: UserWarning: The use of `x.T` on tensors of dimension other than 2 to reverse their shape is deprecated and it will throw an error in a future release. Consider `x.mT` to transpose batches of matrices or `x.permute(*torch.arange(x.ndim - 1, -1, -1))` to reverse the dimensions of a tensor. (Triggered internally at /Users/runner/work/pytorch/pytorch/pytorch/aten/src/ATen/native/TensorShape.cpp:3281.)\n",
      "  step_scores = self.step_scores[step_fn_id].T.unsqueeze(1)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<br/><b>0th instance:</b><br/>\n",
       "<html>\n",
       "<div id=\"usieyiccwpusnzjovqwp_viz_container\">\n",
       "    <div id=\"usieyiccwpusnzjovqwp_content\" style=\"padding:15px;border-style:solid;margin:5px;\">\n",
       "        <div id = \"usieyiccwpusnzjovqwp_saliency_plot_container\" class=\"usieyiccwpusnzjovqwp_viz_container\" style=\"display:block\">\n",
       "            \n",
       "<div id=\"qahftrxckafnafqofpva_saliency_plot\" class=\"qahftrxckafnafqofpva_viz_content\">\n",
       "    <div style=\"margin:5px;font-family:sans-serif;font-weight:bold;\">\n",
       "        <span style=\"font-size: 20px;\">Source Saliency Heatmap</span>\n",
       "        <br>\n",
       "        x: Generated tokens, y: Attributed tokens\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" cellpadding=\"5\" cellspacing=\"5\"\n",
       "    style=\"overflow-x:scroll;display:block;\">\n",
       "    <tr><th></th>\n",
       "<th>▁A</th><th>▁big</th><th>▁dog</th><th>▁and</th><th>▁</th><th>a</th><th>▁smaller</th><th>▁dog</th><th>▁are</th><th>▁two</th><th>▁dogs</th><th>.</th><th>&lt;/s&gt;</th></tr><tr><th>▁</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.12501485442661908)\">0.011</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.003</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.002</th></tr><tr><th>premise</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.006</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.002</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th></tr><tr><th>:</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.12501485442661908)\">0.011</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.002</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>▁A</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.15654585066349747)\">0.014</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.002</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>▁big</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.306318082788671)\">0.026</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.004</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>▁dog</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.2117250940780353)\">0.018</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.003</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>▁and</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.4087938205585263)\">0.035</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.005</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.001</th></tr><tr><th>▁</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14078035254505847)\">0.013</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.002</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>a</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.16442859972271742)\">0.015</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.002</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>▁smaller</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.32208358090711037)\">0.028</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.004</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>▁dog</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.219607843137255)\">0.019</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.004</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.002</th></tr><tr><th>▁fight</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.4245593186769657)\">0.037</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.005</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.005</th></tr><tr><th>▁over</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.13289760348583876)\">0.012</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.002</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.002</th></tr><tr><th>▁</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.008</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>a</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.006</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.001</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th></tr><tr><th>▁piece</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.007</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.002</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th></tr><tr><th>▁of</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1171321053673995)\">0.011</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.002</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>▁fabric</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14078035254505847)\">0.013</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.002</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>.</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.39302832244008706)\">0.034</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.005</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.002</th></tr><tr><th>▁hypothesis</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10136660724896006)\">0.009</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th></tr><tr><th>:</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.20384234501881549)\">0.018</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.003</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>▁Two</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 1.0)\">0.086</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.10924935630817992)\">0.01</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.002</th></tr><tr><th>▁dogs</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.4245593186769657)\">0.037</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.007</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.001</th></tr><tr><th>▁fighting</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.621628045157457)\">0.053</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.005</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.005</th></tr><tr><th>▁over</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.16442859972271742)\">0.014</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.002</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.002</th></tr><tr><th>▁</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.006</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th></tr><tr><th>a</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.007</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.001</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th></tr><tr><th>▁piece</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.008</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.002</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th></tr><tr><th>▁of</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.12501485442661908)\">0.011</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.002</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th></tr><tr><th>▁fabric</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.14078035254505847)\">0.012</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.002</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th></tr><tr><th>&lt;/s&gt;</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.006</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th></tr><tr style=\"outline: thin solid\"><th><b>example_prob_diff</b></th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th><b>0.878</b></th><th>0.013</th><th>0.177</th><th>0.061</th></table>\n",
       "</div>\n",
       "\n",
       "        </div>\n",
       "    </div>\n",
       "</div>\n",
       "</html>\n",
       "<br/><b>0th instance:</b><br/>\n",
       "<html>\n",
       "<div id=\"zqorvufaigfywirkxlgg_viz_container\">\n",
       "    <div id=\"zqorvufaigfywirkxlgg_content\" style=\"padding:15px;border-style:solid;margin:5px;\">\n",
       "        <div id = \"zqorvufaigfywirkxlgg_saliency_plot_container\" class=\"zqorvufaigfywirkxlgg_viz_container\" style=\"display:block\">\n",
       "            \n",
       "<div id=\"sbfhcgchqjjfgyqpesej_saliency_plot\" class=\"sbfhcgchqjjfgyqpesej_viz_content\">\n",
       "    <div style=\"margin:5px;font-family:sans-serif;font-weight:bold;\">\n",
       "        <span style=\"font-size: 20px;\">Target Saliency Heatmap</span>\n",
       "        <br>\n",
       "        x: Generated tokens, y: Attributed tokens\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" cellpadding=\"5\" cellspacing=\"5\"\n",
       "    style=\"overflow-x:scroll;display:block;\">\n",
       "    <tr><th></th>\n",
       "<th>▁A</th><th>▁big</th><th>▁dog</th><th>▁and</th><th>▁</th><th>a</th><th>▁smaller</th><th>▁dog</th><th>▁are</th><th>▁two</th><th>▁dogs</th><th>.</th><th>&lt;/s&gt;</th></tr><tr><th>▁A</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.2590215884333532)\">0.022</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06195286195286207)\">0.006</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.001</th></tr><tr><th>▁big</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.29055258467023165)\">0.025</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.06983561101208159)\">0.006</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>▁dog</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.18019409784115661)\">0.016</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.004</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th></tr><tr><th>▁and</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.4245593186769657)\">0.037</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.08560110913052081)\">0.007</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.002</th></tr><tr><th>▁</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.23537334125569415)\">0.021</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.03830461477520289)\">0.004</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>a</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.219607843137255)\">0.019</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.04618736383442265)\">0.005</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>▁smaller</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.3536145771439889)\">0.03</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.07771836007130124)\">0.007</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.00677361853832443)\">0.001</th></tr><tr><th>▁dog</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.3772628243216479)\">0.033</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.005</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th></tr><tr><th>▁are</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.9842345018815607)\">0.084</th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.0</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.1880768469003763)\">0.017</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.02253911665676371)\">0.003</th></tr><tr><th>▁two</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(230.2941176470614, 26.505882352939775, 102.59215686274348, 0.0)\">0.001</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.12501485442661908)\">0.011</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.030421865715983164)\">0.003</th></tr><tr><th>▁dogs</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.13289760348583876)\">0.012</th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.014656367597544035)\">0.002</th></tr><tr><th>.</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(255.0, 13.0, 87.0, 0.05407011289364243)\">0.005</th></tr><tr><th>&lt;/s&gt;</th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th><th style=\"background:rgba(0.0, 0.0, 0.0, 0.0)\"></th></tr><tr style=\"outline: thin solid\"><th><b>example_prob_diff</b></th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th>0.0</th><th><b>0.878</b></th><th>0.013</th><th>0.177</th><th>0.061</th></table>\n",
       "</div>\n",
       "\n",
       "        </div>\n",
       "    </div>\n",
       "</div>\n",
       "</html>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Register the function defined above\n",
    "# Since outputs are still probabilities, contiguous tokens can still be aggregated using product\n",
    "inseq.register_step_function(\n",
    "    fn=example_prob_diff_fn,\n",
    "    identifier=\"example_prob_diff\",\n",
    "    aggregate_map={\"span_aggregate\": lambda x: x.prod(dim=1, keepdim=True)},\n",
    ")\n",
    "\n",
    "# Pre-compute ids and attention map for the contrastive target\n",
    "contrast = inseq_model_explanation.encode(\"A big dog and a smaller dog are three dogs.\", as_targets=True)\n",
    "\n",
    "# Perform the contrastive attribution:\n",
    "# contrast_ids & contrast_attention_mask are kwargs defined in the function definition\n",
    "out = inseq_model_explanation.attribute(\n",
    "    \"premise: A big dog and a smaller dog fight over a piece of fabric. hypothesis: Two dogs fighting over a piece of fabric\",\n",
    "    \"A big dog and a smaller dog are two dogs.\",\n",
    "    attributed_fn=\"example_prob_diff\",\n",
    "    contrast_ids=contrast.input_ids,\n",
    "    contrast_attention_mask=contrast.attention_mask,\n",
    "    attribute_target=True,\n",
    "    # We also visualize the step score\n",
    "    step_scores=[\"example_prob_diff\"]\n",
    ")\n",
    "\n",
    "# Weight attribution scores by the difference in logits\n",
    "out.weight_attributions(\"example_prob_diff\")\n",
    "out.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp",
   "language": "python",
   "name": "nlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}