reproduction_notebooks/human_evaluations_summary.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ast import literal_eval\n",
    "from collections import defaultdict, Counter\n",
    "from itertools import combinations, chain\n",
    "import json\n",
    "import os\n",
    "from types import SimpleNamespace\n",
    "import typing\n",
    "\n",
    "import krippendorff\n",
    "from IPython.display import display, HTML, Markdown\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.patches as mpatches\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy import stats\n",
    "from tabulate import tabulate\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ExtendedSimpleNamespace(SimpleNamespace):\n",
    "    def __init__(self, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "\n",
    "    def __contains__(self, key):\n",
    "        return hasattr(self, key)\n",
    "\n",
    "    def __getitem__(self, key):\n",
    "        return getattr(self, key)\n",
    "\n",
    "    def get(self, key, default=None):\n",
    "        return getattr(self, key, default)\n",
    "\n",
    "    def keys(self):\n",
    "        return [k for k in vars(self).keys() if not k.startswith(\"_\")]\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "TOP_30_UNMATCHED_FULL_GAME_IDS = [\n",
    "    '(1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1)-unmatched',\n",
    "    '(1, 1, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0)-unmatched',\n",
    "    '(1, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0)-unmatched',\n",
    "    '(1, 1, 4, 0, 0, 1, 1, 1, 0, 1, 0, 0)-unmatched',\n",
    "    '(1, 0, 2, 0, 1, 0, 0, 0, 0, 1, 0, 0)-unmatched',\n",
    "    '(1, 1, 2, 0, 0, 1, 0, 0, 0, 0, 0, 1)-unmatched',\n",
    "    '(1, 0, 4, 0, 0, 0, 0, 0, 3, 0, 0, 0)-unmatched',\n",
    "    '(1, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 1)-unmatched',\n",
    "    '(1, 1, 2, 0, 0, 0, 1, 0, 1, 0, 0, 0)-unmatched',\n",
    "    '(1, 0, 4, 0, 1, 1, 0, 1, 0, 1, 0, 0)-unmatched',\n",
    "    '(1, 0, 2, 1, 0, 0, 1, 0, 0, 0, 0, 0)-unmatched',\n",
    "    '(1, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0)-unmatched',\n",
    "    '(1, 1, 3, 0, 0, 1, 0, 0, 1, 0, 0, 0)-unmatched',\n",
    "    '(1, 1, 4, 0, 2, 0, 0, 0, 1, 0, 0, 0)-unmatched',\n",
    "    '(1, 1, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0)-unmatched',\n",
    "    '(1, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0)-unmatched',\n",
    "    '(1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0)-unmatched',\n",
    "    '(1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0)-unmatched',\n",
    "    '(1, 1, 2, 0, 1, 1, 0, 0, 0, 0, 0, 0)-unmatched',\n",
    "    '(1, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0)-unmatched',\n",
    "    '(1, 1, 3, 1, 0, 0, 1, 0, 0, 0, 1, 0)-unmatched',\n",
    "    '(1, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0)-unmatched',\n",
    "    '(1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0)-unmatched',\n",
    "    '(1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0)-unmatched',\n",
    "    '(1, 1, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0)-unmatched',\n",
    "    '(1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0)-unmatched',\n",
    "    '(1, 1, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0)-unmatched',\n",
    "    '(1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0)-unmatched',\n",
    "    '(1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0)-unmatched',\n",
    "    '(1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0)-unmatched',\n",
    " ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded current participant data with shape (1292, 35) from ./human_evals_data/real-complete_only-main-pilot_02-02_data.json\n",
      "Dataframe shape: (1292, 35)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>participant_id</th>\n",
       "      <th>game_id</th>\n",
       "      <th>real</th>\n",
       "      <th>matched</th>\n",
       "      <th>confident</th>\n",
       "      <th>fun_play</th>\n",
       "      <th>fun_watch</th>\n",
       "      <th>capability</th>\n",
       "      <th>goldilocks</th>\n",
       "      <th>creativity</th>\n",
       "      <th>...</th>\n",
       "      <th>fully_normalized_fun_play</th>\n",
       "      <th>fully_normalized_fun_watch</th>\n",
       "      <th>fully_normalized_capability</th>\n",
       "      <th>fully_normalized_goldilocks</th>\n",
       "      <th>fully_normalized_creativity</th>\n",
       "      <th>fully_normalized_human_likeness</th>\n",
       "      <th>fitness</th>\n",
       "      <th>archive_distance</th>\n",
       "      <th>normalized_fitness</th>\n",
       "      <th>normalized_archive_distance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00c9bf44-28f3-469a-8a71-ea972af61bab-p102</td>\n",
       "      <td>(1, 1, 4, 0, 2, 0, 0, 0, 1, 0, 0, 0)</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0.636636</td>\n",
       "      <td>0.694003</td>\n",
       "      <td>1.696015</td>\n",
       "      <td>0.324430</td>\n",
       "      <td>-0.213210</td>\n",
       "      <td>-0.387244</td>\n",
       "      <td>36.531780</td>\n",
       "      <td>4</td>\n",
       "      <td>-0.094034</td>\n",
       "      <td>1.934673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00c9bf44-28f3-469a-8a71-ea972af61bab-p102</td>\n",
       "      <td>(1, 0, 2, 0, 1, 0, 0, 0, 0, 1, 0, 0)</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0.636636</td>\n",
       "      <td>0.694003</td>\n",
       "      <td>0.878293</td>\n",
       "      <td>0.324430</td>\n",
       "      <td>0.737669</td>\n",
       "      <td>-0.387244</td>\n",
       "      <td>36.065792</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.776234</td>\n",
       "      <td>0.564280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00c9bf44-28f3-469a-8a71-ea972af61bab-p102</td>\n",
       "      <td>(1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0)</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.250393</td>\n",
       "      <td>-0.179182</td>\n",
       "      <td>1.696015</td>\n",
       "      <td>1.277683</td>\n",
       "      <td>0.737669</td>\n",
       "      <td>-0.387244</td>\n",
       "      <td>36.065643</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.776452</td>\n",
       "      <td>-0.806114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00c9bf44-28f3-469a-8a71-ea972af61bab-p102</td>\n",
       "      <td>(1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0)</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>1.523665</td>\n",
       "      <td>1.567189</td>\n",
       "      <td>0.060572</td>\n",
       "      <td>-0.628822</td>\n",
       "      <td>-0.213210</td>\n",
       "      <td>-0.387244</td>\n",
       "      <td>37.139469</td>\n",
       "      <td>0</td>\n",
       "      <td>0.795615</td>\n",
       "      <td>-0.806114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00c9bf44-28f3-469a-8a71-ea972af61bab-p102</td>\n",
       "      <td>(1, 1, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0)</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.137421</td>\n",
       "      <td>-1.052368</td>\n",
       "      <td>-1.574871</td>\n",
       "      <td>2.230935</td>\n",
       "      <td>-1.164089</td>\n",
       "      <td>0.437610</td>\n",
       "      <td>35.682575</td>\n",
       "      <td>3</td>\n",
       "      <td>-1.337259</td>\n",
       "      <td>1.249477</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              participant_id  \\\n",
       "0  00c9bf44-28f3-469a-8a71-ea972af61bab-p102   \n",
       "1  00c9bf44-28f3-469a-8a71-ea972af61bab-p102   \n",
       "2  00c9bf44-28f3-469a-8a71-ea972af61bab-p102   \n",
       "3  00c9bf44-28f3-469a-8a71-ea972af61bab-p102   \n",
       "4  00c9bf44-28f3-469a-8a71-ea972af61bab-p102   \n",
       "\n",
       "                                game_id   real  matched  confident  fun_play  \\\n",
       "0  (1, 1, 4, 0, 2, 0, 0, 0, 1, 0, 0, 0)  False    False          4         3   \n",
       "1  (1, 0, 2, 0, 1, 0, 0, 0, 0, 1, 0, 0)  False    False          4         3   \n",
       "2  (1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0)   True     True          4         2   \n",
       "3  (1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0)  False     True          5         4   \n",
       "4  (1, 1, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0)  False    False          1         1   \n",
       "\n",
       "   fun_watch  capability  goldilocks  creativity  ...  \\\n",
       "0          3           5           3           2  ...   \n",
       "1          3           4           3           3  ...   \n",
       "2          2           5           4           3  ...   \n",
       "3          4           3           2           2  ...   \n",
       "4          1           1           5           1  ...   \n",
       "\n",
       "   fully_normalized_fun_play fully_normalized_fun_watch  \\\n",
       "0                   0.636636                   0.694003   \n",
       "1                   0.636636                   0.694003   \n",
       "2                  -0.250393                  -0.179182   \n",
       "3                   1.523665                   1.567189   \n",
       "4                  -1.137421                  -1.052368   \n",
       "\n",
       "  fully_normalized_capability fully_normalized_goldilocks  \\\n",
       "0                    1.696015                    0.324430   \n",
       "1                    0.878293                    0.324430   \n",
       "2                    1.696015                    1.277683   \n",
       "3                    0.060572                   -0.628822   \n",
       "4                   -1.574871                    2.230935   \n",
       "\n",
       "  fully_normalized_creativity  fully_normalized_human_likeness    fitness  \\\n",
       "0                   -0.213210                        -0.387244  36.531780   \n",
       "1                    0.737669                        -0.387244  36.065792   \n",
       "2                    0.737669                        -0.387244  36.065643   \n",
       "3                   -0.213210                        -0.387244  37.139469   \n",
       "4                   -1.164089                         0.437610  35.682575   \n",
       "\n",
       "   archive_distance  normalized_fitness  normalized_archive_distance  \n",
       "0                 4           -0.094034                     1.934673  \n",
       "1                 2           -0.776234                     0.564280  \n",
       "2                 0           -0.776452                    -0.806114  \n",
       "3                 0            0.795615                    -0.806114  \n",
       "4                 3           -1.337259                     1.249477  \n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CURRENT_DATA_PATH = './human_evals_data/real-complete_only-main-pilot_02-02_data.json'\n",
    "FULL_GAME_ID_TO_FITNESS_PATH = './human_evals_data/full_game_id_to_fitness.csv'\n",
    "FULL_GAME_ID_TO_NODE_COUNT_PATH = './human_evals_data/node_counts.csv'\n",
    "CURRENT_DF_PATH = './human_evals_data/current_participants_df.csv'\n",
    "\n",
    "GAME_KEYS = [\n",
    "    'id', 'real', 'matched', \n",
    "    'confident', 'fun_play', 'fun_watch', \n",
    "    'capability', 'goldilocks', 'creativity',\n",
    "    'human-likeness', 'explain', 'overall'\n",
    "]\n",
    "\n",
    "RENAMED_GAME_KEYS = {\n",
    "    'id': 'game_id',\n",
    "    'human-likeness': 'human_likeness'\n",
    "}\n",
    "\n",
    "NUMBER_SCORE_ATTRIBUTES = [\n",
    "    'confident', 'fun_play', \n",
    "    'fun_watch',  'capability', \n",
    "    'goldilocks', 'creativity',\n",
    "    'human_likeness',\n",
    "]\n",
    "\n",
    "NUMBER_SCORE_AXIS_NAMES = {\n",
    "    'confident': 'How confident are you that you understand the game?',\n",
    "    'fun_play': 'How fun would it be to play the game yourself?',\n",
    "    'fun_watch': 'How fun would it be to watch someone else play the game?',\n",
    "    'capability': 'How helpful would it be for learning to interact',\n",
    "    'goldilocks': 'Too easy, appropriately difficult, or too hard for you?',\n",
    "    'creativity': 'How creatively designed is the game?',\n",
    "    'human_likeness': 'How human-like do you think this game is?',\n",
    "}\n",
    "\n",
    "\n",
    "def load_data(data_path: str) -> typing.Tuple[typing.List[ExtendedSimpleNamespace], typing.List[ExtendedSimpleNamespace]]:\n",
    "    with open(data_path, 'r') as f:\n",
    "        raw_data = json.load(f, object_hook=lambda d: ExtendedSimpleNamespace(**d))\n",
    "\n",
    "    prolific_participants_data = []\n",
    "    non_prolific_participants_data = []\n",
    "\n",
    "    for participant_data in raw_data:\n",
    "        if participant_data.data.recruitment_service == 'prolific':\n",
    "            prolific_participants_data.append(participant_data)\n",
    "        else:\n",
    "            non_prolific_participants_data.append(participant_data)\n",
    "\n",
    "    return prolific_participants_data, non_prolific_participants_data\n",
    "\n",
    "\n",
    "def real_matched_to_str(series: pd.Series) -> str:\n",
    "    if series.real:\n",
    "        return 'real'\n",
    "\n",
    "    if series.matched:\n",
    "        return 'matched'\n",
    "\n",
    "    return 'unmatched'\n",
    "\n",
    "\n",
    "def full_game_id_to_str(series: pd.Series) -> str:\n",
    "    return f\"{series.game_id}-{series.game_type}\"\n",
    "\n",
    "\n",
    "def loaded_data_to_df(loaded_data: typing.List[ExtendedSimpleNamespace]):\n",
    "    rows = []\n",
    "\n",
    "    for participant in loaded_data:\n",
    "        participant_id = participant.id\n",
    "\n",
    "        for game_result in participant.data.single_game_results:\n",
    "            rows.append({\n",
    "                'participant_id': participant_id,\n",
    "                **{RENAMED_GAME_KEYS.get(key, key): game_result[key] for key in GAME_KEYS}\n",
    "            })\n",
    "    \n",
    "    df = pd.DataFrame(rows)\n",
    "    df = df.assign(game_type=df.apply(real_matched_to_str, axis=1))\n",
    "    df = df.assign(full_game_id=df.apply(full_game_id_to_str, axis=1), **{f'normalized_{attribute}': 0.0 for attribute in NUMBER_SCORE_ATTRIBUTES},\n",
    "                   num_preferences=[literal_eval(x)[2] for x in df.game_id])\n",
    "    \n",
    "    full_game_id_to_node_count_df = pd.read_csv(FULL_GAME_ID_TO_NODE_COUNT_PATH)\n",
    "    df = pd.merge(df, full_game_id_to_node_count_df, on='full_game_id', how='left', suffixes=('', '_y'))\n",
    "    df = df.drop(columns=[c for c in df.columns if c.endswith('_y')])\n",
    "\n",
    "    participant_ids = df.participant_id.unique()\n",
    "\n",
    "    for participant_id in participant_ids:\n",
    "        for attribute in NUMBER_SCORE_ATTRIBUTES:\n",
    "            participant_attr_mean = df[df.participant_id == participant_id][attribute].mean()\n",
    "            participant_attr_std = df[df.participant_id == participant_id][attribute].std()\n",
    "\n",
    "            if participant_attr_std == 0:\n",
    "                df.loc[df.participant_id == participant_id, f'normalized_{attribute}'] = 0\n",
    "            else:\n",
    "                df.loc[df.participant_id == participant_id, f'normalized_{attribute}'] = (df.loc[df.participant_id == participant_id, attribute] - participant_attr_mean) / participant_attr_std\n",
    "\n",
    "\n",
    "    df = df.assign(**{f'fully_normalized_{attribute}': (df[attribute] - df[attribute].mean()) / df[attribute].std() for attribute in NUMBER_SCORE_ATTRIBUTES})\n",
    "\n",
    "    top_30_df = df[df.full_game_id.isin(TOP_30_UNMATCHED_FULL_GAME_IDS)].copy()\n",
    "    top_30_df.game_type = 'unmatched_top_30'\n",
    "    top_30_df = top_30_df.assign(full_game_id=top_30_df.apply(full_game_id_to_str, axis=1))\n",
    "\n",
    "    df = pd.concat([df, top_30_df])\n",
    "    df = df.assign(\n",
    "        participant_id=pd.Categorical(df.participant_id, categories=sorted(df.participant_id.unique()), ordered=True),\n",
    "        game_type=pd.Categorical(df.game_type, categories=['real', 'matched', 'unmatched', 'unmatched_top_30'], ordered=True),\n",
    "    )\n",
    "    \n",
    "    full_game_id_to_fitness_df = pd.read_csv(FULL_GAME_ID_TO_FITNESS_PATH)\n",
    "    df = pd.merge(df, full_game_id_to_fitness_df, on='full_game_id', how='left')\n",
    "    df = df.assign(\n",
    "        normalized_fitness=(df.fitness - df.fitness.mean()) / (df.fitness.std()),\n",
    "        normalized_archive_distance=(df.archive_distance - df.archive_distance.mean()) / (df.archive_distance.std()),\n",
    "        full_game_id=pd.Categorical(df.full_game_id, categories=sorted(df.full_game_id.unique()), ordered=True),\n",
    "    )\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def load_data_to_df(data_path: str, prolific_only: bool = True):\n",
    "    prolific_participants_data, non_prolific_participants_data = load_data(data_path)\n",
    "    if not prolific_only:\n",
    "        prolific_participants_data.extend(non_prolific_participants_data)\n",
    "\n",
    "    prolific_participants_data = list(filter(lambda p: p.data.get('quiz_passed', True), prolific_participants_data))\n",
    "\n",
    "    return prolific_participants_data, loaded_data_to_df(prolific_participants_data)\n",
    "\n",
    "\n",
    "current_participants_raw_data, current_participants_df = load_data_to_df(CURRENT_DATA_PATH)\n",
    "print(f'Loaded current participant data with shape {current_participants_df.shape} from {CURRENT_DATA_PATH}')\n",
    "\n",
    "\n",
    "print(f'Dataframe shape: {current_participants_df.shape}')\n",
    "current_participants_df.to_csv(CURRENT_DF_PATH, index=False)\n",
    "current_participants_df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_linebreaks(string: str, n_lines: int = 2, min_break_length: int = 36):\n",
    "    if len(string) <= min_break_length:\n",
    "        return string\n",
    "\n",
    "    prev_break_index = 0\n",
    "    line_parts = []\n",
    "\n",
    "    for break_index in range(1, n_lines):\n",
    "        break_index = string.find(' ', int(break_index * len(string) / n_lines))\n",
    "        if break_index == -1:\n",
    "            break\n",
    "\n",
    "        line_parts.append(string[prev_break_index:break_index])\n",
    "        prev_break_index = break_index + 1\n",
    "\n",
    "    line_parts.append(string[prev_break_index:])\n",
    "    return '\\n'.join(line_parts)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Human evaluation result summary\n",
    "\n",
    "Table SI-2, from which Table ED-1 is derived."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lllllll}\n",
      "\\hline\n",
      " Attribute                                                & Real              & Matched           & Unmatched         & U-stat, p-value      & U-stat, p-value                          & U-stat, p-value                          \\\\\n",
      "\\hline\n",
      " How confident are you that you understand the game?      & $3.943 \\pm 0.068$ & $3.923 \\pm 0.070$ & $3.331 \\pm 0.075$ & 45088.0, $P = 0.906$ & 55921.5, $P = \\num{ 1.718e-09 }^{ *** }$ & 55846.0, $P = \\num{ 3.733e-09 }^{ *** }$ \\\\\n",
      " How fun would it be to play the game yourself?           & $2.522 \\pm 0.066$ & $2.430 \\pm 0.064$ & $2.068 \\pm 0.062$ & 46752.5, $P = 0.352$ & 54040.5, $P = \\num{ 3.235e-07 }^{ *** }$ & 52539.5, $P = \\num{ 1.826e-05 }^{ *** }$ \\\\\n",
      " How fun would it be to watch someone else play the game? & $2.385 \\pm 0.068$ & $2.313 \\pm 0.066$ & $2.024 \\pm 0.064$ & 46169.0, $P = 0.519$ & 51636.5, $P = \\num{ 8.793e-05 }^{ *** }$ & 50515.0, $P = \\num{ 1.027e-03 }^{ ** }$  \\\\\n",
      " How helpful would it be for learning to interact         & $2.997 \\pm 0.068$ & $2.987 \\pm 0.066$ & $2.840 \\pm 0.073$ & 44802.0, $P = 0.982$ & 47372.5, $P = 0.078$                     & 47559.0, $P = 0.075$                     \\\\\n",
      " Too easy, appropriately difficult, or too hard for you?  & $2.582 \\pm 0.055$ & $2.660 \\pm 0.056$ & $2.676 \\pm 0.066$ & 42921.5, $P = 0.326$ & 42218.5, $P = 0.419$                     & 44081.0, $P = 0.947$                     \\\\\n",
      " How creatively designed is the game?                     & $2.318 \\pm 0.061$ & $2.213 \\pm 0.056$ & $2.143 \\pm 0.064$ & 47036.0, $P = 0.282$ & 48286.0, $P = 0.025^{ * }$               & 46615.0, $P = 0.182$                     \\\\\n",
      " How human-like do you think this game is?                & $2.813 \\pm 0.066$ & $2.670 \\pm 0.070$ & $2.119 \\pm 0.067$ & 47698.0, $P = 0.167$ & 58679.0, $P = \\num{ 1.702e-13 }^{ *** }$ & 55434.5, $P = \\num{ 1.333e-08 }^{ *** }$ \\\\\n",
      "\\hline\n",
      "\\end{tabular}\n"
     ]
    }
   ],
   "source": [
    "rows = []\n",
    "UNMATCHED_TOP_30 = 'unmatched_top_30'\n",
    "REAL = 'real'\n",
    "MATCHED = 'matched'\n",
    "AST = '*'\n",
    "USE_NORMALIZED = False\n",
    "TRUNCATE_P_VALUES = False\n",
    "RESULT_COLUMN_TEXT_SIZE = 'footnotesize'\n",
    "\n",
    "# category_order_for_t_tests = [UNMATCHED_TOP_30, REAL, MATCHED]\n",
    "category_order_for_means = [REAL, MATCHED, UNMATCHED_TOP_30]\n",
    "\n",
    "for attribute in NUMBER_SCORE_ATTRIBUTES:\n",
    "    if USE_NORMALIZED:\n",
    "        attribute = f'normalized_{attribute}'\n",
    "\n",
    "    attribute_name = NUMBER_SCORE_AXIS_NAMES[attribute]\n",
    "    # attribute_name = attribute.replace('_', '\\\\_').replace('alized', '')\n",
    "\n",
    "    row = [attribute_name]\n",
    "    \n",
    "    for category in category_order_for_means:\n",
    "        curr = current_participants_df[current_participants_df.game_type == category][attribute]\n",
    "        row.append(f'${curr.mean():.3f} \\\\pm {curr.sem():.3f}$')\n",
    "\n",
    "    for first, second in combinations(category_order_for_means, 2):\n",
    "        first_data = current_participants_df[current_participants_df.game_type == first][attribute]\n",
    "        second_data = current_participants_df[current_participants_df.game_type == second][attribute]\n",
    "        # result = stats.ttest_ind(first_data, second_data)\n",
    "        result = stats.mannwhitneyu(first_data, second_data)\n",
    "        statistic, p_value = result.statistic, result.pvalue\n",
    "        stars = AST * int(p_value < 0.05) + AST * int(p_value < 0.01) + AST * int(p_value < 0.001)\n",
    "        if stars:\n",
    "            # stars = f'\\\\textsuperscript{{ \\\\textbf {{ {stars} }} }}'\n",
    "            stars = f'^{{ {stars} }}'\n",
    "        # else:\n",
    "        #     stars = '$'\n",
    "\n",
    "        if TRUNCATE_P_VALUES and p_value < 1e-5:\n",
    "            p_value = r'$P < \\num{1e-5}' + stars + '$'\n",
    "\n",
    "        elif TRUNCATE_P_VALUES and p_value < 1e-3:\n",
    "            p_value = r'$P < \\num{1e-3}' + stars + '$'\n",
    "\n",
    "        else:\n",
    "            use_scientific = p_value < 1e-2\n",
    "            if use_scientific:\n",
    "                p_value = f'$P = \\\\num{{ {p_value:.3e} }}{stars}$'\n",
    "            else:\n",
    "                p_value = f'$P = {p_value:.3f}{stars}$'\n",
    "\n",
    "        # row.extend([f'{{ \\\\{RESULT_COLUMN_TEXT_SIZE} {statistic:.1f} }}', f'{{ \\\\{RESULT_COLUMN_TEXT_SIZE} {p_value} }}'])\n",
    "        # row.extend([f'{{ \\\\{RESULT_COLUMN_TEXT_SIZE} {statistic:.1f} }}, {{ \\\\{RESULT_COLUMN_TEXT_SIZE} {p_value} }}'])\n",
    "        row.extend([f'{statistic:.1f}, {p_value}'])\n",
    "\n",
    "    rows.append(row)\n",
    "\n",
    "\n",
    "HEADERS = [\n",
    "    'Attribute', 'Real', 'Matched', 'Unmatched',\n",
    "    'U-stat, p-value', 'U-stat, p-value', 'U-stat, p-value'\n",
    "]\n",
    "\n",
    "tabulated = tabulate(rows, headers=HEADERS, floatfmt='.3f', tablefmt='latex_raw')\n",
    "print(tabulated)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inter-rater agreement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confident: α = 0.233\n",
      "fun_play: α = 0.117\n",
      "fun_watch: α = 0.077\n",
      "capability: α = 0.023\n",
      "goldilocks: α = 0.211\n",
      "creativity: α = 0.129\n",
      "human_likeness: α = 0.151\n"
     ]
    }
   ],
   "source": [
    "categories = [REAL, MATCHED, UNMATCHED_TOP_30]\n",
    "tab = '    '\n",
    "\n",
    "df = current_participants_df[current_participants_df.game_type.isin(categories)]\n",
    "\n",
    "for attribute in NUMBER_SCORE_ATTRIBUTES:\n",
    "    pivot_table = pd.pivot_table(df, values=attribute, index='participant_id', columns='full_game_id')\n",
    "    alpha = krippendorff.alpha(pivot_table.values, level_of_measurement='ordinal')\n",
    "    print(f'{attribute}: \\u03b1 = {alpha:.3f}') \n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "game-gen",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}