pickle_process.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## process pickle result for statistics info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import re\n",
    "import numpy as np\n",
    "# load pickle\n",
    "class my_pickle:\n",
    "    def __init__(self, file_path) -> None:\n",
    "        with open(file_path, 'rb') as f:\n",
    "            result = pickle.load(f)\n",
    "        if isinstance(result, list):\n",
    "            version = \"old\"\n",
    "        elif isinstance(result, dict):\n",
    "            version = \"new\"\n",
    "        else:\n",
    "            raise ValueError(\"not dict or list type\")\n",
    "        if version == \"old\":\n",
    "            [self.correct_num, self.error_num, self.answers_list, self.contents_list] = result\n",
    "        else:\n",
    "            self.correct_num = result[\"correct_num\"]\n",
    "            self.error_num = result[\"error_num\"]\n",
    "            self.answers_list = result[\"answers_list\"]\n",
    "            self.contents_list = result[\"contents_list\"]\n",
    "        file_name = os.path.basename(file_path)\n",
    "    \n",
    "        \n",
    "    def avg_acc(self, case_num, reason_num):\n",
    "        correct_num = 0\n",
    "        error_num = 0\n",
    "        for case_id in range(case_num):\n",
    "            case_answers = self.answers_list[case_id]\n",
    "            for answer_id in range(reason_num):\n",
    "                answer = case_answers[answer_id]\n",
    "                if isinstance(answer, list):\n",
    "                    if answer[1] == 1:\n",
    "                        correct_num += 1  \n",
    "                    else: \n",
    "                        error_num += 1\n",
    "        print(\"acc:{}, correct num: {}, err num: {}\".format(correct_num / (case_num * reason_num), correct_num, error_num))\n",
    "        \n",
    "        return correct_num / (case_num * reason_num), correct_num, error_num\n",
    "    \n",
    "    def std(self, case_num, reason_num):\n",
    "        correct_num = [0 for _ in range(reason_num)]\n",
    "        for case_id in range(case_num):\n",
    "            case_answers = self.answers_list[case_id]\n",
    "            case_answers = self.answers_list[case_id]\n",
    "            for answer_id in range(reason_num):\n",
    "                answer = case_answers[answer_id]\n",
    "                if isinstance(answer, list):\n",
    "                    if answer[1] == 1:\n",
    "                        correct_num[answer_id] += 1  \n",
    "        accs = [correct/case_num * 100 for correct in correct_num]\n",
    "        print(accs)\n",
    "        std = np.std(accs, ddof=1)\n",
    "        mean = np.mean(accs)\n",
    "        print(\"mean:{}, std:{}\".format(mean, std))\n",
    "                    \n",
    "                    \n",
    "    def vote(self, case_num, reason_num):\n",
    "        from collections import Counter\n",
    "        valid_count = 0\n",
    "        vote_right_count = 0\n",
    "        for case_id in range(case_num):\n",
    "            answers = self.answers_list[case_id][:reason_num]\n",
    "            answers = [answer for answer in answers if isinstance(answer, list)]  # clean answers\n",
    "            if len(answers) == 0:\n",
    "                continue\n",
    "            else:\n",
    "                valid_count += 1\n",
    "            second_elements_are_1 = [sublist[1] == 1 for sublist in answers]\n",
    "            any_second_element_is_1 = any(second_elements_are_1)\n",
    "            if not any_second_element_is_1:\n",
    "                continue\n",
    "            true_answer = next((sublist[0] for sublist in answers if sublist[1] == 1), None)\n",
    "            counter = Counter(sublist[0] for sublist in answers)\n",
    "            guess_value, _ = counter.most_common(1)[0]\n",
    "            if guess_value == true_answer:\n",
    "                vote_right_count += 1\n",
    "\n",
    "        print(\"vote_correct_num:{}, vaild_num:{}, vote_correct_num:{}\".format(vote_right_count, valid_count, vote_right_count / valid_count))        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "class my_log:\n",
    "    def __init__(self, file_path) -> None:\n",
    "        pattern = r'^index \\d+/299, correct_num'\n",
    "        self.error_num = 0\n",
    "        self.correct_num = 0\n",
    "        self.answers_list = []\n",
    "        self.answers_content = []\n",
    "        with open(file_path, 'r') as f:\n",
    "            this_answer = []\n",
    "            this_content = []\n",
    "            content = \"\"\n",
    "            plan_flag = 0\n",
    "            for line in f:\n",
    "                if line.startswith(\"right\"):\n",
    "                    this_answer.append(1)\n",
    "                    self.error_num += 1\n",
    "                if line.startswith(\"wrong\"):\n",
    "                    this_answer.append(0)\n",
    "                    self.correct_num += 1\n",
    "                if line.startswith(\"Plan end.\"):\n",
    "                    plan_flag = 0\n",
    "                    this_content.append(content)\n",
    "                    content = \"\"\n",
    "                if plan_flag:\n",
    "                    content += line\n",
    "                if line.startswith(\"Plan:\"):\n",
    "                    plan_flag = 1\n",
    "                if bool(re.match(pattern, line)):\n",
    "                    \n",
    "                    self.answers_list.append(this_answer)\n",
    "                    this_answer = []\n",
    "                    self.answers_content.append(this_content)\n",
    "                    this_content = []\n",
    "                \n",
    "    def acc(self, case_num):\n",
    "        correct_num = 0 \n",
    "        error_num = 0\n",
    "        for i in range(case_num):\n",
    "            answers = self.answers_list[i]\n",
    "            for answer in answers:\n",
    "                if answer == 1:\n",
    "                    correct_num+=1\n",
    "                else:\n",
    "                    error_num+=1\n",
    "        \n",
    "        print(\"correct_num {}, error_num {}, acc {}\".format(correct_num, error_num, correct_num/ (correct_num+error_num)))\n",
    "    \n",
    "    def vote(self, case_num):\n",
    "        correct_case_num = 0 \n",
    "        for i in range(case_num):\n",
    "            correct_num = 0\n",
    "            answers = self.answers_list[i]\n",
    "            for answer in answers:\n",
    "                if answer == 1:\n",
    "                    correct_num+=1\n",
    "            if correct_num >= 3:\n",
    "                correct_case_num += 1\n",
    "        print(\"correct_num {}, error_num {}, acc {}\".format(correct_case_num, case_num - correct_case_num, correct_case_num/case_num))\n",
    "        return \n",
    "        \n",
    "        \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = \"\"\n",
    "pck = my_pickle(file_path)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gptapi",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}