main_cross_subject_LOSO.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2cad76e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [-1, 8, 3, 1000]             512\n",
      "       BatchNorm2d-2           [-1, 8, 3, 1000]              16\n",
      "            Conv2d-3          [-1, 16, 1, 1000]              48\n",
      "       BatchNorm2d-4          [-1, 16, 1, 1000]              32\n",
      "               ELU-5          [-1, 16, 1, 1000]               0\n",
      "         AvgPool2d-6           [-1, 16, 1, 125]               0\n",
      "           Dropout-7           [-1, 16, 1, 125]               0\n",
      "            Conv2d-8           [-1, 16, 1, 125]           4,096\n",
      "       BatchNorm2d-9           [-1, 16, 1, 125]              32\n",
      "              ELU-10           [-1, 16, 1, 125]               0\n",
      "        AvgPool2d-11            [-1, 16, 1, 15]               0\n",
      "          Dropout-12            [-1, 16, 1, 15]               0\n",
      "        Rearrange-13               [-1, 15, 16]               0\n",
      "PatchEmbeddingCNN-14               [-1, 15, 16]               0\n",
      "          Dropout-15               [-1, 15, 16]               0\n",
      "PositioinalEncoding-16               [-1, 15, 16]               0\n",
      "           Linear-17               [-1, 15, 16]             272\n",
      "           Linear-18               [-1, 15, 16]             272\n",
      "           Linear-19               [-1, 15, 16]             272\n",
      "          Dropout-20            [-1, 2, 15, 15]               0\n",
      "           Linear-21               [-1, 15, 16]             272\n",
      "MultiHeadAttention-22               [-1, 15, 16]               0\n",
      "          Dropout-23               [-1, 15, 16]               0\n",
      "        LayerNorm-24               [-1, 15, 16]              32\n",
      "      ResidualAdd-25               [-1, 15, 16]               0\n",
      "           Linear-26               [-1, 15, 64]           1,088\n",
      "             GELU-27               [-1, 15, 64]               0\n",
      "          Dropout-28               [-1, 15, 64]               0\n",
      "           Linear-29               [-1, 15, 16]           1,040\n",
      "          Dropout-30               [-1, 15, 16]               0\n",
      "        LayerNorm-31               [-1, 15, 16]              32\n",
      "      ResidualAdd-32               [-1, 15, 16]               0\n",
      "           Linear-33               [-1, 15, 16]             272\n",
      "           Linear-34               [-1, 15, 16]             272\n",
      "           Linear-35               [-1, 15, 16]             272\n",
      "          Dropout-36            [-1, 2, 15, 15]               0\n",
      "           Linear-37               [-1, 15, 16]             272\n",
      "MultiHeadAttention-38               [-1, 15, 16]               0\n",
      "          Dropout-39               [-1, 15, 16]               0\n",
      "        LayerNorm-40               [-1, 15, 16]              32\n",
      "      ResidualAdd-41               [-1, 15, 16]               0\n",
      "           Linear-42               [-1, 15, 64]           1,088\n",
      "             GELU-43               [-1, 15, 64]               0\n",
      "          Dropout-44               [-1, 15, 64]               0\n",
      "           Linear-45               [-1, 15, 16]           1,040\n",
      "          Dropout-46               [-1, 15, 16]               0\n",
      "        LayerNorm-47               [-1, 15, 16]              32\n",
      "      ResidualAdd-48               [-1, 15, 16]               0\n",
      "           Linear-49               [-1, 15, 16]             272\n",
      "           Linear-50               [-1, 15, 16]             272\n",
      "           Linear-51               [-1, 15, 16]             272\n",
      "          Dropout-52            [-1, 2, 15, 15]               0\n",
      "           Linear-53               [-1, 15, 16]             272\n",
      "MultiHeadAttention-54               [-1, 15, 16]               0\n",
      "          Dropout-55               [-1, 15, 16]               0\n",
      "        LayerNorm-56               [-1, 15, 16]              32\n",
      "      ResidualAdd-57               [-1, 15, 16]               0\n",
      "           Linear-58               [-1, 15, 64]           1,088\n",
      "             GELU-59               [-1, 15, 64]               0\n",
      "          Dropout-60               [-1, 15, 64]               0\n",
      "           Linear-61               [-1, 15, 16]           1,040\n",
      "          Dropout-62               [-1, 15, 16]               0\n",
      "        LayerNorm-63               [-1, 15, 16]              32\n",
      "      ResidualAdd-64               [-1, 15, 16]               0\n",
      "           Linear-65               [-1, 15, 16]             272\n",
      "           Linear-66               [-1, 15, 16]             272\n",
      "           Linear-67               [-1, 15, 16]             272\n",
      "          Dropout-68            [-1, 2, 15, 15]               0\n",
      "           Linear-69               [-1, 15, 16]             272\n",
      "MultiHeadAttention-70               [-1, 15, 16]               0\n",
      "          Dropout-71               [-1, 15, 16]               0\n",
      "        LayerNorm-72               [-1, 15, 16]              32\n",
      "      ResidualAdd-73               [-1, 15, 16]               0\n",
      "           Linear-74               [-1, 15, 64]           1,088\n",
      "             GELU-75               [-1, 15, 64]               0\n",
      "          Dropout-76               [-1, 15, 64]               0\n",
      "           Linear-77               [-1, 15, 16]           1,040\n",
      "          Dropout-78               [-1, 15, 16]               0\n",
      "        LayerNorm-79               [-1, 15, 16]              32\n",
      "      ResidualAdd-80               [-1, 15, 16]               0\n",
      "           Linear-81               [-1, 15, 16]             272\n",
      "           Linear-82               [-1, 15, 16]             272\n",
      "           Linear-83               [-1, 15, 16]             272\n",
      "          Dropout-84            [-1, 2, 15, 15]               0\n",
      "           Linear-85               [-1, 15, 16]             272\n",
      "MultiHeadAttention-86               [-1, 15, 16]               0\n",
      "          Dropout-87               [-1, 15, 16]               0\n",
      "        LayerNorm-88               [-1, 15, 16]              32\n",
      "      ResidualAdd-89               [-1, 15, 16]               0\n",
      "           Linear-90               [-1, 15, 64]           1,088\n",
      "             GELU-91               [-1, 15, 64]               0\n",
      "          Dropout-92               [-1, 15, 64]               0\n",
      "           Linear-93               [-1, 15, 16]           1,040\n",
      "          Dropout-94               [-1, 15, 16]               0\n",
      "        LayerNorm-95               [-1, 15, 16]              32\n",
      "      ResidualAdd-96               [-1, 15, 16]               0\n",
      "           Linear-97               [-1, 15, 16]             272\n",
      "           Linear-98               [-1, 15, 16]             272\n",
      "           Linear-99               [-1, 15, 16]             272\n",
      "         Dropout-100            [-1, 2, 15, 15]               0\n",
      "          Linear-101               [-1, 15, 16]             272\n",
      "MultiHeadAttention-102               [-1, 15, 16]               0\n",
      "         Dropout-103               [-1, 15, 16]               0\n",
      "       LayerNorm-104               [-1, 15, 16]              32\n",
      "     ResidualAdd-105               [-1, 15, 16]               0\n",
      "          Linear-106               [-1, 15, 64]           1,088\n",
      "            GELU-107               [-1, 15, 64]               0\n",
      "         Dropout-108               [-1, 15, 64]               0\n",
      "          Linear-109               [-1, 15, 16]           1,040\n",
      "         Dropout-110               [-1, 15, 16]               0\n",
      "       LayerNorm-111               [-1, 15, 16]              32\n",
      "     ResidualAdd-112               [-1, 15, 16]               0\n",
      "         Flatten-113                  [-1, 240]               0\n",
      "         Dropout-114                  [-1, 240]               0\n",
      "          Linear-115                    [-1, 2]             482\n",
      "================================================================\n",
      "Total params: 24,898\n",
      "Trainable params: 24,898\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.01\n",
      "Forward/backward pass size (MB): 1.11\n",
      "Params size (MB): 0.09\n",
      "Estimated Total Size (MB): 1.21\n",
      "----------------------------------------------------------------\n",
      "Tue Apr 30 10:34:56 2024\n",
      "seed is 226\n",
      "Subject 1\n",
      "(21460, 1, 3, 1000) (21460,) (1740, 1, 3, 1000) (1740,) (720, 1, 3, 1000) (720,)\n",
      "1_0 train_acc: 0.5826 train_loss: 0.001701\tval_acc: 0.709195 val_loss: 0.002291742, acc:0.583333\n",
      "1_1 train_acc: 0.7152 train_loss: 0.001091\tval_acc: 0.754023 val_loss: 0.002080160, acc:0.652778\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1_2 train_acc: 0.7885 train_loss: 0.000887\tval_acc: 0.763793 val_loss: 0.002071527, acc:0.691667\n",
      "1_3 train_acc: 0.8119 train_loss: 0.000817\tval_acc: 0.780460 val_loss: 0.002015371, acc:0.716667\n",
      "1_4 train_acc: 0.8249 train_loss: 0.000780\tval_acc: 0.790805 val_loss: 0.001941937, acc:0.718056\n",
      "1_5 train_acc: 0.8290 train_loss: 0.000755\tval_acc: 0.792529 val_loss: 0.001821356, acc:0.729167\n",
      "1_6 train_acc: 0.8333 train_loss: 0.000742\tval_acc: 0.791954 val_loss: 0.001784709, acc:0.711111\n",
      "1_8 train_acc: 0.8410 train_loss: 0.000716\tval_acc: 0.805172 val_loss: 0.001781061, acc:0.755556\n",
      "1_9 train_acc: 0.8465 train_loss: 0.000696\tval_acc: 0.794828 val_loss: 0.001726704, acc:0.741667\n",
      "1_13 train_acc: 0.8482 train_loss: 0.000685\tval_acc: 0.812644 val_loss: 0.001608711, acc:0.772222\n",
      "1_15 train_acc: 0.8566 train_loss: 0.000654\tval_acc: 0.809195 val_loss: 0.001595979, acc:0.752778\n",
      "1_21 train_acc: 0.8646 train_loss: 0.000632\tval_acc: 0.814943 val_loss: 0.001577248, acc:0.756944\n",
      "1_23 train_acc: 0.8624 train_loss: 0.000637\tval_acc: 0.816667 val_loss: 0.001538149, acc:0.769444\n",
      "1_32 train_acc: 0.8703 train_loss: 0.000595\tval_acc: 0.831034 val_loss: 0.001532875, acc:0.765278\n",
      "1_33 train_acc: 0.8690 train_loss: 0.000604\tval_acc: 0.823563 val_loss: 0.001509666, acc:0.754167\n",
      "1_37 train_acc: 0.8711 train_loss: 0.000599\tval_acc: 0.825287 val_loss: 0.001483664, acc:0.751389\n",
      "1_41 train_acc: 0.8745 train_loss: 0.000586\tval_acc: 0.829310 val_loss: 0.001475281, acc:0.762500\n",
      "1_49 train_acc: 0.8776 train_loss: 0.000570\tval_acc: 0.835632 val_loss: 0.001440754, acc:0.762500\n",
      "1_52 train_acc: 0.8761 train_loss: 0.000580\tval_acc: 0.835632 val_loss: 0.001390767, acc:0.768056\n",
      "1_73 train_acc: 0.8850 train_loss: 0.000540\tval_acc: 0.850000 val_loss: 0.001346471, acc:0.779167\n",
      "1_96 train_acc: 0.8897 train_loss: 0.000528\tval_acc: 0.851149 val_loss: 0.001334096, acc:0.773611\n",
      "1_104 train_acc: 0.8864 train_loss: 0.000531\tval_acc: 0.855172 val_loss: 0.001319922, acc:0.765278\n",
      "1_157 train_acc: 0.8940 train_loss: 0.000497\tval_acc: 0.859770 val_loss: 0.001312775, acc:0.773611\n",
      "1_166 train_acc: 0.8958 train_loss: 0.000492\tval_acc: 0.856897 val_loss: 0.001285731, acc:0.772222\n",
      "1_174 train_acc: 0.8974 train_loss: 0.000486\tval_acc: 0.857471 val_loss: 0.001252421, acc:0.776389\n",
      "1_225 train_acc: 0.8976 train_loss: 0.000477\tval_acc: 0.863218 val_loss: 0.001201012, acc:0.770833\n",
      "1_235 train_acc: 0.9002 train_loss: 0.000473\tval_acc: 0.864943 val_loss: 0.001176752, acc:0.780556\n",
      "1_248 train_acc: 0.8974 train_loss: 0.000483\tval_acc: 0.866667 val_loss: 0.001163153, acc:0.779167\n",
      "1_303 train_acc: 0.9035 train_loss: 0.000459\tval_acc: 0.860345 val_loss: 0.001162358, acc:0.788889\n",
      "1_308 train_acc: 0.9046 train_loss: 0.000458\tval_acc: 0.869540 val_loss: 0.001142844, acc:0.775000\n",
      "1_374 train_acc: 0.9034 train_loss: 0.000454\tval_acc: 0.863218 val_loss: 0.001136157, acc:0.768056\n",
      "1_526 train_acc: 0.9089 train_loss: 0.000438\tval_acc: 0.875287 val_loss: 0.001128392, acc:0.763889\n",
      "1_535 train_acc: 0.9093 train_loss: 0.000436\tval_acc: 0.875287 val_loss: 0.001111887, acc:0.762500\n",
      "epoch:  535 \tThe test accuracy is: 0.7625\n",
      " THE BEST ACCURACY IS 0.7625\tkappa is 0.525\n",
      "subject 1 duration: 9:12:51.724740\n",
      "seed is 499\n",
      "Subject 2\n",
      "(21608, 1, 3, 1000) (21608,) (1752, 1, 3, 1000) (1752,) (680, 1, 3, 1000) (680,)\n",
      "2_0 train_acc: 0.5939 train_loss: 0.001729\tval_acc: 0.708333 val_loss: 0.002253543, acc:0.639706\n",
      "2_2 train_acc: 0.7738 train_loss: 0.000940\tval_acc: 0.765982 val_loss: 0.001993858, acc:0.677941\n",
      "2_3 train_acc: 0.8082 train_loss: 0.000836\tval_acc: 0.789384 val_loss: 0.001840643, acc:0.673529\n",
      "2_4 train_acc: 0.8256 train_loss: 0.000783\tval_acc: 0.798516 val_loss: 0.001814705, acc:0.713235\n",
      "2_6 train_acc: 0.8367 train_loss: 0.000732\tval_acc: 0.797945 val_loss: 0.001768142, acc:0.689706\n",
      "2_8 train_acc: 0.8507 train_loss: 0.000690\tval_acc: 0.804224 val_loss: 0.001678226, acc:0.613235\n",
      "2_12 train_acc: 0.8614 train_loss: 0.000651\tval_acc: 0.804795 val_loss: 0.001613595, acc:0.700000\n",
      "2_27 train_acc: 0.8729 train_loss: 0.000606\tval_acc: 0.823059 val_loss: 0.001521542, acc:0.708824\n",
      "2_36 train_acc: 0.8746 train_loss: 0.000593\tval_acc: 0.823630 val_loss: 0.001519597, acc:0.682353\n",
      "2_51 train_acc: 0.8809 train_loss: 0.000562\tval_acc: 0.831621 val_loss: 0.001500065, acc:0.719118\n",
      "2_56 train_acc: 0.8854 train_loss: 0.000554\tval_acc: 0.829909 val_loss: 0.001443773, acc:0.702941\n",
      "2_65 train_acc: 0.8862 train_loss: 0.000551\tval_acc: 0.832763 val_loss: 0.001443137, acc:0.695588\n",
      "2_91 train_acc: 0.8899 train_loss: 0.000528\tval_acc: 0.836187 val_loss: 0.001439593, acc:0.713235\n",
      "2_105 train_acc: 0.8924 train_loss: 0.000518\tval_acc: 0.836758 val_loss: 0.001429131, acc:0.676471\n",
      "2_123 train_acc: 0.8975 train_loss: 0.000499\tval_acc: 0.840753 val_loss: 0.001399110, acc:0.710294\n",
      "2_129 train_acc: 0.8972 train_loss: 0.000504\tval_acc: 0.832763 val_loss: 0.001380875, acc:0.697059\n",
      "2_140 train_acc: 0.8968 train_loss: 0.000502\tval_acc: 0.844749 val_loss: 0.001359015, acc:0.713235\n",
      "2_170 train_acc: 0.9005 train_loss: 0.000490\tval_acc: 0.847603 val_loss: 0.001323620, acc:0.729412\n",
      "2_267 train_acc: 0.9043 train_loss: 0.000473\tval_acc: 0.854452 val_loss: 0.001323513, acc:0.727941\n",
      "2_471 train_acc: 0.9100 train_loss: 0.000442\tval_acc: 0.853881 val_loss: 0.001320396, acc:0.719118\n",
      "2_535 train_acc: 0.9125 train_loss: 0.000434\tval_acc: 0.858447 val_loss: 0.001276341, acc:0.710294\n",
      "epoch:  535 \tThe test accuracy is: 0.7102941176470589\n",
      " THE BEST ACCURACY IS 0.7102941176470589\tkappa is 0.4205882352941176\n",
      "subject 2 duration: 8:11:37.588192\n",
      "seed is 1042\n",
      "Subject 3\n",
      "(21460, 1, 3, 1000) (21460,) (1740, 1, 3, 1000) (1740,) (720, 1, 3, 1000) (720,)\n",
      "3_0 train_acc: 0.6036 train_loss: 0.001654\tval_acc: 0.722989 val_loss: 0.002126398, acc:0.652778\n",
      "3_1 train_acc: 0.7583 train_loss: 0.000983\tval_acc: 0.768966 val_loss: 0.001968478, acc:0.637500\n",
      "3_3 train_acc: 0.8310 train_loss: 0.000758\tval_acc: 0.777011 val_loss: 0.001945274, acc:0.654167\n",
      "3_4 train_acc: 0.8415 train_loss: 0.000712\tval_acc: 0.790230 val_loss: 0.001807878, acc:0.647222\n",
      "3_5 train_acc: 0.8465 train_loss: 0.000696\tval_acc: 0.798851 val_loss: 0.001790578, acc:0.655556\n",
      "3_7 train_acc: 0.8555 train_loss: 0.000661\tval_acc: 0.800000 val_loss: 0.001749452, acc:0.666667\n",
      "3_10 train_acc: 0.8616 train_loss: 0.000632\tval_acc: 0.797126 val_loss: 0.001722201, acc:0.654167\n",
      "3_12 train_acc: 0.8656 train_loss: 0.000617\tval_acc: 0.808046 val_loss: 0.001660626, acc:0.637500\n",
      "3_17 train_acc: 0.8742 train_loss: 0.000588\tval_acc: 0.817241 val_loss: 0.001637989, acc:0.648611\n",
      "3_19 train_acc: 0.8768 train_loss: 0.000576\tval_acc: 0.817241 val_loss: 0.001611129, acc:0.654167\n",
      "3_26 train_acc: 0.8842 train_loss: 0.000553\tval_acc: 0.818391 val_loss: 0.001592278, acc:0.650000\n",
      "3_37 train_acc: 0.8897 train_loss: 0.000528\tval_acc: 0.818966 val_loss: 0.001585280, acc:0.655556\n",
      "3_39 train_acc: 0.8900 train_loss: 0.000521\tval_acc: 0.827011 val_loss: 0.001540814, acc:0.641667\n",
      "3_44 train_acc: 0.8899 train_loss: 0.000518\tval_acc: 0.818966 val_loss: 0.001533024, acc:0.655556\n",
      "3_49 train_acc: 0.8950 train_loss: 0.000503\tval_acc: 0.822414 val_loss: 0.001504227, acc:0.650000\n",
      "3_56 train_acc: 0.8966 train_loss: 0.000495\tval_acc: 0.827011 val_loss: 0.001484847, acc:0.650000\n",
      "3_70 train_acc: 0.8975 train_loss: 0.000488\tval_acc: 0.828736 val_loss: 0.001469245, acc:0.634722\n",
      "3_76 train_acc: 0.9019 train_loss: 0.000480\tval_acc: 0.831034 val_loss: 0.001444601, acc:0.656944\n",
      "3_80 train_acc: 0.8972 train_loss: 0.000484\tval_acc: 0.833908 val_loss: 0.001429389, acc:0.637500\n",
      "3_92 train_acc: 0.8993 train_loss: 0.000473\tval_acc: 0.835057 val_loss: 0.001423775, acc:0.662500\n",
      "3_111 train_acc: 0.9037 train_loss: 0.000465\tval_acc: 0.840230 val_loss: 0.001419143, acc:0.668056\n",
      "3_117 train_acc: 0.9038 train_loss: 0.000460\tval_acc: 0.840805 val_loss: 0.001390107, acc:0.673611\n",
      "3_154 train_acc: 0.9097 train_loss: 0.000441\tval_acc: 0.845402 val_loss: 0.001371668, acc:0.663889\n",
      "3_157 train_acc: 0.9065 train_loss: 0.000445\tval_acc: 0.847701 val_loss: 0.001363305, acc:0.673611\n",
      "3_188 train_acc: 0.9095 train_loss: 0.000437\tval_acc: 0.847701 val_loss: 0.001357883, acc:0.668056\n",
      "3_191 train_acc: 0.9082 train_loss: 0.000442\tval_acc: 0.851149 val_loss: 0.001347153, acc:0.675000\n",
      "3_195 train_acc: 0.9115 train_loss: 0.000430\tval_acc: 0.844828 val_loss: 0.001337587, acc:0.673611\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3_209 train_acc: 0.9110 train_loss: 0.000423\tval_acc: 0.853448 val_loss: 0.001331546, acc:0.665278\n",
      "3_213 train_acc: 0.9090 train_loss: 0.000428\tval_acc: 0.858621 val_loss: 0.001306774, acc:0.665278\n",
      "3_228 train_acc: 0.9083 train_loss: 0.000434\tval_acc: 0.848276 val_loss: 0.001302337, acc:0.666667\n",
      "3_243 train_acc: 0.9112 train_loss: 0.000421\tval_acc: 0.858621 val_loss: 0.001299327, acc:0.666667\n",
      "3_259 train_acc: 0.9116 train_loss: 0.000420\tval_acc: 0.850000 val_loss: 0.001292892, acc:0.655556\n",
      "3_278 train_acc: 0.9110 train_loss: 0.000419\tval_acc: 0.852299 val_loss: 0.001274231, acc:0.684722\n",
      "3_341 train_acc: 0.9165 train_loss: 0.000404\tval_acc: 0.852299 val_loss: 0.001271016, acc:0.666667\n",
      "3_365 train_acc: 0.9151 train_loss: 0.000403\tval_acc: 0.861494 val_loss: 0.001253015, acc:0.676389\n",
      "3_390 train_acc: 0.9158 train_loss: 0.000405\tval_acc: 0.861494 val_loss: 0.001241993, acc:0.661111\n",
      "3_417 train_acc: 0.9167 train_loss: 0.000401\tval_acc: 0.858621 val_loss: 0.001234943, acc:0.666667\n",
      "3_434 train_acc: 0.9178 train_loss: 0.000398\tval_acc: 0.859770 val_loss: 0.001224511, acc:0.656944\n",
      "3_489 train_acc: 0.9203 train_loss: 0.000382\tval_acc: 0.865517 val_loss: 0.001214191, acc:0.670833\n",
      "3_506 train_acc: 0.9210 train_loss: 0.000389\tval_acc: 0.865517 val_loss: 0.001209442, acc:0.672222\n",
      "3_512 train_acc: 0.9201 train_loss: 0.000386\tval_acc: 0.858621 val_loss: 0.001208496, acc:0.666667\n",
      "3_520 train_acc: 0.9210 train_loss: 0.000382\tval_acc: 0.864943 val_loss: 0.001204482, acc:0.652778\n",
      "3_523 train_acc: 0.9213 train_loss: 0.000380\tval_acc: 0.864368 val_loss: 0.001192734, acc:0.666667\n",
      "3_547 train_acc: 0.9227 train_loss: 0.000382\tval_acc: 0.866092 val_loss: 0.001177833, acc:0.663889\n",
      "epoch:  547 \tThe test accuracy is: 0.6638888888888889\n",
      " THE BEST ACCURACY IS 0.6638888888888889\tkappa is 0.3277777777777777\n",
      "subject 3 duration: 7:28:20.887902\n",
      "seed is 839\n",
      "Subject 4\n",
      "(21386, 1, 3, 1000) (21386,) (1734, 1, 3, 1000) (1734,) (740, 1, 3, 1000) (740,)\n",
      "4_0 train_acc: 0.6034 train_loss: 0.001674\tval_acc: 0.709919 val_loss: 0.002269877, acc:0.682432\n",
      "4_1 train_acc: 0.7035 train_loss: 0.001136\tval_acc: 0.737601 val_loss: 0.002155213, acc:0.821622\n",
      "4_4 train_acc: 0.7960 train_loss: 0.000865\tval_acc: 0.777393 val_loss: 0.002100950, acc:0.812162\n",
      "4_6 train_acc: 0.8097 train_loss: 0.000814\tval_acc: 0.787197 val_loss: 0.001953197, acc:0.793243\n",
      "4_7 train_acc: 0.8157 train_loss: 0.000798\tval_acc: 0.788927 val_loss: 0.001900949, acc:0.790541\n",
      "4_11 train_acc: 0.8261 train_loss: 0.000763\tval_acc: 0.797001 val_loss: 0.001804903, acc:0.793243\n",
      "4_13 train_acc: 0.8329 train_loss: 0.000739\tval_acc: 0.791234 val_loss: 0.001800441, acc:0.791892\n",
      "4_21 train_acc: 0.8404 train_loss: 0.000708\tval_acc: 0.805652 val_loss: 0.001750939, acc:0.790541\n",
      "4_23 train_acc: 0.8464 train_loss: 0.000686\tval_acc: 0.806228 val_loss: 0.001710952, acc:0.786486\n",
      "4_28 train_acc: 0.8505 train_loss: 0.000672\tval_acc: 0.811995 val_loss: 0.001689196, acc:0.790541\n",
      "4_30 train_acc: 0.8520 train_loss: 0.000667\tval_acc: 0.806228 val_loss: 0.001678310, acc:0.787838\n",
      "4_32 train_acc: 0.8516 train_loss: 0.000667\tval_acc: 0.805075 val_loss: 0.001654681, acc:0.797297\n",
      "4_35 train_acc: 0.8519 train_loss: 0.000661\tval_acc: 0.815456 val_loss: 0.001640009, acc:0.801351\n",
      "4_36 train_acc: 0.8559 train_loss: 0.000656\tval_acc: 0.814879 val_loss: 0.001595396, acc:0.789189\n",
      "4_66 train_acc: 0.8653 train_loss: 0.000622\tval_acc: 0.818916 val_loss: 0.001563176, acc:0.810811\n",
      "4_72 train_acc: 0.8646 train_loss: 0.000620\tval_acc: 0.821223 val_loss: 0.001562788, acc:0.779730\n",
      "4_96 train_acc: 0.8741 train_loss: 0.000589\tval_acc: 0.826990 val_loss: 0.001532280, acc:0.801351\n",
      "4_107 train_acc: 0.8705 train_loss: 0.000593\tval_acc: 0.828143 val_loss: 0.001527704, acc:0.821622\n",
      "4_129 train_acc: 0.8780 train_loss: 0.000575\tval_acc: 0.829296 val_loss: 0.001525863, acc:0.813514\n",
      "4_131 train_acc: 0.8773 train_loss: 0.000574\tval_acc: 0.828720 val_loss: 0.001503360, acc:0.814865\n",
      "4_143 train_acc: 0.8765 train_loss: 0.000568\tval_acc: 0.833333 val_loss: 0.001500010, acc:0.791892\n",
      "4_154 train_acc: 0.8796 train_loss: 0.000560\tval_acc: 0.837947 val_loss: 0.001472586, acc:0.821622\n",
      "4_171 train_acc: 0.8800 train_loss: 0.000559\tval_acc: 0.841984 val_loss: 0.001421585, acc:0.802703\n",
      "4_250 train_acc: 0.8849 train_loss: 0.000535\tval_acc: 0.843714 val_loss: 0.001411421, acc:0.813514\n",
      "4_262 train_acc: 0.8861 train_loss: 0.000534\tval_acc: 0.839100 val_loss: 0.001382070, acc:0.813514\n",
      "4_322 train_acc: 0.8873 train_loss: 0.000526\tval_acc: 0.843137 val_loss: 0.001371103, acc:0.822973\n",
      "4_324 train_acc: 0.8917 train_loss: 0.000522\tval_acc: 0.844867 val_loss: 0.001363009, acc:0.789189\n",
      "4_344 train_acc: 0.8877 train_loss: 0.000530\tval_acc: 0.843714 val_loss: 0.001360557, acc:0.800000\n",
      "4_361 train_acc: 0.8868 train_loss: 0.000523\tval_acc: 0.832180 val_loss: 0.001339347, acc:0.808108\n",
      "4_446 train_acc: 0.8903 train_loss: 0.000510\tval_acc: 0.840254 val_loss: 0.001330591, acc:0.816216\n",
      "4_486 train_acc: 0.8908 train_loss: 0.000514\tval_acc: 0.853518 val_loss: 0.001327968, acc:0.813514\n",
      "4_523 train_acc: 0.8962 train_loss: 0.000489\tval_acc: 0.850058 val_loss: 0.001325596, acc:0.812162\n",
      "4_537 train_acc: 0.8960 train_loss: 0.000493\tval_acc: 0.840254 val_loss: 0.001291129, acc:0.817568\n",
      "epoch:  537 \tThe test accuracy is: 0.8175675675675675\n",
      " THE BEST ACCURACY IS 0.8175675675675675\tkappa is 0.6351351351351351\n",
      "subject 4 duration: 6:05:54.212682\n",
      "seed is 958\n",
      "Subject 5\n",
      "(21386, 1, 3, 1000) (21386,) (1734, 1, 3, 1000) (1734,) (740, 1, 3, 1000) (740,)\n",
      "5_0 train_acc: 0.5998 train_loss: 0.001693\tval_acc: 0.719146 val_loss: 0.002215713, acc:0.787838\n",
      "5_1 train_acc: 0.7462 train_loss: 0.001024\tval_acc: 0.736448 val_loss: 0.002168233, acc:0.809459\n",
      "5_2 train_acc: 0.7945 train_loss: 0.000874\tval_acc: 0.755479 val_loss: 0.002067552, acc:0.810811\n",
      "5_3 train_acc: 0.8131 train_loss: 0.000808\tval_acc: 0.757209 val_loss: 0.001902341, acc:0.797297\n",
      "5_6 train_acc: 0.8317 train_loss: 0.000747\tval_acc: 0.767013 val_loss: 0.001866316, acc:0.794595\n",
      "5_12 train_acc: 0.8436 train_loss: 0.000692\tval_acc: 0.797578 val_loss: 0.001772771, acc:0.809459\n",
      "5_15 train_acc: 0.8494 train_loss: 0.000682\tval_acc: 0.791234 val_loss: 0.001768377, acc:0.797297\n",
      "5_31 train_acc: 0.8610 train_loss: 0.000632\tval_acc: 0.801615 val_loss: 0.001672999, acc:0.809459\n",
      "5_35 train_acc: 0.8623 train_loss: 0.000624\tval_acc: 0.805652 val_loss: 0.001643921, acc:0.808108\n",
      "5_47 train_acc: 0.8659 train_loss: 0.000608\tval_acc: 0.817186 val_loss: 0.001603390, acc:0.814865\n",
      "5_51 train_acc: 0.8739 train_loss: 0.000590\tval_acc: 0.820646 val_loss: 0.001584238, acc:0.813514\n",
      "5_54 train_acc: 0.8688 train_loss: 0.000604\tval_acc: 0.820069 val_loss: 0.001561106, acc:0.821622\n",
      "5_68 train_acc: 0.8721 train_loss: 0.000587\tval_acc: 0.820069 val_loss: 0.001535530, acc:0.829730\n",
      "5_74 train_acc: 0.8732 train_loss: 0.000582\tval_acc: 0.820069 val_loss: 0.001534719, acc:0.810811\n",
      "5_79 train_acc: 0.8744 train_loss: 0.000575\tval_acc: 0.824683 val_loss: 0.001534515, acc:0.822973\n",
      "5_104 train_acc: 0.8783 train_loss: 0.000558\tval_acc: 0.826413 val_loss: 0.001531349, acc:0.821622\n",
      "5_111 train_acc: 0.8767 train_loss: 0.000561\tval_acc: 0.825260 val_loss: 0.001515783, acc:0.817568\n",
      "5_144 train_acc: 0.8827 train_loss: 0.000543\tval_acc: 0.829296 val_loss: 0.001501259, acc:0.825676\n",
      "5_166 train_acc: 0.8837 train_loss: 0.000536\tval_acc: 0.829296 val_loss: 0.001475970, acc:0.828378\n",
      "5_265 train_acc: 0.8954 train_loss: 0.000496\tval_acc: 0.830450 val_loss: 0.001473425, acc:0.816216\n",
      "5_296 train_acc: 0.8945 train_loss: 0.000500\tval_acc: 0.835063 val_loss: 0.001455105, acc:0.818919\n",
      "5_306 train_acc: 0.8932 train_loss: 0.000506\tval_acc: 0.834487 val_loss: 0.001454673, acc:0.798649\n",
      "5_312 train_acc: 0.8921 train_loss: 0.000496\tval_acc: 0.835640 val_loss: 0.001427923, acc:0.806757\n",
      "5_372 train_acc: 0.8955 train_loss: 0.000490\tval_acc: 0.843137 val_loss: 0.001426360, acc:0.814865\n",
      "5_409 train_acc: 0.8993 train_loss: 0.000484\tval_acc: 0.844291 val_loss: 0.001409221, acc:0.817568\n",
      "5_443 train_acc: 0.8966 train_loss: 0.000480\tval_acc: 0.841984 val_loss: 0.001387751, acc:0.835135\n",
      "5_482 train_acc: 0.8974 train_loss: 0.000486\tval_acc: 0.841984 val_loss: 0.001366167, acc:0.831081\n",
      "epoch:  482 \tThe test accuracy is: 0.831081081081081\n",
      " THE BEST ACCURACY IS 0.831081081081081\tkappa is 0.6621621621621622\n",
      "subject 5 duration: 5:36:21.528118\n",
      "seed is 64\n",
      "Subject 6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(21460, 1, 3, 1000) (21460,) (1740, 1, 3, 1000) (1740,) (720, 1, 3, 1000) (720,)\n",
      "6_0 train_acc: 0.6149 train_loss: 0.001648\tval_acc: 0.650575 val_loss: 0.002640955, acc:0.626389\n",
      "6_1 train_acc: 0.7544 train_loss: 0.000995\tval_acc: 0.741954 val_loss: 0.002253704, acc:0.672222\n",
      "6_2 train_acc: 0.8021 train_loss: 0.000849\tval_acc: 0.772414 val_loss: 0.001912321, acc:0.712500\n",
      "6_5 train_acc: 0.8353 train_loss: 0.000732\tval_acc: 0.772989 val_loss: 0.001890226, acc:0.687500\n",
      "6_6 train_acc: 0.8405 train_loss: 0.000710\tval_acc: 0.778161 val_loss: 0.001860066, acc:0.765278\n",
      "6_7 train_acc: 0.8413 train_loss: 0.000709\tval_acc: 0.781034 val_loss: 0.001853633, acc:0.719444\n",
      "6_9 train_acc: 0.8481 train_loss: 0.000679\tval_acc: 0.789080 val_loss: 0.001740157, acc:0.738889\n",
      "6_18 train_acc: 0.8589 train_loss: 0.000637\tval_acc: 0.794828 val_loss: 0.001720891, acc:0.745833\n",
      "6_21 train_acc: 0.8650 train_loss: 0.000622\tval_acc: 0.792529 val_loss: 0.001712501, acc:0.762500\n",
      "6_23 train_acc: 0.8650 train_loss: 0.000619\tval_acc: 0.797701 val_loss: 0.001690063, acc:0.733333\n",
      "6_24 train_acc: 0.8692 train_loss: 0.000603\tval_acc: 0.800575 val_loss: 0.001661134, acc:0.727778\n",
      "6_26 train_acc: 0.8687 train_loss: 0.000603\tval_acc: 0.801149 val_loss: 0.001612135, acc:0.738889\n",
      "6_28 train_acc: 0.8669 train_loss: 0.000602\tval_acc: 0.804023 val_loss: 0.001604791, acc:0.741667\n",
      "6_37 train_acc: 0.8754 train_loss: 0.000582\tval_acc: 0.806897 val_loss: 0.001582152, acc:0.730556\n",
      "6_52 train_acc: 0.8775 train_loss: 0.000566\tval_acc: 0.807471 val_loss: 0.001572225, acc:0.766667\n",
      "6_56 train_acc: 0.8795 train_loss: 0.000561\tval_acc: 0.815517 val_loss: 0.001516013, acc:0.720833\n",
      "6_88 train_acc: 0.8863 train_loss: 0.000535\tval_acc: 0.817816 val_loss: 0.001509015, acc:0.755556\n",
      "6_93 train_acc: 0.8871 train_loss: 0.000526\tval_acc: 0.813793 val_loss: 0.001494199, acc:0.763889\n",
      "6_100 train_acc: 0.8871 train_loss: 0.000525\tval_acc: 0.815517 val_loss: 0.001471676, acc:0.751389\n",
      "6_108 train_acc: 0.8900 train_loss: 0.000516\tval_acc: 0.816667 val_loss: 0.001468619, acc:0.794444\n",
      "6_132 train_acc: 0.8907 train_loss: 0.000511\tval_acc: 0.817241 val_loss: 0.001468540, acc:0.765278\n",
      "6_137 train_acc: 0.8924 train_loss: 0.000508\tval_acc: 0.821839 val_loss: 0.001409212, acc:0.755556\n",
      "6_185 train_acc: 0.8960 train_loss: 0.000495\tval_acc: 0.827586 val_loss: 0.001405896, acc:0.759722\n",
      "6_231 train_acc: 0.8973 train_loss: 0.000485\tval_acc: 0.825287 val_loss: 0.001372198, acc:0.766667\n",
      "6_263 train_acc: 0.9009 train_loss: 0.000476\tval_acc: 0.829310 val_loss: 0.001366997, acc:0.755556\n",
      "6_288 train_acc: 0.9016 train_loss: 0.000469\tval_acc: 0.828736 val_loss: 0.001363381, acc:0.743056\n",
      "6_337 train_acc: 0.9015 train_loss: 0.000465\tval_acc: 0.828161 val_loss: 0.001354970, acc:0.777778\n",
      "6_341 train_acc: 0.9039 train_loss: 0.000463\tval_acc: 0.833908 val_loss: 0.001344527, acc:0.777778\n",
      "6_345 train_acc: 0.9014 train_loss: 0.000461\tval_acc: 0.825862 val_loss: 0.001335116, acc:0.772222\n",
      "epoch:  345 \tThe test accuracy is: 0.7722222222222223\n",
      " THE BEST ACCURACY IS 0.7722222222222223\tkappa is 0.5444444444444445\n",
      "subject 6 duration: 4:13:19.228954\n",
      "seed is 1685\n",
      "Subject 7\n",
      "(21460, 1, 3, 1000) (21460,) (1740, 1, 3, 1000) (1740,) (720, 1, 3, 1000) (720,)\n",
      "7_0 train_acc: 0.5711 train_loss: 0.001835\tval_acc: 0.707471 val_loss: 0.002184408, acc:0.761111\n",
      "7_1 train_acc: 0.7197 train_loss: 0.001113\tval_acc: 0.743678 val_loss: 0.002171447, acc:0.730556\n",
      "7_2 train_acc: 0.7699 train_loss: 0.000945\tval_acc: 0.755172 val_loss: 0.002123256, acc:0.730556\n",
      "7_3 train_acc: 0.8009 train_loss: 0.000854\tval_acc: 0.763793 val_loss: 0.002059348, acc:0.747222\n",
      "7_4 train_acc: 0.8123 train_loss: 0.000816\tval_acc: 0.768391 val_loss: 0.002032213, acc:0.769444\n",
      "7_5 train_acc: 0.8165 train_loss: 0.000794\tval_acc: 0.776437 val_loss: 0.002003562, acc:0.762500\n",
      "7_7 train_acc: 0.8226 train_loss: 0.000772\tval_acc: 0.774138 val_loss: 0.001929596, acc:0.795833\n",
      "7_8 train_acc: 0.8321 train_loss: 0.000744\tval_acc: 0.781609 val_loss: 0.001878611, acc:0.802778\n",
      "7_11 train_acc: 0.8374 train_loss: 0.000719\tval_acc: 0.777586 val_loss: 0.001844388, acc:0.800000\n",
      "7_12 train_acc: 0.8383 train_loss: 0.000714\tval_acc: 0.794828 val_loss: 0.001786919, acc:0.805556\n",
      "7_15 train_acc: 0.8445 train_loss: 0.000694\tval_acc: 0.789655 val_loss: 0.001783316, acc:0.800000\n",
      "7_18 train_acc: 0.8467 train_loss: 0.000683\tval_acc: 0.797126 val_loss: 0.001734624, acc:0.804167\n",
      "7_20 train_acc: 0.8534 train_loss: 0.000662\tval_acc: 0.789080 val_loss: 0.001657789, acc:0.798611\n",
      "7_25 train_acc: 0.8567 train_loss: 0.000644\tval_acc: 0.801149 val_loss: 0.001635537, acc:0.808333\n",
      "7_29 train_acc: 0.8575 train_loss: 0.000640\tval_acc: 0.797701 val_loss: 0.001629501, acc:0.790278\n",
      "7_30 train_acc: 0.8623 train_loss: 0.000629\tval_acc: 0.804023 val_loss: 0.001623224, acc:0.802778\n",
      "7_33 train_acc: 0.8603 train_loss: 0.000628\tval_acc: 0.797126 val_loss: 0.001620706, acc:0.800000\n",
      "7_36 train_acc: 0.8623 train_loss: 0.000624\tval_acc: 0.805747 val_loss: 0.001613232, acc:0.798611\n",
      "7_37 train_acc: 0.8661 train_loss: 0.000617\tval_acc: 0.816667 val_loss: 0.001585431, acc:0.804167\n",
      "7_43 train_acc: 0.8634 train_loss: 0.000619\tval_acc: 0.813793 val_loss: 0.001577749, acc:0.809722\n",
      "7_45 train_acc: 0.8715 train_loss: 0.000599\tval_acc: 0.811494 val_loss: 0.001568131, acc:0.806944\n",
      "7_48 train_acc: 0.8696 train_loss: 0.000594\tval_acc: 0.807471 val_loss: 0.001560787, acc:0.806944\n",
      "7_53 train_acc: 0.8730 train_loss: 0.000591\tval_acc: 0.821264 val_loss: 0.001550503, acc:0.795833\n",
      "7_57 train_acc: 0.8720 train_loss: 0.000587\tval_acc: 0.817816 val_loss: 0.001544698, acc:0.804167\n",
      "7_61 train_acc: 0.8732 train_loss: 0.000588\tval_acc: 0.823563 val_loss: 0.001494448, acc:0.804167\n",
      "7_70 train_acc: 0.8737 train_loss: 0.000573\tval_acc: 0.821839 val_loss: 0.001480218, acc:0.820833\n",
      "7_74 train_acc: 0.8748 train_loss: 0.000576\tval_acc: 0.821264 val_loss: 0.001478539, acc:0.819444\n",
      "7_129 train_acc: 0.8814 train_loss: 0.000536\tval_acc: 0.829885 val_loss: 0.001459341, acc:0.797222\n",
      "7_130 train_acc: 0.8824 train_loss: 0.000541\tval_acc: 0.831034 val_loss: 0.001443589, acc:0.825000\n",
      "7_141 train_acc: 0.8828 train_loss: 0.000541\tval_acc: 0.836782 val_loss: 0.001425120, acc:0.802778\n",
      "7_147 train_acc: 0.8854 train_loss: 0.000534\tval_acc: 0.836782 val_loss: 0.001392271, acc:0.812500\n",
      "7_175 train_acc: 0.8851 train_loss: 0.000532\tval_acc: 0.838506 val_loss: 0.001364894, acc:0.823611\n",
      "7_207 train_acc: 0.8875 train_loss: 0.000520\tval_acc: 0.841954 val_loss: 0.001323742, acc:0.802778\n",
      "7_236 train_acc: 0.8918 train_loss: 0.000516\tval_acc: 0.840805 val_loss: 0.001322599, acc:0.808333\n",
      "7_273 train_acc: 0.8920 train_loss: 0.000500\tval_acc: 0.842529 val_loss: 0.001316182, acc:0.805556\n",
      "7_281 train_acc: 0.8911 train_loss: 0.000502\tval_acc: 0.841954 val_loss: 0.001304726, acc:0.786111\n",
      "7_287 train_acc: 0.8929 train_loss: 0.000496\tval_acc: 0.854023 val_loss: 0.001301511, acc:0.800000\n",
      "7_291 train_acc: 0.8934 train_loss: 0.000492\tval_acc: 0.851149 val_loss: 0.001266629, acc:0.801389\n",
      "7_379 train_acc: 0.8979 train_loss: 0.000478\tval_acc: 0.847126 val_loss: 0.001264829, acc:0.791667\n",
      "7_381 train_acc: 0.8974 train_loss: 0.000484\tval_acc: 0.854023 val_loss: 0.001255806, acc:0.798611\n",
      "7_392 train_acc: 0.8967 train_loss: 0.000487\tval_acc: 0.847701 val_loss: 0.001246470, acc:0.802778\n",
      "7_422 train_acc: 0.8974 train_loss: 0.000478\tval_acc: 0.846552 val_loss: 0.001243651, acc:0.790278\n",
      "7_437 train_acc: 0.8988 train_loss: 0.000468\tval_acc: 0.850000 val_loss: 0.001228044, acc:0.793056\n",
      "7_564 train_acc: 0.8994 train_loss: 0.000469\tval_acc: 0.850575 val_loss: 0.001222834, acc:0.791667\n",
      "epoch:  564 \tThe test accuracy is: 0.7916666666666666\n",
      " THE BEST ACCURACY IS 0.7916666666666666\tkappa is 0.5833333333333333\n",
      "subject 7 duration: 3:45:51.173940\n",
      "seed is 324\n",
      "Subject 8\n",
      "(21312, 1, 3, 1000) (21312,) (1728, 1, 3, 1000) (1728,) (760, 1, 3, 1000) (760,)\n",
      "8_0 train_acc: 0.6025 train_loss: 0.001627\tval_acc: 0.738426 val_loss: 0.001857437, acc:0.785526\n",
      "8_1 train_acc: 0.7226 train_loss: 0.001077\tval_acc: 0.770255 val_loss: 0.001638635, acc:0.740789\n",
      "8_2 train_acc: 0.7842 train_loss: 0.000916\tval_acc: 0.790509 val_loss: 0.001574780, acc:0.731579\n",
      "8_3 train_acc: 0.8025 train_loss: 0.000845\tval_acc: 0.798032 val_loss: 0.001532632, acc:0.747368\n",
      "8_6 train_acc: 0.8294 train_loss: 0.000748\tval_acc: 0.800347 val_loss: 0.001527850, acc:0.750000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8_7 train_acc: 0.8344 train_loss: 0.000734\tval_acc: 0.812500 val_loss: 0.001509017, acc:0.757895\n",
      "8_13 train_acc: 0.8483 train_loss: 0.000688\tval_acc: 0.822917 val_loss: 0.001461438, acc:0.759211\n",
      "8_19 train_acc: 0.8567 train_loss: 0.000656\tval_acc: 0.826389 val_loss: 0.001449819, acc:0.780263\n",
      "8_28 train_acc: 0.8668 train_loss: 0.000620\tval_acc: 0.831019 val_loss: 0.001435933, acc:0.764474\n",
      "8_29 train_acc: 0.8657 train_loss: 0.000619\tval_acc: 0.833912 val_loss: 0.001383229, acc:0.763158\n",
      "8_35 train_acc: 0.8657 train_loss: 0.000615\tval_acc: 0.829861 val_loss: 0.001378083, acc:0.763158\n",
      "8_52 train_acc: 0.8758 train_loss: 0.000588\tval_acc: 0.837384 val_loss: 0.001312970, acc:0.764474\n",
      "8_65 train_acc: 0.8751 train_loss: 0.000574\tval_acc: 0.842593 val_loss: 0.001281239, acc:0.763158\n",
      "8_73 train_acc: 0.8793 train_loss: 0.000557\tval_acc: 0.849537 val_loss: 0.001270197, acc:0.767105\n",
      "8_105 train_acc: 0.8836 train_loss: 0.000544\tval_acc: 0.850694 val_loss: 0.001260951, acc:0.753947\n",
      "8_117 train_acc: 0.8854 train_loss: 0.000536\tval_acc: 0.844329 val_loss: 0.001252403, acc:0.752632\n",
      "8_120 train_acc: 0.8827 train_loss: 0.000536\tval_acc: 0.850116 val_loss: 0.001238597, acc:0.755263\n",
      "8_181 train_acc: 0.8909 train_loss: 0.000515\tval_acc: 0.853009 val_loss: 0.001222200, acc:0.751316\n",
      "8_219 train_acc: 0.8950 train_loss: 0.000500\tval_acc: 0.854745 val_loss: 0.001219273, acc:0.744737\n",
      "8_260 train_acc: 0.8950 train_loss: 0.000490\tval_acc: 0.853588 val_loss: 0.001215693, acc:0.731579\n",
      "8_262 train_acc: 0.8952 train_loss: 0.000485\tval_acc: 0.853009 val_loss: 0.001214374, acc:0.752632\n",
      "8_283 train_acc: 0.8981 train_loss: 0.000480\tval_acc: 0.854167 val_loss: 0.001195455, acc:0.732895\n",
      "8_330 train_acc: 0.8974 train_loss: 0.000484\tval_acc: 0.860532 val_loss: 0.001191495, acc:0.735526\n",
      "8_347 train_acc: 0.9009 train_loss: 0.000470\tval_acc: 0.855903 val_loss: 0.001176302, acc:0.746053\n",
      "8_354 train_acc: 0.8993 train_loss: 0.000470\tval_acc: 0.858218 val_loss: 0.001172899, acc:0.728947\n",
      "8_393 train_acc: 0.9016 train_loss: 0.000464\tval_acc: 0.859375 val_loss: 0.001146488, acc:0.742105\n",
      "8_477 train_acc: 0.9005 train_loss: 0.000461\tval_acc: 0.862847 val_loss: 0.001145915, acc:0.739474\n",
      "8_494 train_acc: 0.9003 train_loss: 0.000467\tval_acc: 0.864005 val_loss: 0.001144116, acc:0.747368\n",
      "8_495 train_acc: 0.9027 train_loss: 0.000455\tval_acc: 0.859954 val_loss: 0.001139080, acc:0.740789\n",
      "8_553 train_acc: 0.9037 train_loss: 0.000455\tval_acc: 0.857639 val_loss: 0.001111610, acc:0.735526\n",
      "epoch:  553 \tThe test accuracy is: 0.7355263157894737\n",
      " THE BEST ACCURACY IS 0.7355263157894737\tkappa is 0.4710526315789474\n",
      "subject 8 duration: 2:56:47.845484\n",
      "seed is 1162\n",
      "Subject 9\n",
      "(21460, 1, 3, 1000) (21460,) (1740, 1, 3, 1000) (1740,) (720, 1, 3, 1000) (720,)\n",
      "9_0 train_acc: 0.6049 train_loss: 0.001626\tval_acc: 0.735057 val_loss: 0.002152430, acc:0.581944\n",
      "9_3 train_acc: 0.8067 train_loss: 0.000824\tval_acc: 0.770690 val_loss: 0.002005007, acc:0.740278\n",
      "9_5 train_acc: 0.8283 train_loss: 0.000755\tval_acc: 0.786207 val_loss: 0.001919011, acc:0.751389\n",
      "9_6 train_acc: 0.8338 train_loss: 0.000732\tval_acc: 0.790805 val_loss: 0.001902908, acc:0.748611\n",
      "9_11 train_acc: 0.8464 train_loss: 0.000680\tval_acc: 0.797701 val_loss: 0.001820368, acc:0.758333\n",
      "9_14 train_acc: 0.8503 train_loss: 0.000666\tval_acc: 0.803448 val_loss: 0.001798412, acc:0.756944\n",
      "9_19 train_acc: 0.8579 train_loss: 0.000649\tval_acc: 0.804598 val_loss: 0.001760186, acc:0.765278\n",
      "9_23 train_acc: 0.8648 train_loss: 0.000621\tval_acc: 0.807471 val_loss: 0.001724430, acc:0.773611\n",
      "9_27 train_acc: 0.8644 train_loss: 0.000615\tval_acc: 0.810920 val_loss: 0.001711798, acc:0.777778\n",
      "9_32 train_acc: 0.8711 train_loss: 0.000598\tval_acc: 0.815517 val_loss: 0.001660927, acc:0.769444\n",
      "9_39 train_acc: 0.8725 train_loss: 0.000591\tval_acc: 0.816092 val_loss: 0.001646260, acc:0.772222\n",
      "9_43 train_acc: 0.8741 train_loss: 0.000583\tval_acc: 0.818966 val_loss: 0.001641856, acc:0.780556\n",
      "9_46 train_acc: 0.8757 train_loss: 0.000574\tval_acc: 0.822414 val_loss: 0.001638006, acc:0.776389\n",
      "9_51 train_acc: 0.8772 train_loss: 0.000571\tval_acc: 0.823563 val_loss: 0.001637387, acc:0.783333\n",
      "9_53 train_acc: 0.8797 train_loss: 0.000565\tval_acc: 0.829885 val_loss: 0.001602858, acc:0.777778\n",
      "9_55 train_acc: 0.8814 train_loss: 0.000562\tval_acc: 0.827011 val_loss: 0.001587098, acc:0.779167\n",
      "9_61 train_acc: 0.8801 train_loss: 0.000555\tval_acc: 0.829885 val_loss: 0.001564698, acc:0.769444\n",
      "9_77 train_acc: 0.8826 train_loss: 0.000544\tval_acc: 0.828736 val_loss: 0.001537035, acc:0.781944\n",
      "9_80 train_acc: 0.8845 train_loss: 0.000542\tval_acc: 0.840230 val_loss: 0.001524626, acc:0.781944\n",
      "9_83 train_acc: 0.8854 train_loss: 0.000546\tval_acc: 0.840805 val_loss: 0.001485242, acc:0.768056\n",
      "9_85 train_acc: 0.8842 train_loss: 0.000537\tval_acc: 0.839655 val_loss: 0.001485019, acc:0.768056\n",
      "9_99 train_acc: 0.8874 train_loss: 0.000525\tval_acc: 0.840805 val_loss: 0.001476961, acc:0.772222\n",
      "9_119 train_acc: 0.8875 train_loss: 0.000525\tval_acc: 0.837931 val_loss: 0.001452917, acc:0.765278\n",
      "9_122 train_acc: 0.8887 train_loss: 0.000514\tval_acc: 0.847126 val_loss: 0.001449777, acc:0.776389\n",
      "9_124 train_acc: 0.8916 train_loss: 0.000512\tval_acc: 0.839080 val_loss: 0.001438394, acc:0.769444\n",
      "9_138 train_acc: 0.8939 train_loss: 0.000501\tval_acc: 0.841954 val_loss: 0.001428254, acc:0.776389\n",
      "9_155 train_acc: 0.8918 train_loss: 0.000502\tval_acc: 0.840805 val_loss: 0.001411004, acc:0.769444\n",
      "9_161 train_acc: 0.8972 train_loss: 0.000489\tval_acc: 0.841954 val_loss: 0.001410337, acc:0.779167\n",
      "9_167 train_acc: 0.8967 train_loss: 0.000492\tval_acc: 0.844253 val_loss: 0.001390467, acc:0.775000\n",
      "9_202 train_acc: 0.8965 train_loss: 0.000484\tval_acc: 0.850575 val_loss: 0.001386294, acc:0.775000\n",
      "9_219 train_acc: 0.9006 train_loss: 0.000476\tval_acc: 0.853448 val_loss: 0.001336781, acc:0.770833\n",
      "9_261 train_acc: 0.9007 train_loss: 0.000475\tval_acc: 0.851149 val_loss: 0.001331380, acc:0.762500\n",
      "9_307 train_acc: 0.8996 train_loss: 0.000466\tval_acc: 0.850000 val_loss: 0.001315295, acc:0.763889\n",
      "9_348 train_acc: 0.9035 train_loss: 0.000459\tval_acc: 0.852874 val_loss: 0.001307962, acc:0.784722\n",
      "9_373 train_acc: 0.9020 train_loss: 0.000466\tval_acc: 0.852299 val_loss: 0.001306150, acc:0.783333\n",
      "9_386 train_acc: 0.9065 train_loss: 0.000440\tval_acc: 0.855172 val_loss: 0.001292910, acc:0.773611\n",
      "9_405 train_acc: 0.9055 train_loss: 0.000443\tval_acc: 0.853448 val_loss: 0.001284802, acc:0.779167\n",
      "9_408 train_acc: 0.9034 train_loss: 0.000448\tval_acc: 0.851724 val_loss: 0.001261190, acc:0.783333\n",
      "9_428 train_acc: 0.9058 train_loss: 0.000443\tval_acc: 0.859195 val_loss: 0.001258981, acc:0.779167\n",
      "epoch:  428 \tThe test accuracy is: 0.7791666666666667\n",
      " THE BEST ACCURACY IS 0.7791666666666667\tkappa is 0.5583333333333333\n",
      "subject 9 duration: 2:40:34.357891\n",
      "**The average Best accuracy is: 76.26570585032917kappa is: 52.53141170065834\n",
      "\n",
      "best epochs:  [535, 535, 547, 537, 482, 345, 564, 553, 428]\n",
      "---------  all result  ---------\n",
      "        accuray  precision     recall         f1      kappa\n",
      "0     76.250000  77.441016  76.250000  75.989469  52.500000\n",
      "1     71.029412  71.254645  71.029412  70.952458  42.058824\n",
      "2     66.388889  66.390912  66.388889  66.387851  32.777778\n",
      "3     81.756757  86.633663  81.756757  81.128689  63.513514\n",
      "4     83.108108  86.007944  83.108108  82.761031  66.216216\n",
      "5     77.222222  82.238020  77.222222  76.300388  54.444444\n",
      "6     79.166667  79.276000  79.166667  79.147198  58.333333\n",
      "7     73.552632  79.190377  73.552632  72.210853  47.105263\n",
      "8     77.916667  78.687192  77.916667  77.767377  55.833333\n",
      "mean  76.265706  78.568863  76.265706  75.849479  52.531412\n",
      "std    5.260199   6.498233   5.260199   5.210070  10.520399\n",
      "****************************************\n",
      "Thu May  2 12:46:35 2024\n",
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [-1, 8, 3, 1000]             512\n",
      "       BatchNorm2d-2           [-1, 8, 3, 1000]              16\n",
      "            Conv2d-3          [-1, 16, 1, 1000]              48\n",
      "       BatchNorm2d-4          [-1, 16, 1, 1000]              32\n",
      "               ELU-5          [-1, 16, 1, 1000]               0\n",
      "         AvgPool2d-6           [-1, 16, 1, 125]               0\n",
      "           Dropout-7           [-1, 16, 1, 125]               0\n",
      "            Conv2d-8           [-1, 16, 1, 125]           4,096\n",
      "       BatchNorm2d-9           [-1, 16, 1, 125]              32\n",
      "              ELU-10           [-1, 16, 1, 125]               0\n",
      "        AvgPool2d-11            [-1, 16, 1, 15]               0\n",
      "          Dropout-12            [-1, 16, 1, 15]               0\n",
      "        Rearrange-13               [-1, 15, 16]               0\n",
      "PatchEmbeddingCNN-14               [-1, 15, 16]               0\n",
      "          Dropout-15               [-1, 15, 16]               0\n",
      "PositioinalEncoding-16               [-1, 15, 16]               0\n",
      "           Linear-17               [-1, 15, 16]             272\n",
      "           Linear-18               [-1, 15, 16]             272\n",
      "           Linear-19               [-1, 15, 16]             272\n",
      "          Dropout-20            [-1, 2, 15, 15]               0\n",
      "           Linear-21               [-1, 15, 16]             272\n",
      "MultiHeadAttention-22               [-1, 15, 16]               0\n",
      "          Dropout-23               [-1, 15, 16]               0\n",
      "        LayerNorm-24               [-1, 15, 16]              32\n",
      "      ResidualAdd-25               [-1, 15, 16]               0\n",
      "           Linear-26               [-1, 15, 64]           1,088\n",
      "             GELU-27               [-1, 15, 64]               0\n",
      "          Dropout-28               [-1, 15, 64]               0\n",
      "           Linear-29               [-1, 15, 16]           1,040\n",
      "          Dropout-30               [-1, 15, 16]               0\n",
      "        LayerNorm-31               [-1, 15, 16]              32\n",
      "      ResidualAdd-32               [-1, 15, 16]               0\n",
      "           Linear-33               [-1, 15, 16]             272\n",
      "           Linear-34               [-1, 15, 16]             272\n",
      "           Linear-35               [-1, 15, 16]             272\n",
      "          Dropout-36            [-1, 2, 15, 15]               0\n",
      "           Linear-37               [-1, 15, 16]             272\n",
      "MultiHeadAttention-38               [-1, 15, 16]               0\n",
      "          Dropout-39               [-1, 15, 16]               0\n",
      "        LayerNorm-40               [-1, 15, 16]              32\n",
      "      ResidualAdd-41               [-1, 15, 16]               0\n",
      "           Linear-42               [-1, 15, 64]           1,088\n",
      "             GELU-43               [-1, 15, 64]               0\n",
      "          Dropout-44               [-1, 15, 64]               0\n",
      "           Linear-45               [-1, 15, 16]           1,040\n",
      "          Dropout-46               [-1, 15, 16]               0\n",
      "        LayerNorm-47               [-1, 15, 16]              32\n",
      "      ResidualAdd-48               [-1, 15, 16]               0\n",
      "           Linear-49               [-1, 15, 16]             272\n",
      "           Linear-50               [-1, 15, 16]             272\n",
      "           Linear-51               [-1, 15, 16]             272\n",
      "          Dropout-52            [-1, 2, 15, 15]               0\n",
      "           Linear-53               [-1, 15, 16]             272\n",
      "MultiHeadAttention-54               [-1, 15, 16]               0\n",
      "          Dropout-55               [-1, 15, 16]               0\n",
      "        LayerNorm-56               [-1, 15, 16]              32\n",
      "      ResidualAdd-57               [-1, 15, 16]               0\n",
      "           Linear-58               [-1, 15, 64]           1,088\n",
      "             GELU-59               [-1, 15, 64]               0\n",
      "          Dropout-60               [-1, 15, 64]               0\n",
      "           Linear-61               [-1, 15, 16]           1,040\n",
      "          Dropout-62               [-1, 15, 16]               0\n",
      "        LayerNorm-63               [-1, 15, 16]              32\n",
      "      ResidualAdd-64               [-1, 15, 16]               0\n",
      "           Linear-65               [-1, 15, 16]             272\n",
      "           Linear-66               [-1, 15, 16]             272\n",
      "           Linear-67               [-1, 15, 16]             272\n",
      "          Dropout-68            [-1, 2, 15, 15]               0\n",
      "           Linear-69               [-1, 15, 16]             272\n",
      "MultiHeadAttention-70               [-1, 15, 16]               0\n",
      "          Dropout-71               [-1, 15, 16]               0\n",
      "        LayerNorm-72               [-1, 15, 16]              32\n",
      "      ResidualAdd-73               [-1, 15, 16]               0\n",
      "           Linear-74               [-1, 15, 64]           1,088\n",
      "             GELU-75               [-1, 15, 64]               0\n",
      "          Dropout-76               [-1, 15, 64]               0\n",
      "           Linear-77               [-1, 15, 16]           1,040\n",
      "          Dropout-78               [-1, 15, 16]               0\n",
      "        LayerNorm-79               [-1, 15, 16]              32\n",
      "      ResidualAdd-80               [-1, 15, 16]               0\n",
      "           Linear-81               [-1, 15, 16]             272\n",
      "           Linear-82               [-1, 15, 16]             272\n",
      "           Linear-83               [-1, 15, 16]             272\n",
      "          Dropout-84            [-1, 2, 15, 15]               0\n",
      "           Linear-85               [-1, 15, 16]             272\n",
      "MultiHeadAttention-86               [-1, 15, 16]               0\n",
      "          Dropout-87               [-1, 15, 16]               0\n",
      "        LayerNorm-88               [-1, 15, 16]              32\n",
      "      ResidualAdd-89               [-1, 15, 16]               0\n",
      "           Linear-90               [-1, 15, 64]           1,088\n",
      "             GELU-91               [-1, 15, 64]               0\n",
      "          Dropout-92               [-1, 15, 64]               0\n",
      "           Linear-93               [-1, 15, 16]           1,040\n",
      "          Dropout-94               [-1, 15, 16]               0\n",
      "        LayerNorm-95               [-1, 15, 16]              32\n",
      "      ResidualAdd-96               [-1, 15, 16]               0\n",
      "           Linear-97               [-1, 15, 16]             272\n",
      "           Linear-98               [-1, 15, 16]             272\n",
      "           Linear-99               [-1, 15, 16]             272\n",
      "         Dropout-100            [-1, 2, 15, 15]               0\n",
      "          Linear-101               [-1, 15, 16]             272\n",
      "MultiHeadAttention-102               [-1, 15, 16]               0\n",
      "         Dropout-103               [-1, 15, 16]               0\n",
      "       LayerNorm-104               [-1, 15, 16]              32\n",
      "     ResidualAdd-105               [-1, 15, 16]               0\n",
      "          Linear-106               [-1, 15, 64]           1,088\n",
      "            GELU-107               [-1, 15, 64]               0\n",
      "         Dropout-108               [-1, 15, 64]               0\n",
      "          Linear-109               [-1, 15, 16]           1,040\n",
      "         Dropout-110               [-1, 15, 16]               0\n",
      "       LayerNorm-111               [-1, 15, 16]              32\n",
      "     ResidualAdd-112               [-1, 15, 16]               0\n",
      "         Flatten-113                  [-1, 240]               0\n",
      "         Dropout-114                  [-1, 240]               0\n",
      "          Linear-115                    [-1, 2]             482\n",
      "================================================================\n",
      "Total params: 24,898\n",
      "Trainable params: 24,898\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.01\n",
      "Forward/backward pass size (MB): 1.11\n",
      "Params size (MB): 0.09\n",
      "Estimated Total Size (MB): 1.21\n",
      "----------------------------------------------------------------\n",
      "Thu May  2 12:46:35 2024\n",
      "seed is 1973\n",
      "Subject 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(21460, 1, 3, 1000) (21460,) (1740, 1, 3, 1000) (1740,) (720, 1, 3, 1000) (720,)\n",
      "1_0 train_acc: 0.5746 train_loss: 0.001792\tval_acc: 0.674713 val_loss: 0.002470924, acc:0.600000\n",
      "1_1 train_acc: 0.6788 train_loss: 0.001198\tval_acc: 0.699425 val_loss: 0.002289551, acc:0.631944\n",
      "1_2 train_acc: 0.7476 train_loss: 0.001000\tval_acc: 0.755747 val_loss: 0.002288638, acc:0.670833\n",
      "1_5 train_acc: 0.8344 train_loss: 0.000737\tval_acc: 0.783908 val_loss: 0.002140287, acc:0.733333\n",
      "1_9 train_acc: 0.8489 train_loss: 0.000683\tval_acc: 0.772414 val_loss: 0.002039800, acc:0.730556\n",
      "1_12 train_acc: 0.8560 train_loss: 0.000649\tval_acc: 0.792529 val_loss: 0.001926101, acc:0.754167\n",
      "1_22 train_acc: 0.8657 train_loss: 0.000611\tval_acc: 0.790805 val_loss: 0.001899680, acc:0.765278\n",
      "1_23 train_acc: 0.8648 train_loss: 0.000615\tval_acc: 0.788506 val_loss: 0.001835259, acc:0.751389\n",
      "1_30 train_acc: 0.8738 train_loss: 0.000582\tval_acc: 0.797126 val_loss: 0.001751210, acc:0.744444\n",
      "1_48 train_acc: 0.8784 train_loss: 0.000559\tval_acc: 0.805172 val_loss: 0.001701266, acc:0.745833\n",
      "1_61 train_acc: 0.8820 train_loss: 0.000546\tval_acc: 0.816667 val_loss: 0.001698681, acc:0.763889\n",
      "1_62 train_acc: 0.8853 train_loss: 0.000543\tval_acc: 0.816092 val_loss: 0.001692280, acc:0.763889\n",
      "1_63 train_acc: 0.8849 train_loss: 0.000537\tval_acc: 0.808046 val_loss: 0.001684910, acc:0.759722\n",
      "1_72 train_acc: 0.8847 train_loss: 0.000528\tval_acc: 0.817816 val_loss: 0.001662746, acc:0.763889\n",
      "1_104 train_acc: 0.8925 train_loss: 0.000510\tval_acc: 0.824713 val_loss: 0.001655098, acc:0.768056\n",
      "1_155 train_acc: 0.8974 train_loss: 0.000492\tval_acc: 0.825862 val_loss: 0.001635727, acc:0.763889\n",
      "1_176 train_acc: 0.8960 train_loss: 0.000487\tval_acc: 0.822989 val_loss: 0.001633285, acc:0.768056\n",
      "1_177 train_acc: 0.8959 train_loss: 0.000486\tval_acc: 0.820690 val_loss: 0.001612565, acc:0.768056\n",
      "1_180 train_acc: 0.8963 train_loss: 0.000484\tval_acc: 0.821264 val_loss: 0.001603082, acc:0.759722\n",
      "1_287 train_acc: 0.9052 train_loss: 0.000452\tval_acc: 0.835057 val_loss: 0.001593441, acc:0.780556\n",
      "1_312 train_acc: 0.9072 train_loss: 0.000446\tval_acc: 0.835632 val_loss: 0.001574321, acc:0.783333\n",
      "1_340 train_acc: 0.9062 train_loss: 0.000444\tval_acc: 0.833908 val_loss: 0.001543566, acc:0.770833\n",
      "1_378 train_acc: 0.9045 train_loss: 0.000441\tval_acc: 0.840805 val_loss: 0.001500669, acc:0.775000\n",
      "1_421 train_acc: 0.9078 train_loss: 0.000435\tval_acc: 0.834483 val_loss: 0.001490732, acc:0.768056\n",
      "1_453 train_acc: 0.9082 train_loss: 0.000435\tval_acc: 0.839080 val_loss: 0.001484842, acc:0.773611\n",
      "1_468 train_acc: 0.9089 train_loss: 0.000432\tval_acc: 0.841954 val_loss: 0.001477751, acc:0.784722\n",
      "1_490 train_acc: 0.9061 train_loss: 0.000437\tval_acc: 0.833908 val_loss: 0.001473654, acc:0.768056\n",
      "1_507 train_acc: 0.9100 train_loss: 0.000427\tval_acc: 0.838506 val_loss: 0.001466488, acc:0.768056\n",
      "1_509 train_acc: 0.9102 train_loss: 0.000425\tval_acc: 0.839655 val_loss: 0.001439425, acc:0.773611\n",
      "1_537 train_acc: 0.9092 train_loss: 0.000427\tval_acc: 0.850000 val_loss: 0.001430137, acc:0.788889\n",
      "1_588 train_acc: 0.9100 train_loss: 0.000417\tval_acc: 0.843103 val_loss: 0.001395925, acc:0.777778\n",
      "epoch:  588 \tThe test accuracy is: 0.7777777777777778\n",
      " THE BEST ACCURACY IS 0.7777777777777778\tkappa is 0.5555555555555556\n",
      "subject 1 duration: 2:41:00.293464\n",
      "seed is 981\n",
      "Subject 2\n",
      "(21608, 1, 3, 1000) (21608,) (1752, 1, 3, 1000) (1752,) (680, 1, 3, 1000) (680,)\n",
      "2_0 train_acc: 0.5896 train_loss: 0.001714\tval_acc: 0.695776 val_loss: 0.002246118, acc:0.633824\n",
      "2_1 train_acc: 0.7164 train_loss: 0.001104\tval_acc: 0.771119 val_loss: 0.001977716, acc:0.663235\n",
      "2_3 train_acc: 0.8210 train_loss: 0.000801\tval_acc: 0.776256 val_loss: 0.001969523, acc:0.660294\n",
      "2_4 train_acc: 0.8304 train_loss: 0.000766\tval_acc: 0.789384 val_loss: 0.001797005, acc:0.677941\n",
      "2_7 train_acc: 0.8477 train_loss: 0.000699\tval_acc: 0.792237 val_loss: 0.001770215, acc:0.613235\n",
      "2_8 train_acc: 0.8500 train_loss: 0.000692\tval_acc: 0.798516 val_loss: 0.001688515, acc:0.673529\n",
      "2_10 train_acc: 0.8565 train_loss: 0.000672\tval_acc: 0.801370 val_loss: 0.001653948, acc:0.685294\n",
      "2_12 train_acc: 0.8619 train_loss: 0.000652\tval_acc: 0.808219 val_loss: 0.001635083, acc:0.688235\n",
      "2_17 train_acc: 0.8699 train_loss: 0.000618\tval_acc: 0.807648 val_loss: 0.001584442, acc:0.650000\n",
      "2_19 train_acc: 0.8695 train_loss: 0.000619\tval_acc: 0.809932 val_loss: 0.001548642, acc:0.675000\n",
      "2_22 train_acc: 0.8736 train_loss: 0.000600\tval_acc: 0.814498 val_loss: 0.001533400, acc:0.691176\n",
      "2_24 train_acc: 0.8772 train_loss: 0.000588\tval_acc: 0.816781 val_loss: 0.001525394, acc:0.691176\n",
      "2_26 train_acc: 0.8767 train_loss: 0.000586\tval_acc: 0.819064 val_loss: 0.001501562, acc:0.708824\n",
      "2_34 train_acc: 0.8826 train_loss: 0.000570\tval_acc: 0.817922 val_loss: 0.001471106, acc:0.677941\n",
      "2_40 train_acc: 0.8848 train_loss: 0.000562\tval_acc: 0.816781 val_loss: 0.001431974, acc:0.669118\n",
      "2_48 train_acc: 0.8873 train_loss: 0.000543\tval_acc: 0.836187 val_loss: 0.001416004, acc:0.713235\n",
      "2_50 train_acc: 0.8883 train_loss: 0.000544\tval_acc: 0.832763 val_loss: 0.001375027, acc:0.692647\n",
      "2_89 train_acc: 0.8935 train_loss: 0.000515\tval_acc: 0.835616 val_loss: 0.001331157, acc:0.697059\n",
      "2_133 train_acc: 0.9014 train_loss: 0.000491\tval_acc: 0.844749 val_loss: 0.001322975, acc:0.679412\n",
      "2_155 train_acc: 0.8995 train_loss: 0.000489\tval_acc: 0.841324 val_loss: 0.001299572, acc:0.691176\n",
      "2_185 train_acc: 0.9015 train_loss: 0.000471\tval_acc: 0.839612 val_loss: 0.001288623, acc:0.677941\n",
      "2_187 train_acc: 0.9011 train_loss: 0.000482\tval_acc: 0.851598 val_loss: 0.001261006, acc:0.705882\n",
      "2_229 train_acc: 0.9086 train_loss: 0.000458\tval_acc: 0.855023 val_loss: 0.001207755, acc:0.689706\n",
      "2_251 train_acc: 0.9056 train_loss: 0.000457\tval_acc: 0.849315 val_loss: 0.001206771, acc:0.677941\n",
      "2_444 train_acc: 0.9107 train_loss: 0.000439\tval_acc: 0.847603 val_loss: 0.001201962, acc:0.670588\n",
      "2_561 train_acc: 0.9152 train_loss: 0.000418\tval_acc: 0.859018 val_loss: 0.001137437, acc:0.683824\n",
      "epoch:  561 \tThe test accuracy is: 0.6838235294117647\n",
      " THE BEST ACCURACY IS 0.6838235294117647\tkappa is 0.36764705882352944\n",
      "subject 2 duration: 3:03:06.741103\n",
      "seed is 1057\n",
      "Subject 3\n",
      "(21460, 1, 3, 1000) (21460,) (1740, 1, 3, 1000) (1740,) (720, 1, 3, 1000) (720,)\n",
      "3_0 train_acc: 0.6140 train_loss: 0.001683\tval_acc: 0.758046 val_loss: 0.002067841, acc:0.669444\n",
      "3_2 train_acc: 0.8034 train_loss: 0.000838\tval_acc: 0.805747 val_loss: 0.002042696, acc:0.629167\n",
      "3_3 train_acc: 0.8268 train_loss: 0.000757\tval_acc: 0.818966 val_loss: 0.001871284, acc:0.625000\n",
      "3_4 train_acc: 0.8403 train_loss: 0.000726\tval_acc: 0.819540 val_loss: 0.001870499, acc:0.655556\n",
      "3_5 train_acc: 0.8449 train_loss: 0.000697\tval_acc: 0.825862 val_loss: 0.001836802, acc:0.641667\n",
      "3_9 train_acc: 0.8603 train_loss: 0.000641\tval_acc: 0.825862 val_loss: 0.001828319, acc:0.620833\n",
      "3_10 train_acc: 0.8575 train_loss: 0.000647\tval_acc: 0.829885 val_loss: 0.001801368, acc:0.637500\n",
      "3_12 train_acc: 0.8633 train_loss: 0.000628\tval_acc: 0.832184 val_loss: 0.001712656, acc:0.647222\n",
      "3_27 train_acc: 0.8781 train_loss: 0.000575\tval_acc: 0.841954 val_loss: 0.001680774, acc:0.658333\n",
      "3_29 train_acc: 0.8786 train_loss: 0.000570\tval_acc: 0.835057 val_loss: 0.001659726, acc:0.651389\n",
      "3_31 train_acc: 0.8785 train_loss: 0.000564\tval_acc: 0.839080 val_loss: 0.001623121, acc:0.651389\n",
      "3_44 train_acc: 0.8841 train_loss: 0.000548\tval_acc: 0.846552 val_loss: 0.001596845, acc:0.663889\n",
      "3_49 train_acc: 0.8863 train_loss: 0.000538\tval_acc: 0.844253 val_loss: 0.001581025, acc:0.663889\n",
      "3_51 train_acc: 0.8849 train_loss: 0.000544\tval_acc: 0.852874 val_loss: 0.001573355, acc:0.654167\n",
      "3_53 train_acc: 0.8874 train_loss: 0.000532\tval_acc: 0.846552 val_loss: 0.001548736, acc:0.661111\n",
      "3_67 train_acc: 0.8912 train_loss: 0.000519\tval_acc: 0.848276 val_loss: 0.001490391, acc:0.655556\n",
      "3_82 train_acc: 0.8934 train_loss: 0.000507\tval_acc: 0.859195 val_loss: 0.001469399, acc:0.661111\n",
      "3_93 train_acc: 0.8944 train_loss: 0.000499\tval_acc: 0.854598 val_loss: 0.001461223, acc:0.656944\n",
      "3_105 train_acc: 0.8977 train_loss: 0.000493\tval_acc: 0.868391 val_loss: 0.001424138, acc:0.662500\n",
      "3_132 train_acc: 0.8978 train_loss: 0.000488\tval_acc: 0.858046 val_loss: 0.001420018, acc:0.658333\n",
      "3_150 train_acc: 0.9015 train_loss: 0.000474\tval_acc: 0.869540 val_loss: 0.001392144, acc:0.675000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3_162 train_acc: 0.9017 train_loss: 0.000472\tval_acc: 0.868966 val_loss: 0.001388880, acc:0.677778\n",
      "3_163 train_acc: 0.9021 train_loss: 0.000466\tval_acc: 0.866092 val_loss: 0.001386167, acc:0.675000\n",
      "3_180 train_acc: 0.9026 train_loss: 0.000464\tval_acc: 0.872989 val_loss: 0.001384812, acc:0.686111\n",
      "3_194 train_acc: 0.9030 train_loss: 0.000462\tval_acc: 0.875287 val_loss: 0.001366194, acc:0.688889\n",
      "3_202 train_acc: 0.9068 train_loss: 0.000460\tval_acc: 0.864943 val_loss: 0.001357199, acc:0.673611\n",
      "3_218 train_acc: 0.9054 train_loss: 0.000456\tval_acc: 0.877011 val_loss: 0.001334957, acc:0.679167\n",
      "3_256 train_acc: 0.9077 train_loss: 0.000448\tval_acc: 0.876437 val_loss: 0.001305893, acc:0.695833\n",
      "3_285 train_acc: 0.9080 train_loss: 0.000438\tval_acc: 0.865517 val_loss: 0.001269788, acc:0.676389\n",
      "3_370 train_acc: 0.9130 train_loss: 0.000421\tval_acc: 0.872989 val_loss: 0.001256486, acc:0.673611\n",
      "3_513 train_acc: 0.9165 train_loss: 0.000402\tval_acc: 0.874138 val_loss: 0.001249257, acc:0.676389\n",
      "3_546 train_acc: 0.9165 train_loss: 0.000405\tval_acc: 0.872989 val_loss: 0.001247788, acc:0.666667\n",
      "3_555 train_acc: 0.9155 train_loss: 0.000408\tval_acc: 0.874138 val_loss: 0.001234862, acc:0.668056\n",
      "3_578 train_acc: 0.9144 train_loss: 0.000407\tval_acc: 0.875862 val_loss: 0.001216852, acc:0.663889\n",
      "epoch:  578 \tThe test accuracy is: 0.6638888888888889\n",
      " THE BEST ACCURACY IS 0.6638888888888889\tkappa is 0.3277777777777777\n",
      "subject 3 duration: 5:39:28.759797\n",
      "seed is 313\n",
      "Subject 4\n",
      "(21386, 1, 3, 1000) (21386,) (1734, 1, 3, 1000) (1734,) (740, 1, 3, 1000) (740,)\n",
      "4_0 train_acc: 0.6211 train_loss: 0.001677\tval_acc: 0.710496 val_loss: 0.002190566, acc:0.563514\n",
      "4_1 train_acc: 0.7030 train_loss: 0.001142\tval_acc: 0.718570 val_loss: 0.002173275, acc:0.641892\n",
      "4_3 train_acc: 0.7651 train_loss: 0.000962\tval_acc: 0.752595 val_loss: 0.002051798, acc:0.824324\n",
      "4_4 train_acc: 0.7951 train_loss: 0.000868\tval_acc: 0.781430 val_loss: 0.001963118, acc:0.816216\n",
      "4_5 train_acc: 0.8102 train_loss: 0.000815\tval_acc: 0.782007 val_loss: 0.001945222, acc:0.818919\n",
      "4_10 train_acc: 0.8404 train_loss: 0.000716\tval_acc: 0.798155 val_loss: 0.001889531, acc:0.793243\n",
      "4_13 train_acc: 0.8434 train_loss: 0.000700\tval_acc: 0.799885 val_loss: 0.001845741, acc:0.795946\n",
      "4_20 train_acc: 0.8544 train_loss: 0.000665\tval_acc: 0.805652 val_loss: 0.001844958, acc:0.797297\n",
      "4_32 train_acc: 0.8605 train_loss: 0.000643\tval_acc: 0.809112 val_loss: 0.001829880, acc:0.798649\n",
      "4_38 train_acc: 0.8667 train_loss: 0.000629\tval_acc: 0.806228 val_loss: 0.001765380, acc:0.808108\n",
      "4_46 train_acc: 0.8620 train_loss: 0.000633\tval_acc: 0.813725 val_loss: 0.001760503, acc:0.808108\n",
      "4_92 train_acc: 0.8790 train_loss: 0.000581\tval_acc: 0.810842 val_loss: 0.001757150, acc:0.809459\n",
      "4_177 train_acc: 0.8851 train_loss: 0.000543\tval_acc: 0.832180 val_loss: 0.001739813, acc:0.822973\n",
      "4_201 train_acc: 0.8844 train_loss: 0.000539\tval_acc: 0.829873 val_loss: 0.001738922, acc:0.820270\n",
      "4_278 train_acc: 0.8912 train_loss: 0.000513\tval_acc: 0.834487 val_loss: 0.001735266, acc:0.816216\n",
      "4_297 train_acc: 0.8909 train_loss: 0.000514\tval_acc: 0.832757 val_loss: 0.001695051, acc:0.821622\n",
      "4_384 train_acc: 0.8939 train_loss: 0.000503\tval_acc: 0.842561 val_loss: 0.001686564, acc:0.821622\n",
      "4_401 train_acc: 0.8945 train_loss: 0.000496\tval_acc: 0.831027 val_loss: 0.001655464, acc:0.837838\n",
      "epoch:  401 \tThe test accuracy is: 0.8378378378378378\n",
      " THE BEST ACCURACY IS 0.8378378378378378\tkappa is 0.6756756756756757\n",
      "subject 4 duration: 8:08:12.618672\n",
      "seed is 1592\n",
      "Subject 5\n",
      "(21386, 1, 3, 1000) (21386,) (1734, 1, 3, 1000) (1734,) (740, 1, 3, 1000) (740,)\n",
      "5_0 train_acc: 0.5722 train_loss: 0.001810\tval_acc: 0.679354 val_loss: 0.002629578, acc:0.783784\n",
      "5_1 train_acc: 0.7034 train_loss: 0.001128\tval_acc: 0.729527 val_loss: 0.002410855, acc:0.777027\n",
      "5_2 train_acc: 0.7741 train_loss: 0.000939\tval_acc: 0.767013 val_loss: 0.002281839, acc:0.812162\n",
      "5_3 train_acc: 0.8008 train_loss: 0.000852\tval_acc: 0.774510 val_loss: 0.002163939, acc:0.800000\n",
      "5_4 train_acc: 0.8124 train_loss: 0.000802\tval_acc: 0.777393 val_loss: 0.002096942, acc:0.786486\n",
      "5_5 train_acc: 0.8237 train_loss: 0.000778\tval_acc: 0.785467 val_loss: 0.002066970, acc:0.804054\n",
      "5_7 train_acc: 0.8320 train_loss: 0.000744\tval_acc: 0.786044 val_loss: 0.001983798, acc:0.804054\n",
      "5_15 train_acc: 0.8503 train_loss: 0.000684\tval_acc: 0.794694 val_loss: 0.001905935, acc:0.800000\n",
      "5_16 train_acc: 0.8493 train_loss: 0.000682\tval_acc: 0.790081 val_loss: 0.001828797, acc:0.794595\n",
      "5_25 train_acc: 0.8560 train_loss: 0.000651\tval_acc: 0.811995 val_loss: 0.001798153, acc:0.801351\n",
      "5_31 train_acc: 0.8643 train_loss: 0.000626\tval_acc: 0.806228 val_loss: 0.001729459, acc:0.791892\n",
      "5_34 train_acc: 0.8610 train_loss: 0.000636\tval_acc: 0.806228 val_loss: 0.001713129, acc:0.813514\n",
      "5_44 train_acc: 0.8678 train_loss: 0.000609\tval_acc: 0.812572 val_loss: 0.001656853, acc:0.820270\n",
      "5_49 train_acc: 0.8690 train_loss: 0.000600\tval_acc: 0.808535 val_loss: 0.001638809, acc:0.817568\n",
      "5_50 train_acc: 0.8690 train_loss: 0.000602\tval_acc: 0.818339 val_loss: 0.001623958, acc:0.821622\n",
      "5_64 train_acc: 0.8738 train_loss: 0.000580\tval_acc: 0.815456 val_loss: 0.001622245, acc:0.814865\n",
      "5_75 train_acc: 0.8761 train_loss: 0.000577\tval_acc: 0.826990 val_loss: 0.001615832, acc:0.832432\n",
      "5_77 train_acc: 0.8763 train_loss: 0.000571\tval_acc: 0.818339 val_loss: 0.001605718, acc:0.810811\n",
      "5_82 train_acc: 0.8739 train_loss: 0.000580\tval_acc: 0.826413 val_loss: 0.001597404, acc:0.828378\n",
      "5_98 train_acc: 0.8819 train_loss: 0.000553\tval_acc: 0.823529 val_loss: 0.001566629, acc:0.827027\n",
      "5_107 train_acc: 0.8802 train_loss: 0.000553\tval_acc: 0.820069 val_loss: 0.001557234, acc:0.817568\n",
      "5_110 train_acc: 0.8825 train_loss: 0.000548\tval_acc: 0.826413 val_loss: 0.001545229, acc:0.822973\n",
      "5_118 train_acc: 0.8831 train_loss: 0.000540\tval_acc: 0.824106 val_loss: 0.001545031, acc:0.820270\n",
      "5_138 train_acc: 0.8882 train_loss: 0.000524\tval_acc: 0.833333 val_loss: 0.001542698, acc:0.813514\n",
      "5_145 train_acc: 0.8872 train_loss: 0.000526\tval_acc: 0.834487 val_loss: 0.001490915, acc:0.820270\n",
      "5_191 train_acc: 0.8949 train_loss: 0.000504\tval_acc: 0.843714 val_loss: 0.001454059, acc:0.822973\n",
      "5_215 train_acc: 0.8937 train_loss: 0.000499\tval_acc: 0.841407 val_loss: 0.001440821, acc:0.822973\n",
      "5_256 train_acc: 0.8939 train_loss: 0.000495\tval_acc: 0.842561 val_loss: 0.001429253, acc:0.817568\n",
      "5_272 train_acc: 0.8940 train_loss: 0.000493\tval_acc: 0.841407 val_loss: 0.001414400, acc:0.814865\n",
      "5_335 train_acc: 0.8975 train_loss: 0.000481\tval_acc: 0.854095 val_loss: 0.001410553, acc:0.831081\n",
      "5_432 train_acc: 0.9019 train_loss: 0.000463\tval_acc: 0.851788 val_loss: 0.001386399, acc:0.812162\n",
      "epoch:  432 \tThe test accuracy is: 0.8121621621621622\n",
      " THE BEST ACCURACY IS 0.8121621621621622\tkappa is 0.6243243243243244\n",
      "subject 5 duration: 6:53:38.374226\n",
      "seed is 1474\n",
      "Subject 6\n",
      "(21460, 1, 3, 1000) (21460,) (1740, 1, 3, 1000) (1740,) (720, 1, 3, 1000) (720,)\n",
      "6_0 train_acc: 0.6211 train_loss: 0.001654\tval_acc: 0.735057 val_loss: 0.002296954, acc:0.675000\n",
      "6_1 train_acc: 0.7569 train_loss: 0.001000\tval_acc: 0.755747 val_loss: 0.002149692, acc:0.729167\n",
      "6_2 train_acc: 0.7989 train_loss: 0.000863\tval_acc: 0.771264 val_loss: 0.001851761, acc:0.719444\n",
      "6_3 train_acc: 0.8093 train_loss: 0.000816\tval_acc: 0.775862 val_loss: 0.001790090, acc:0.711111\n",
      "6_4 train_acc: 0.8178 train_loss: 0.000790\tval_acc: 0.783333 val_loss: 0.001759604, acc:0.688889\n",
      "6_6 train_acc: 0.8309 train_loss: 0.000746\tval_acc: 0.786207 val_loss: 0.001731276, acc:0.752778\n",
      "6_7 train_acc: 0.8337 train_loss: 0.000729\tval_acc: 0.793103 val_loss: 0.001696213, acc:0.745833\n",
      "6_8 train_acc: 0.8397 train_loss: 0.000715\tval_acc: 0.793103 val_loss: 0.001670559, acc:0.766667\n",
      "6_9 train_acc: 0.8435 train_loss: 0.000700\tval_acc: 0.793103 val_loss: 0.001656168, acc:0.743056\n",
      "6_10 train_acc: 0.8481 train_loss: 0.000682\tval_acc: 0.794828 val_loss: 0.001648128, acc:0.763889\n",
      "6_11 train_acc: 0.8479 train_loss: 0.000687\tval_acc: 0.796552 val_loss: 0.001631891, acc:0.750000\n",
      "6_13 train_acc: 0.8526 train_loss: 0.000666\tval_acc: 0.809195 val_loss: 0.001540951, acc:0.811111\n",
      "6_17 train_acc: 0.8598 train_loss: 0.000646\tval_acc: 0.816092 val_loss: 0.001499496, acc:0.791667\n",
      "6_22 train_acc: 0.8652 train_loss: 0.000627\tval_acc: 0.817241 val_loss: 0.001488885, acc:0.780556\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6_25 train_acc: 0.8671 train_loss: 0.000611\tval_acc: 0.820690 val_loss: 0.001466543, acc:0.787500\n",
      "6_26 train_acc: 0.8643 train_loss: 0.000621\tval_acc: 0.817241 val_loss: 0.001434033, acc:0.798611\n",
      "6_28 train_acc: 0.8669 train_loss: 0.000611\tval_acc: 0.818966 val_loss: 0.001427874, acc:0.779167\n",
      "6_30 train_acc: 0.8661 train_loss: 0.000607\tval_acc: 0.815517 val_loss: 0.001421349, acc:0.827778\n",
      "6_33 train_acc: 0.8718 train_loss: 0.000600\tval_acc: 0.817816 val_loss: 0.001393795, acc:0.768056\n",
      "6_36 train_acc: 0.8710 train_loss: 0.000597\tval_acc: 0.828736 val_loss: 0.001384462, acc:0.795833\n",
      "6_41 train_acc: 0.8722 train_loss: 0.000590\tval_acc: 0.820690 val_loss: 0.001383728, acc:0.798611\n",
      "6_51 train_acc: 0.8727 train_loss: 0.000573\tval_acc: 0.833908 val_loss: 0.001365911, acc:0.788889\n",
      "6_52 train_acc: 0.8721 train_loss: 0.000578\tval_acc: 0.827586 val_loss: 0.001363703, acc:0.798611\n",
      "6_61 train_acc: 0.8778 train_loss: 0.000565\tval_acc: 0.829885 val_loss: 0.001343122, acc:0.802778\n",
      "6_64 train_acc: 0.8803 train_loss: 0.000562\tval_acc: 0.829310 val_loss: 0.001342381, acc:0.825000\n",
      "6_70 train_acc: 0.8776 train_loss: 0.000562\tval_acc: 0.831609 val_loss: 0.001338124, acc:0.794444\n",
      "6_74 train_acc: 0.8815 train_loss: 0.000551\tval_acc: 0.828736 val_loss: 0.001330204, acc:0.801389\n",
      "6_79 train_acc: 0.8788 train_loss: 0.000558\tval_acc: 0.831609 val_loss: 0.001311480, acc:0.787500\n",
      "6_92 train_acc: 0.8823 train_loss: 0.000540\tval_acc: 0.835057 val_loss: 0.001292688, acc:0.779167\n",
      "6_96 train_acc: 0.8830 train_loss: 0.000542\tval_acc: 0.836782 val_loss: 0.001292376, acc:0.783333\n",
      "6_141 train_acc: 0.8908 train_loss: 0.000512\tval_acc: 0.843103 val_loss: 0.001289074, acc:0.775000\n",
      "6_143 train_acc: 0.8883 train_loss: 0.000518\tval_acc: 0.847701 val_loss: 0.001270593, acc:0.784722\n",
      "6_146 train_acc: 0.8886 train_loss: 0.000520\tval_acc: 0.845402 val_loss: 0.001249442, acc:0.766667\n",
      "6_210 train_acc: 0.8941 train_loss: 0.000501\tval_acc: 0.847701 val_loss: 0.001237303, acc:0.766667\n",
      "6_266 train_acc: 0.8957 train_loss: 0.000491\tval_acc: 0.848276 val_loss: 0.001236987, acc:0.775000\n",
      "6_282 train_acc: 0.8973 train_loss: 0.000479\tval_acc: 0.851149 val_loss: 0.001236066, acc:0.787500\n",
      "6_340 train_acc: 0.8994 train_loss: 0.000473\tval_acc: 0.855172 val_loss: 0.001234463, acc:0.761111\n",
      "6_343 train_acc: 0.9000 train_loss: 0.000468\tval_acc: 0.848851 val_loss: 0.001228291, acc:0.751389\n",
      "6_386 train_acc: 0.8992 train_loss: 0.000470\tval_acc: 0.852874 val_loss: 0.001224942, acc:0.766667\n",
      "6_395 train_acc: 0.9024 train_loss: 0.000465\tval_acc: 0.857471 val_loss: 0.001222082, acc:0.756944\n",
      "6_398 train_acc: 0.9015 train_loss: 0.000469\tval_acc: 0.863218 val_loss: 0.001200454, acc:0.754167\n",
      "6_481 train_acc: 0.9071 train_loss: 0.000454\tval_acc: 0.863218 val_loss: 0.001190599, acc:0.754167\n",
      "epoch:  481 \tThe test accuracy is: 0.7541666666666667\n",
      " THE BEST ACCURACY IS 0.7541666666666667\tkappa is 0.5083333333333333\n",
      "subject 6 duration: 5:31:08.405979\n",
      "seed is 1157\n",
      "Subject 7\n",
      "(21460, 1, 3, 1000) (21460,) (1740, 1, 3, 1000) (1740,) (720, 1, 3, 1000) (720,)\n",
      "7_0 train_acc: 0.5744 train_loss: 0.001714\tval_acc: 0.701724 val_loss: 0.002218592, acc:0.788889\n",
      "7_1 train_acc: 0.7158 train_loss: 0.001090\tval_acc: 0.743103 val_loss: 0.002099890, acc:0.781944\n",
      "7_9 train_acc: 0.8356 train_loss: 0.000730\tval_acc: 0.766667 val_loss: 0.002060789, acc:0.795833\n",
      "7_10 train_acc: 0.8343 train_loss: 0.000731\tval_acc: 0.774713 val_loss: 0.001971792, acc:0.804167\n",
      "7_11 train_acc: 0.8393 train_loss: 0.000720\tval_acc: 0.768391 val_loss: 0.001930464, acc:0.808333\n",
      "7_18 train_acc: 0.8493 train_loss: 0.000677\tval_acc: 0.787931 val_loss: 0.001864591, acc:0.811111\n",
      "7_20 train_acc: 0.8540 train_loss: 0.000656\tval_acc: 0.784483 val_loss: 0.001856467, acc:0.811111\n",
      "7_29 train_acc: 0.8624 train_loss: 0.000626\tval_acc: 0.793103 val_loss: 0.001821972, acc:0.806944\n",
      "7_51 train_acc: 0.8745 train_loss: 0.000577\tval_acc: 0.805172 val_loss: 0.001729025, acc:0.819444\n",
      "7_82 train_acc: 0.8785 train_loss: 0.000557\tval_acc: 0.820115 val_loss: 0.001683405, acc:0.836111\n",
      "7_86 train_acc: 0.8821 train_loss: 0.000555\tval_acc: 0.820690 val_loss: 0.001675375, acc:0.822222\n",
      "7_119 train_acc: 0.8890 train_loss: 0.000531\tval_acc: 0.823563 val_loss: 0.001657828, acc:0.830556\n",
      "7_125 train_acc: 0.8858 train_loss: 0.000527\tval_acc: 0.823563 val_loss: 0.001647467, acc:0.826389\n",
      "7_137 train_acc: 0.8875 train_loss: 0.000522\tval_acc: 0.832759 val_loss: 0.001638840, acc:0.819444\n",
      "7_146 train_acc: 0.8914 train_loss: 0.000515\tval_acc: 0.824138 val_loss: 0.001637288, acc:0.825000\n",
      "7_152 train_acc: 0.8897 train_loss: 0.000507\tval_acc: 0.836782 val_loss: 0.001620673, acc:0.822222\n",
      "7_156 train_acc: 0.8926 train_loss: 0.000507\tval_acc: 0.835632 val_loss: 0.001615586, acc:0.830556\n",
      "7_159 train_acc: 0.8891 train_loss: 0.000505\tval_acc: 0.843103 val_loss: 0.001591060, acc:0.829167\n",
      "7_165 train_acc: 0.8866 train_loss: 0.000521\tval_acc: 0.830460 val_loss: 0.001585417, acc:0.823611\n",
      "7_169 train_acc: 0.8923 train_loss: 0.000507\tval_acc: 0.837931 val_loss: 0.001512700, acc:0.826389\n",
      "7_458 train_acc: 0.9064 train_loss: 0.000447\tval_acc: 0.843678 val_loss: 0.001487509, acc:0.818056\n",
      "7_559 train_acc: 0.9057 train_loss: 0.000442\tval_acc: 0.831609 val_loss: 0.001478078, acc:0.829167\n",
      "7_561 train_acc: 0.9040 train_loss: 0.000446\tval_acc: 0.834483 val_loss: 0.001468464, acc:0.820833\n",
      "7_581 train_acc: 0.9034 train_loss: 0.000451\tval_acc: 0.839655 val_loss: 0.001458164, acc:0.816667\n",
      "7_595 train_acc: 0.9030 train_loss: 0.000457\tval_acc: 0.843678 val_loss: 0.001445559, acc:0.819444\n",
      "epoch:  595 \tThe test accuracy is: 0.8194444444444444\n",
      " THE BEST ACCURACY IS 0.8194444444444444\tkappa is 0.6388888888888888\n",
      "subject 7 duration: 5:33:59.919624\n",
      "seed is 1586\n",
      "Subject 8\n",
      "(21312, 1, 3, 1000) (21312,) (1728, 1, 3, 1000) (1728,) (760, 1, 3, 1000) (760,)\n",
      "8_0 train_acc: 0.5861 train_loss: 0.001667\tval_acc: 0.685185 val_loss: 0.002022070, acc:0.785526\n",
      "8_1 train_acc: 0.6692 train_loss: 0.001211\tval_acc: 0.729745 val_loss: 0.001850090, acc:0.748684\n",
      "8_2 train_acc: 0.7355 train_loss: 0.001043\tval_acc: 0.783565 val_loss: 0.001630054, acc:0.702632\n",
      "8_3 train_acc: 0.7805 train_loss: 0.000931\tval_acc: 0.783565 val_loss: 0.001575128, acc:0.703947\n",
      "8_4 train_acc: 0.7972 train_loss: 0.000871\tval_acc: 0.792824 val_loss: 0.001531073, acc:0.715789\n",
      "8_5 train_acc: 0.8083 train_loss: 0.000825\tval_acc: 0.790509 val_loss: 0.001518612, acc:0.748684\n",
      "8_6 train_acc: 0.8170 train_loss: 0.000801\tval_acc: 0.802662 val_loss: 0.001507821, acc:0.747368\n",
      "8_7 train_acc: 0.8264 train_loss: 0.000772\tval_acc: 0.807292 val_loss: 0.001438429, acc:0.752632\n",
      "8_15 train_acc: 0.8553 train_loss: 0.000667\tval_acc: 0.813657 val_loss: 0.001430963, acc:0.751316\n",
      "8_21 train_acc: 0.8617 train_loss: 0.000643\tval_acc: 0.827546 val_loss: 0.001385056, acc:0.751316\n",
      "8_33 train_acc: 0.8663 train_loss: 0.000614\tval_acc: 0.833912 val_loss: 0.001316566, acc:0.740789\n",
      "8_65 train_acc: 0.8770 train_loss: 0.000570\tval_acc: 0.837384 val_loss: 0.001306857, acc:0.757895\n",
      "8_72 train_acc: 0.8785 train_loss: 0.000572\tval_acc: 0.850694 val_loss: 0.001306037, acc:0.750000\n",
      "8_79 train_acc: 0.8817 train_loss: 0.000564\tval_acc: 0.844907 val_loss: 0.001286971, acc:0.763158\n",
      "8_116 train_acc: 0.8851 train_loss: 0.000539\tval_acc: 0.841435 val_loss: 0.001271114, acc:0.752632\n",
      "8_147 train_acc: 0.8908 train_loss: 0.000517\tval_acc: 0.847222 val_loss: 0.001259274, acc:0.761842\n",
      "8_150 train_acc: 0.8887 train_loss: 0.000522\tval_acc: 0.849537 val_loss: 0.001250579, acc:0.748684\n",
      "8_183 train_acc: 0.8921 train_loss: 0.000512\tval_acc: 0.853588 val_loss: 0.001237445, acc:0.750000\n",
      "8_236 train_acc: 0.8960 train_loss: 0.000491\tval_acc: 0.848380 val_loss: 0.001234192, acc:0.757895\n",
      "8_266 train_acc: 0.8982 train_loss: 0.000487\tval_acc: 0.857060 val_loss: 0.001232790, acc:0.756579\n",
      "8_276 train_acc: 0.8964 train_loss: 0.000484\tval_acc: 0.859954 val_loss: 0.001204099, acc:0.750000\n",
      "8_304 train_acc: 0.8980 train_loss: 0.000481\tval_acc: 0.859954 val_loss: 0.001202223, acc:0.757895\n",
      "8_322 train_acc: 0.8986 train_loss: 0.000482\tval_acc: 0.853588 val_loss: 0.001187186, acc:0.760526\n",
      "8_418 train_acc: 0.9025 train_loss: 0.000455\tval_acc: 0.861111 val_loss: 0.001187162, acc:0.742105\n",
      "8_456 train_acc: 0.9075 train_loss: 0.000445\tval_acc: 0.858796 val_loss: 0.001169939, acc:0.748684\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8_469 train_acc: 0.9052 train_loss: 0.000460\tval_acc: 0.860532 val_loss: 0.001162418, acc:0.740789\n",
      "8_511 train_acc: 0.9043 train_loss: 0.000457\tval_acc: 0.865162 val_loss: 0.001150005, acc:0.740789\n",
      "epoch:  511 \tThe test accuracy is: 0.7407894736842106\n",
      " THE BEST ACCURACY IS 0.7407894736842106\tkappa is 0.4815789473684211\n",
      "subject 8 duration: 5:20:15.865561\n",
      "seed is 1876\n",
      "Subject 9\n",
      "(21460, 1, 3, 1000) (21460,) (1740, 1, 3, 1000) (1740,) (720, 1, 3, 1000) (720,)\n",
      "9_0 train_acc: 0.6116 train_loss: 0.001713\tval_acc: 0.762069 val_loss: 0.002014939, acc:0.630556\n",
      "9_1 train_acc: 0.7464 train_loss: 0.001043\tval_acc: 0.773563 val_loss: 0.001987236, acc:0.684722\n",
      "9_2 train_acc: 0.7939 train_loss: 0.000874\tval_acc: 0.786782 val_loss: 0.001835961, acc:0.723611\n",
      "9_4 train_acc: 0.8250 train_loss: 0.000767\tval_acc: 0.794253 val_loss: 0.001822396, acc:0.751389\n",
      "9_5 train_acc: 0.8317 train_loss: 0.000738\tval_acc: 0.797126 val_loss: 0.001808278, acc:0.754167\n",
      "9_6 train_acc: 0.8367 train_loss: 0.000723\tval_acc: 0.800000 val_loss: 0.001754885, acc:0.754167\n",
      "9_9 train_acc: 0.8483 train_loss: 0.000681\tval_acc: 0.811494 val_loss: 0.001689996, acc:0.762500\n",
      "9_12 train_acc: 0.8558 train_loss: 0.000664\tval_acc: 0.811494 val_loss: 0.001646373, acc:0.754167\n",
      "9_13 train_acc: 0.8566 train_loss: 0.000653\tval_acc: 0.816092 val_loss: 0.001619167, acc:0.754167\n",
      "9_20 train_acc: 0.8624 train_loss: 0.000633\tval_acc: 0.823563 val_loss: 0.001561893, acc:0.751389\n",
      "9_29 train_acc: 0.8718 train_loss: 0.000600\tval_acc: 0.827011 val_loss: 0.001550754, acc:0.765278\n",
      "9_41 train_acc: 0.8757 train_loss: 0.000580\tval_acc: 0.838506 val_loss: 0.001518330, acc:0.784722\n",
      "9_43 train_acc: 0.8774 train_loss: 0.000576\tval_acc: 0.839655 val_loss: 0.001496618, acc:0.777778\n",
      "9_47 train_acc: 0.8803 train_loss: 0.000561\tval_acc: 0.836207 val_loss: 0.001478754, acc:0.779167\n",
      "9_52 train_acc: 0.8772 train_loss: 0.000570\tval_acc: 0.832759 val_loss: 0.001476879, acc:0.777778\n",
      "9_60 train_acc: 0.8819 train_loss: 0.000550\tval_acc: 0.845402 val_loss: 0.001456262, acc:0.781944\n",
      "9_72 train_acc: 0.8830 train_loss: 0.000547\tval_acc: 0.844828 val_loss: 0.001410133, acc:0.786111\n",
      "9_83 train_acc: 0.8870 train_loss: 0.000536\tval_acc: 0.844828 val_loss: 0.001398963, acc:0.777778\n",
      "9_108 train_acc: 0.8880 train_loss: 0.000524\tval_acc: 0.839080 val_loss: 0.001384961, acc:0.776389\n",
      "9_134 train_acc: 0.8914 train_loss: 0.000509\tval_acc: 0.850000 val_loss: 0.001355824, acc:0.783333\n",
      "9_137 train_acc: 0.8917 train_loss: 0.000503\tval_acc: 0.844828 val_loss: 0.001353885, acc:0.788889\n",
      "9_150 train_acc: 0.8939 train_loss: 0.000499\tval_acc: 0.845977 val_loss: 0.001331341, acc:0.787500\n",
      "9_179 train_acc: 0.8958 train_loss: 0.000490\tval_acc: 0.851724 val_loss: 0.001324293, acc:0.794444\n",
      "9_206 train_acc: 0.8964 train_loss: 0.000491\tval_acc: 0.854023 val_loss: 0.001299460, acc:0.788889\n",
      "9_239 train_acc: 0.8989 train_loss: 0.000474\tval_acc: 0.858046 val_loss: 0.001283564, acc:0.787500\n",
      "9_382 train_acc: 0.9066 train_loss: 0.000447\tval_acc: 0.860345 val_loss: 0.001255558, acc:0.773611\n",
      "epoch:  382 \tThe test accuracy is: 0.7736111111111111\n",
      " THE BEST ACCURACY IS 0.7736111111111111\tkappa is 0.5472222222222223\n",
      "subject 9 duration: 5:43:47.698139\n",
      "**The average Best accuracy is: 76.26113213316516kappa is: 52.52226426633032\n",
      "\n",
      "best epochs:  [588, 561, 578, 401, 432, 481, 595, 511, 382]\n",
      "---------  all result  ---------\n",
      "        accuray  precision     recall         f1      kappa\n",
      "0     77.777778  77.847396  77.777778  77.763880  55.555556\n",
      "1     68.382353  71.473540  68.382353  67.202008  36.764706\n",
      "2     66.388889  67.588023  66.388889  65.806062  32.777778\n",
      "3     83.783784  87.608964  83.783784  83.360691  67.567568\n",
      "4     81.216216  84.954482  81.216216  80.700204  62.432432\n",
      "5     75.416667  81.200568  75.416667  74.221997  50.833333\n",
      "6     81.944444  81.945430  81.944444  81.944305  63.888889\n",
      "7     74.078947  80.374505  74.078947  72.662420  48.157895\n",
      "8     77.361111  77.686038  77.361111  77.294493  54.722222\n",
      "mean  76.261132  78.964327  76.261132  75.661784  52.522264\n",
      "std    5.936800   6.273972   5.936800   6.234839  11.873600\n",
      "****************************************\n",
      "Sat May  4 13:21:14 2024\n",
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [-1, 8, 3, 1000]             512\n",
      "       BatchNorm2d-2           [-1, 8, 3, 1000]              16\n",
      "            Conv2d-3          [-1, 16, 1, 1000]              48\n",
      "       BatchNorm2d-4          [-1, 16, 1, 1000]              32\n",
      "               ELU-5          [-1, 16, 1, 1000]               0\n",
      "         AvgPool2d-6           [-1, 16, 1, 125]               0\n",
      "           Dropout-7           [-1, 16, 1, 125]               0\n",
      "            Conv2d-8           [-1, 16, 1, 125]           4,096\n",
      "       BatchNorm2d-9           [-1, 16, 1, 125]              32\n",
      "              ELU-10           [-1, 16, 1, 125]               0\n",
      "        AvgPool2d-11            [-1, 16, 1, 15]               0\n",
      "          Dropout-12            [-1, 16, 1, 15]               0\n",
      "        Rearrange-13               [-1, 15, 16]               0\n",
      "PatchEmbeddingCNN-14               [-1, 15, 16]               0\n",
      "          Dropout-15               [-1, 15, 16]               0\n",
      "PositioinalEncoding-16               [-1, 15, 16]               0\n",
      "           Linear-17               [-1, 15, 16]             272\n",
      "           Linear-18               [-1, 15, 16]             272\n",
      "           Linear-19               [-1, 15, 16]             272\n",
      "          Dropout-20            [-1, 2, 15, 15]               0\n",
      "           Linear-21               [-1, 15, 16]             272\n",
      "MultiHeadAttention-22               [-1, 15, 16]               0\n",
      "          Dropout-23               [-1, 15, 16]               0\n",
      "        LayerNorm-24               [-1, 15, 16]              32\n",
      "      ResidualAdd-25               [-1, 15, 16]               0\n",
      "           Linear-26               [-1, 15, 64]           1,088\n",
      "             GELU-27               [-1, 15, 64]               0\n",
      "          Dropout-28               [-1, 15, 64]               0\n",
      "           Linear-29               [-1, 15, 16]           1,040\n",
      "          Dropout-30               [-1, 15, 16]               0\n",
      "        LayerNorm-31               [-1, 15, 16]              32\n",
      "      ResidualAdd-32               [-1, 15, 16]               0\n",
      "           Linear-33               [-1, 15, 16]             272\n",
      "           Linear-34               [-1, 15, 16]             272\n",
      "           Linear-35               [-1, 15, 16]             272\n",
      "          Dropout-36            [-1, 2, 15, 15]               0\n",
      "           Linear-37               [-1, 15, 16]             272\n",
      "MultiHeadAttention-38               [-1, 15, 16]               0\n",
      "          Dropout-39               [-1, 15, 16]               0\n",
      "        LayerNorm-40               [-1, 15, 16]              32\n",
      "      ResidualAdd-41               [-1, 15, 16]               0\n",
      "           Linear-42               [-1, 15, 64]           1,088\n",
      "             GELU-43               [-1, 15, 64]               0\n",
      "          Dropout-44               [-1, 15, 64]               0\n",
      "           Linear-45               [-1, 15, 16]           1,040\n",
      "          Dropout-46               [-1, 15, 16]               0\n",
      "        LayerNorm-47               [-1, 15, 16]              32\n",
      "      ResidualAdd-48               [-1, 15, 16]               0\n",
      "           Linear-49               [-1, 15, 16]             272\n",
      "           Linear-50               [-1, 15, 16]             272\n",
      "           Linear-51               [-1, 15, 16]             272\n",
      "          Dropout-52            [-1, 2, 15, 15]               0\n",
      "           Linear-53               [-1, 15, 16]             272\n",
      "MultiHeadAttention-54               [-1, 15, 16]               0\n",
      "          Dropout-55               [-1, 15, 16]               0\n",
      "        LayerNorm-56               [-1, 15, 16]              32\n",
      "      ResidualAdd-57               [-1, 15, 16]               0\n",
      "           Linear-58               [-1, 15, 64]           1,088\n",
      "             GELU-59               [-1, 15, 64]               0\n",
      "          Dropout-60               [-1, 15, 64]               0\n",
      "           Linear-61               [-1, 15, 16]           1,040\n",
      "          Dropout-62               [-1, 15, 16]               0\n",
      "        LayerNorm-63               [-1, 15, 16]              32\n",
      "      ResidualAdd-64               [-1, 15, 16]               0\n",
      "           Linear-65               [-1, 15, 16]             272\n",
      "           Linear-66               [-1, 15, 16]             272\n",
      "           Linear-67               [-1, 15, 16]             272\n",
      "          Dropout-68            [-1, 2, 15, 15]               0\n",
      "           Linear-69               [-1, 15, 16]             272\n",
      "MultiHeadAttention-70               [-1, 15, 16]               0\n",
      "          Dropout-71               [-1, 15, 16]               0\n",
      "        LayerNorm-72               [-1, 15, 16]              32\n",
      "      ResidualAdd-73               [-1, 15, 16]               0\n",
      "           Linear-74               [-1, 15, 64]           1,088\n",
      "             GELU-75               [-1, 15, 64]               0\n",
      "          Dropout-76               [-1, 15, 64]               0\n",
      "           Linear-77               [-1, 15, 16]           1,040\n",
      "          Dropout-78               [-1, 15, 16]               0\n",
      "        LayerNorm-79               [-1, 15, 16]              32\n",
      "      ResidualAdd-80               [-1, 15, 16]               0\n",
      "           Linear-81               [-1, 15, 16]             272\n",
      "           Linear-82               [-1, 15, 16]             272\n",
      "           Linear-83               [-1, 15, 16]             272\n",
      "          Dropout-84            [-1, 2, 15, 15]               0\n",
      "           Linear-85               [-1, 15, 16]             272\n",
      "MultiHeadAttention-86               [-1, 15, 16]               0\n",
      "          Dropout-87               [-1, 15, 16]               0\n",
      "        LayerNorm-88               [-1, 15, 16]              32\n",
      "      ResidualAdd-89               [-1, 15, 16]               0\n",
      "           Linear-90               [-1, 15, 64]           1,088\n",
      "             GELU-91               [-1, 15, 64]               0\n",
      "          Dropout-92               [-1, 15, 64]               0\n",
      "           Linear-93               [-1, 15, 16]           1,040\n",
      "          Dropout-94               [-1, 15, 16]               0\n",
      "        LayerNorm-95               [-1, 15, 16]              32\n",
      "      ResidualAdd-96               [-1, 15, 16]               0\n",
      "           Linear-97               [-1, 15, 16]             272\n",
      "           Linear-98               [-1, 15, 16]             272\n",
      "           Linear-99               [-1, 15, 16]             272\n",
      "         Dropout-100            [-1, 2, 15, 15]               0\n",
      "          Linear-101               [-1, 15, 16]             272\n",
      "MultiHeadAttention-102               [-1, 15, 16]               0\n",
      "         Dropout-103               [-1, 15, 16]               0\n",
      "       LayerNorm-104               [-1, 15, 16]              32\n",
      "     ResidualAdd-105               [-1, 15, 16]               0\n",
      "          Linear-106               [-1, 15, 64]           1,088\n",
      "            GELU-107               [-1, 15, 64]               0\n",
      "         Dropout-108               [-1, 15, 64]               0\n",
      "          Linear-109               [-1, 15, 16]           1,040\n",
      "         Dropout-110               [-1, 15, 16]               0\n",
      "       LayerNorm-111               [-1, 15, 16]              32\n",
      "     ResidualAdd-112               [-1, 15, 16]               0\n",
      "         Flatten-113                  [-1, 240]               0\n",
      "         Dropout-114                  [-1, 240]               0\n",
      "          Linear-115                    [-1, 2]             482\n",
      "================================================================\n",
      "Total params: 24,898\n",
      "Trainable params: 24,898\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.01\n",
      "Forward/backward pass size (MB): 1.11\n",
      "Params size (MB): 0.09\n",
      "Estimated Total Size (MB): 1.21\n",
      "----------------------------------------------------------------\n",
      "Sat May  4 13:21:15 2024\n",
      "seed is 51\n",
      "Subject 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(21460, 1, 3, 1000) (21460,) (1740, 1, 3, 1000) (1740,) (720, 1, 3, 1000) (720,)\n",
      "1_0 train_acc: 0.6007 train_loss: 0.001632\tval_acc: 0.719540 val_loss: 0.002529048, acc:0.638889\n",
      "1_1 train_acc: 0.7475 train_loss: 0.001023\tval_acc: 0.761494 val_loss: 0.002462116, acc:0.665278\n",
      "1_2 train_acc: 0.7975 train_loss: 0.000862\tval_acc: 0.780460 val_loss: 0.002306290, acc:0.701389\n",
      "1_3 train_acc: 0.8137 train_loss: 0.000803\tval_acc: 0.787356 val_loss: 0.002155039, acc:0.691667\n",
      "1_4 train_acc: 0.8248 train_loss: 0.000771\tval_acc: 0.797701 val_loss: 0.002118137, acc:0.715278\n",
      "1_6 train_acc: 0.8354 train_loss: 0.000725\tval_acc: 0.798276 val_loss: 0.002050336, acc:0.711111\n",
      "1_8 train_acc: 0.8461 train_loss: 0.000696\tval_acc: 0.804598 val_loss: 0.002018000, acc:0.720833\n",
      "1_9 train_acc: 0.8458 train_loss: 0.000689\tval_acc: 0.808621 val_loss: 0.001966125, acc:0.726389\n",
      "1_10 train_acc: 0.8475 train_loss: 0.000683\tval_acc: 0.802874 val_loss: 0.001948652, acc:0.720833\n",
      "1_11 train_acc: 0.8493 train_loss: 0.000676\tval_acc: 0.806322 val_loss: 0.001940547, acc:0.740278\n",
      "1_15 train_acc: 0.8596 train_loss: 0.000641\tval_acc: 0.804023 val_loss: 0.001889708, acc:0.723611\n",
      "1_16 train_acc: 0.8550 train_loss: 0.000646\tval_acc: 0.818391 val_loss: 0.001850613, acc:0.752778\n",
      "1_25 train_acc: 0.8658 train_loss: 0.000606\tval_acc: 0.822989 val_loss: 0.001803615, acc:0.770833\n",
      "1_33 train_acc: 0.8710 train_loss: 0.000586\tval_acc: 0.822414 val_loss: 0.001788863, acc:0.766667\n",
      "1_34 train_acc: 0.8723 train_loss: 0.000586\tval_acc: 0.817816 val_loss: 0.001785759, acc:0.776389\n",
      "1_37 train_acc: 0.8734 train_loss: 0.000587\tval_acc: 0.821839 val_loss: 0.001784244, acc:0.773611\n",
      "1_38 train_acc: 0.8766 train_loss: 0.000572\tval_acc: 0.822989 val_loss: 0.001703413, acc:0.770833\n",
      "1_59 train_acc: 0.8807 train_loss: 0.000554\tval_acc: 0.828161 val_loss: 0.001681812, acc:0.780556\n",
      "1_81 train_acc: 0.8877 train_loss: 0.000535\tval_acc: 0.832759 val_loss: 0.001660823, acc:0.768056\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "CTNet: A Convolution-Transformer Network for EEG-Based Motor Imagery Classification\n",
    "\n",
    "author: zhaowei701@163.com\n",
    "\n",
    "Due to memory constraints, the data augmentation method in LOSO classification was slightly optimized based on the approach used in subject-specific classification (main.py).\n",
    "\n",
    "\n",
    "Cite this work\n",
    "Zhao, W., Jiang, X., Zhang, B. et al. CTNet: a convolutional transformer network for EEG-based motor imagery classification. Sci Rep 14, 20237 (2024). https://doi.org/10.1038/s41598-024-71118-7\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "import os\n",
    "gpus = [0]\n",
    "os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '0'\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "import datetime\n",
    "import time\n",
    "import math\n",
    "from pandas import ExcelWriter\n",
    "from torchsummary import summary\n",
    "import torch\n",
    "from torch.backends import cudnn\n",
    "from utils import calMetrics\n",
    "from utils import calculatePerClass\n",
    "from utils import numberClassChannel\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "cudnn.benchmark = False\n",
    "cudnn.deterministic = True\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch import Tensor\n",
    "from einops.layers.torch import Rearrange, Reduce\n",
    "from einops import rearrange, reduce, repeat\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from utils import numberClassChannel\n",
    "from utils import load_data_evaluate\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from torch.autograd import Variable\n",
    "from sklearn.metrics import accuracy_score  \n",
    "from sklearn.metrics import precision_score \n",
    "from sklearn.metrics import recall_score  \n",
    "from sklearn.metrics import f1_score  \n",
    "\n",
    "\n",
    "\n",
    "class PatchEmbeddingCNN(nn.Module):\n",
    "    def __init__(self, f1=16, kernel_size=64, D=2, pooling_size1=8, pooling_size2=8, dropout_rate=0.3, number_channel=22, emb_size=40):\n",
    "        super().__init__()\n",
    "        f2 = D*f1\n",
    "        self.cnn_module = nn.Sequential(\n",
    "            # temporal conv kernel size 64=0.25fs\n",
    "            nn.Conv2d(1, f1, (1, kernel_size), (1, 1), padding='same', bias=False), # [batch, 22, 1000] \n",
    "            nn.BatchNorm2d(f1),\n",
    "            # channel depth-wise conv\n",
    "            nn.Conv2d(f1, f2, (number_channel, 1), (1, 1), groups=f1, padding='valid', bias=False), # \n",
    "            nn.BatchNorm2d(f2),\n",
    "            nn.ELU(),\n",
    "            # average pooling 1\n",
    "            nn.AvgPool2d((1, pooling_size1)),  # pooling acts as slicing to obtain 'patch' along the time dimension as in ViT\n",
    "            nn.Dropout(dropout_rate),\n",
    "            # spatial conv\n",
    "            nn.Conv2d(f2, f2, (1, 16), padding='same', bias=False), \n",
    "            nn.BatchNorm2d(f2),\n",
    "            nn.ELU(),\n",
    "\n",
    "            # average pooling 2 to adjust the length of feature into transformer encoder\n",
    "            nn.AvgPool2d((1, pooling_size2)),\n",
    "            nn.Dropout(dropout_rate),  \n",
    "                    \n",
    "        )\n",
    "\n",
    "        self.projection = nn.Sequential(\n",
    "            Rearrange('b e (h) (w) -> b (h w) e'),\n",
    "        )\n",
    "        \n",
    "        \n",
    "    def forward(self, x: Tensor) -> Tensor:\n",
    "        b, _, _, _ = x.shape\n",
    "        x = self.cnn_module(x)\n",
    "        x = self.projection(x)\n",
    "        return x\n",
    "    \n",
    "    \n",
    "  \n",
    "    \n",
    "class MultiHeadAttention(nn.Module):\n",
    "    def __init__(self, emb_size, num_heads, dropout):\n",
    "        super().__init__()\n",
    "        self.emb_size = emb_size\n",
    "        self.num_heads = num_heads\n",
    "        self.keys = nn.Linear(emb_size, emb_size)\n",
    "        self.queries = nn.Linear(emb_size, emb_size)\n",
    "        self.values = nn.Linear(emb_size, emb_size)\n",
    "        self.att_drop = nn.Dropout(dropout)\n",
    "        self.projection = nn.Linear(emb_size, emb_size)\n",
    "\n",
    "    def forward(self, x: Tensor, mask: Tensor = None) -> Tensor:\n",
    "        queries = rearrange(self.queries(x), \"b n (h d) -> b h n d\", h=self.num_heads)\n",
    "        keys = rearrange(self.keys(x), \"b n (h d) -> b h n d\", h=self.num_heads)\n",
    "        values = rearrange(self.values(x), \"b n (h d) -> b h n d\", h=self.num_heads)\n",
    "        energy = torch.einsum('bhqd, bhkd -> bhqk', queries, keys)  \n",
    "        if mask is not None:\n",
    "            fill_value = torch.finfo(torch.float32).min\n",
    "            energy.mask_fill(~mask, fill_value)\n",
    "\n",
    "        scaling = self.emb_size ** (1 / 2)\n",
    "        att = F.softmax(energy / scaling, dim=-1)\n",
    "        att = self.att_drop(att)\n",
    "        out = torch.einsum('bhal, bhlv -> bhav ', att, values)\n",
    "        out = rearrange(out, \"b h n d -> b n (h d)\")\n",
    "        out = self.projection(out)\n",
    "        return out\n",
    "    \n",
    "\n",
    "\n",
    "# PointWise FFN\n",
    "class FeedForwardBlock(nn.Sequential):\n",
    "    def __init__(self, emb_size, expansion, drop_p):\n",
    "        super().__init__(\n",
    "            nn.Linear(emb_size, expansion * emb_size),\n",
    "            nn.GELU(),\n",
    "            nn.Dropout(drop_p),\n",
    "            nn.Linear(expansion * emb_size, emb_size),\n",
    "        )\n",
    "\n",
    "\n",
    "\n",
    "class ClassificationHead(nn.Sequential):\n",
    "    def __init__(self, flatten_number, n_classes):\n",
    "        super().__init__()\n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(flatten_number, n_classes)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = self.fc(x)\n",
    "        \n",
    "        return out\n",
    "\n",
    "\n",
    "class ResidualAdd(nn.Module):\n",
    "    def __init__(self, fn, emb_size, drop_p):\n",
    "        super().__init__()\n",
    "        self.fn = fn\n",
    "        self.drop = nn.Dropout(drop_p)\n",
    "        self.layernorm = nn.LayerNorm(emb_size)\n",
    "\n",
    "    def forward(self, x, **kwargs):\n",
    "        x_input = x\n",
    "        res = self.fn(x, **kwargs)\n",
    "        \n",
    "        out = self.layernorm(self.drop(res)+x_input)\n",
    "        return out\n",
    "\n",
    "class TransformerEncoderBlock(nn.Sequential):\n",
    "    def __init__(self,\n",
    "                 emb_size,\n",
    "                 num_heads=4,\n",
    "                 drop_p=0.5,\n",
    "                 forward_expansion=4,\n",
    "                 forward_drop_p=0.5):\n",
    "        super().__init__(\n",
    "            ResidualAdd(nn.Sequential(\n",
    "                MultiHeadAttention(emb_size, num_heads, drop_p),\n",
    "                ), emb_size, drop_p),\n",
    "            ResidualAdd(nn.Sequential(\n",
    "                FeedForwardBlock(emb_size, expansion=forward_expansion, drop_p=forward_drop_p),\n",
    "                ), emb_size, drop_p)\n",
    "            \n",
    "            )    \n",
    "        \n",
    "        \n",
    "class TransformerEncoder(nn.Sequential):\n",
    "    def __init__(self, heads, depth, emb_size):\n",
    "        super().__init__(*[TransformerEncoderBlock(emb_size, heads) for _ in range(depth)])\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "class BranchEEGNetTransformer(nn.Sequential):\n",
    "    def __init__(self, heads=4, \n",
    "                 depth=6, \n",
    "                 emb_size=40, \n",
    "                 number_channel=22,\n",
    "                 f1 = 20,\n",
    "                 kernel_size = 64,\n",
    "                 D = 2,\n",
    "                 pooling_size1 = 8,\n",
    "                 pooling_size2 = 8,\n",
    "                 dropout_rate = 0.3,\n",
    "                 **kwargs):\n",
    "        super().__init__(\n",
    "            PatchEmbeddingCNN(f1=f1, \n",
    "                                 kernel_size=kernel_size,\n",
    "                                 D=D, \n",
    "                                 pooling_size1=pooling_size1, \n",
    "                                 pooling_size2=pooling_size2, \n",
    "                                 dropout_rate=dropout_rate,\n",
    "                                 number_channel=number_channel,\n",
    "                                 emb_size=emb_size),\n",
    "\n",
    "        )\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "        \n",
    "        \n",
    "class PositioinalEncoding(nn.Module):\n",
    "    def __init__(self, embedding, length=100, dropout=0.1):\n",
    "        super().__init__()\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.encoding = nn.Parameter(torch.randn(1, length, embedding))\n",
    "    def forward(self, x): # x-> [batch, embedding, length]\n",
    "        x = x + self.encoding[:, :x.shape[1], :].cuda()\n",
    "        return self.dropout(x)        \n",
    "        \n",
    "   \n",
    "    \n",
    "class EEGTransformer(nn.Module):\n",
    "    def __init__(self, heads=4, \n",
    "                 emb_size=40,\n",
    "                 depth=6, \n",
    "                 database_type='A', \n",
    "                 eeg1_f1 = 20,\n",
    "                 eeg1_kernel_size = 64,\n",
    "                 eeg1_D = 2,\n",
    "                 eeg1_pooling_size1 = 8,\n",
    "                 eeg1_pooling_size2 = 8,\n",
    "                 eeg1_dropout_rate = 0.3,\n",
    "                 eeg1_number_channel = 22,\n",
    "                 flatten_eeg1 = 600,\n",
    "                 **kwargs):\n",
    "        super().__init__()\n",
    "        self.number_class, self.number_channel = numberClassChannel(database_type)\n",
    "        self.emb_size = emb_size\n",
    "        self.flatten_eeg1 = flatten_eeg1\n",
    "        self.flatten = nn.Flatten()\n",
    "        # print('self.number_channel', self.number_channel)\n",
    "        self.cnn = BranchEEGNetTransformer(heads, depth, emb_size, number_channel=self.number_channel,\n",
    "                                              f1 = eeg1_f1,\n",
    "                                              kernel_size = eeg1_kernel_size,\n",
    "                                              D = eeg1_D,\n",
    "                                              pooling_size1 = eeg1_pooling_size1,\n",
    "                                              pooling_size2 = eeg1_pooling_size2,\n",
    "                                              dropout_rate = eeg1_dropout_rate,\n",
    "                                              )\n",
    "        self.position = PositioinalEncoding(emb_size, dropout=0.1)\n",
    "        self.trans = TransformerEncoder(heads, depth, emb_size)\n",
    "\n",
    "        self.flatten = nn.Flatten()\n",
    "        self.classification = ClassificationHead(self.flatten_eeg1 , self.number_class) # FLATTEN_EEGNet + FLATTEN_cnn_module\n",
    "    def forward(self, x):\n",
    "        cnn = self.cnn(x)\n",
    "        cnn = cnn * math.sqrt(self.emb_size)\n",
    "        cnn = self.position(cnn)\n",
    "        trans = self.trans(cnn)\n",
    "        \n",
    "        features = cnn+trans\n",
    "        out = self.classification(self.flatten(features))\n",
    "        return features, out\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "class ExP():\n",
    "    def __init__(self, nsub, data_dir, result_name, \n",
    "                 epochs=2000, \n",
    "                 number_aug=2,\n",
    "                 number_seg=8, \n",
    "                 gpus=[0], \n",
    "                 evaluate_mode = 'subject-dependent',\n",
    "                 heads=4, \n",
    "                 emb_size=40,\n",
    "                 depth=6, \n",
    "                 dataset_type='A',\n",
    "                 eeg1_f1 = 20,\n",
    "                 eeg1_kernel_size = 64,\n",
    "                 eeg1_D = 2,\n",
    "                 eeg1_pooling_size1 = 8,\n",
    "                 eeg1_pooling_size2 = 8,\n",
    "                 eeg1_dropout_rate = 0.3,\n",
    "                 flatten_eeg1 = 600, \n",
    "                 validate_ratio = 0.2,\n",
    "                 learning_rate = 0.001,\n",
    "                 batch_size = 72,  \n",
    "                 ):\n",
    "        \n",
    "        super(ExP, self).__init__()\n",
    "        self.dataset_type = dataset_type\n",
    "        self.batch_size = batch_size\n",
    "        self.lr = learning_rate\n",
    "        self.b1 = 0.5\n",
    "        self.b2 = 0.999\n",
    "        self.n_epochs = epochs\n",
    "        self.nSub = nsub\n",
    "        self.number_augmentation = number_aug\n",
    "        self.number_seg = number_seg\n",
    "        self.root = data_dir\n",
    "        self.heads=heads\n",
    "        self.emb_size=emb_size\n",
    "        self.depth=depth\n",
    "        self.result_name = result_name\n",
    "        self.evaluate_mode = evaluate_mode\n",
    "        self.validate_ratio = validate_ratio\n",
    "\n",
    "        self.Tensor = torch.cuda.FloatTensor\n",
    "        self.LongTensor = torch.cuda.LongTensor\n",
    "        self.criterion_cls = torch.nn.CrossEntropyLoss().cuda()\n",
    "\n",
    "        self.number_class, self.number_channel = numberClassChannel(self.dataset_type)\n",
    "        self.model = EEGTransformer(\n",
    "             heads=self.heads, \n",
    "             emb_size=self.emb_size,\n",
    "             depth=self.depth, \n",
    "            database_type=self.dataset_type, \n",
    "            eeg1_f1=eeg1_f1, \n",
    "            eeg1_D=eeg1_D,\n",
    "            eeg1_kernel_size=eeg1_kernel_size,\n",
    "            eeg1_pooling_size1 = eeg1_pooling_size1,\n",
    "            eeg1_pooling_size2 = eeg1_pooling_size2,\n",
    "            eeg1_dropout_rate = eeg1_dropout_rate,\n",
    "            eeg1_number_channel = self.number_channel,\n",
    "            flatten_eeg1 = flatten_eeg1,  \n",
    "            ).cuda()\n",
    "        #self.model = nn.DataParallel(self.model, device_ids=gpus)\n",
    "        self.model = self.model.cuda()\n",
    "        self.model_filename = self.result_name + '/model_{}.pth'.format(self.nSub)\n",
    "        \n",
    "\n",
    "\n",
    "    # Segmentation and Reconstruction (S&R) data augmentation\n",
    "    def interaug(self, timg, label):  \n",
    "        aug_data = []\n",
    "        aug_label = []\n",
    "        \n",
    "        number_segmentation_points = 1000 // self.number_seg\n",
    "        for clsAug in range(self.number_class):\n",
    "            cls_idx = np.where(label == clsAug + 1)\n",
    "            tmp_data = timg[cls_idx]\n",
    "            number_records_by_augmentation = self.number_augmentation * tmp_data.shape[0]\n",
    "            tmp_aug_data = np.zeros((number_records_by_augmentation, 1, self.number_channel, 1000))\n",
    "            for ri in range(number_records_by_augmentation):\n",
    "                for rj in range(self.number_seg):\n",
    "                    rand_idx = np.random.randint(0, tmp_data.shape[0], self.number_seg)\n",
    "                    tmp_aug_data[ri, :, :, rj * number_segmentation_points:(rj + 1) * number_segmentation_points] = \\\n",
    "                        tmp_data[rand_idx[rj], :, :, rj * number_segmentation_points:(rj + 1) * number_segmentation_points]\n",
    "\n",
    "            aug_data.append(tmp_aug_data)\n",
    "            aug_label.append([clsAug + 1]*number_records_by_augmentation)\n",
    "        aug_data = np.concatenate(aug_data)\n",
    "        aug_label = np.concatenate(aug_label)\n",
    "        # aug_shuffle = np.random.permutation(len(aug_data))\n",
    "        # aug_data = aug_data[aug_shuffle, :, :]\n",
    "        # aug_label = aug_label[aug_shuffle]\n",
    "\n",
    "        # aug_data = torch.from_numpy(aug_data).cuda()\n",
    "        # aug_data = aug_data.float()\n",
    "        # aug_label = torch.from_numpy(aug_label-1).cuda()\n",
    "        # aug_label = aug_label.long()\n",
    "        return aug_data, aug_label\n",
    "\n",
    "\n",
    "\n",
    "    def get_source_data(self):\n",
    "        (self.train_data,    # (batch, channel, length)\n",
    "         self.train_label, \n",
    "         self.test_data, \n",
    "         self.test_label) = load_data_evaluate(self.root, self.dataset_type, self.nSub, mode_evaluate=self.evaluate_mode)\n",
    "\n",
    "        self.train_data = np.expand_dims(self.train_data, axis=1)  # (288, 1, 22, 1000)\n",
    "        self.train_label = np.transpose(self.train_label) \n",
    "        \n",
    "        self.allData = self.train_data\n",
    "        self.allLabel = self.train_label[0]  \n",
    "        # split original allData into training and validate datasets\n",
    "\n",
    "        train_data_list = []\n",
    "        train_label_list = []\n",
    "        validate_data_list = []\n",
    "        validate_label_list = []\n",
    "        for cls in range(self.number_class):\n",
    "            # filter by class \n",
    "            cls_idx = np.where(self.allLabel == cls + 1)\n",
    "            cat_data = self.allData[cls_idx]\n",
    "            cat_label = self.allLabel[cls_idx]\n",
    "\n",
    "            \n",
    "            # each category split\n",
    "            number_sample = cat_data.shape[0]\n",
    "            number_validate = int(self.validate_ratio * number_sample)\n",
    "            # shuffle index\n",
    "            index_shuffle = np.random.permutation(len(cat_data))\n",
    "            index_train = index_shuffle[:-number_validate]\n",
    "            index_validate = index_shuffle[-number_validate:]\n",
    "            \n",
    "            train_data_list.append(cat_data[index_train])\n",
    "            train_label_list.append(cat_label[index_train])\n",
    "            \n",
    "            validate_data_list.append(cat_data[index_validate])\n",
    "            validate_label_list.append(cat_label[index_validate])\n",
    "        \n",
    "        # data augmentation\n",
    "        aug_data, aug_label = self.interaug(self.allData, self.allLabel)\n",
    "            \n",
    "        train_data_list.append(aug_data)\n",
    "        train_label_list.append(aug_label)\n",
    "            \n",
    "        self.trainData = np.concatenate(train_data_list)\n",
    "        self.trainLabel = np.concatenate(train_label_list)\n",
    "        self.validateData = np.concatenate(validate_data_list)\n",
    "        self.validateLabel = np.concatenate(validate_label_list)\n",
    "        \n",
    "        # shuffle in all category\n",
    "        shuffle_num = np.random.permutation(len(self.trainData))\n",
    "        self.trainData = self.trainData[shuffle_num, :, :, :]  # (number of training sample, 1, 22, 1000)\n",
    "        self.trainLabel = self.trainLabel[shuffle_num]\n",
    "\n",
    "        # self.test_data = np.transpose(self.test_data, (2, 1, 0))\n",
    "        self.test_data = np.expand_dims(self.test_data, axis=1)\n",
    "        self.test_label = np.transpose(self.test_label)\n",
    "\n",
    "        self.testData = self.test_data\n",
    "        self.testLabel = self.test_label[0]\n",
    "\n",
    "        # standardize\n",
    "        target_mean = np.mean(self.allData)\n",
    "        target_std = np.std(self.allData)\n",
    "        # self.allData = (self.allData - target_mean) / target_std\n",
    "        self.trainData = (self.trainData - target_mean) / target_std\n",
    "        self.validateData = (self.validateData - target_mean) / target_std\n",
    "        self.testData = (self.testData - target_mean) / target_std\n",
    "        \n",
    "        isSaveDataLabel = False #True\n",
    "        if isSaveDataLabel:\n",
    "            np.save(\"./gradm_data/train_data_{}.npy\".format(self.nSub), self.allData)\n",
    "            np.save(\"./gradm_data/train_lable_{}.npy\".format(self.nSub), self.allLabel)\n",
    "            np.save(\"./gradm_data/test_data_{}.npy\".format(self.nSub), self.testData)\n",
    "            np.save(\"./gradm_data/test_label_{}.npy\".format(self.nSub), self.testLabel)\n",
    "        print(self.trainData.shape, self.trainLabel.shape, self.validateData.shape, self.validateLabel.shape, self.testData.shape, self.testLabel.shape)\n",
    "        # data shape: (trial, conv channel, electrode channel, time samples)\n",
    "        return self.trainData, self.trainLabel, self.validateData, self.validateLabel, self.testData, self.testLabel\n",
    "\n",
    "\n",
    "\n",
    "    def fit_test(self, model, loss_fn, testloader):\n",
    "        y_list = []\n",
    "        y_pred_list = []\n",
    "\n",
    "        test_correct = 0\n",
    "        test_total = 0\n",
    "        test_running_loss = 0\n",
    "        model.eval()  \n",
    "        with torch.no_grad():\n",
    "            for x, y in testloader:\n",
    "                x = Variable(x.type(self.Tensor))\n",
    "                y = Variable(y.type(self.LongTensor))\n",
    "                \n",
    "                features, y_pred = model(x)\n",
    "                loss = loss_fn(y_pred, y)\n",
    "                y_pred = torch.argmax(y_pred, dim=1)\n",
    "                test_correct += (y_pred == y).sum().item()\n",
    "                test_total += y.size(0)\n",
    "                test_running_loss += loss.item()\n",
    "                y_pred = y_pred.cpu().numpy()\n",
    "                y = y.cpu().numpy()\n",
    "                y_list.extend(y)  \n",
    "                y_pred_list.extend(y_pred)  \n",
    "\n",
    "        acc_score = accuracy_score(y_list, y_pred_list)\n",
    "        epoch_test_loss = test_running_loss / len(testloader.dataset)\n",
    "\n",
    "        return epoch_test_loss, acc_score, y_list, y_pred_list\n",
    "    \n",
    "    \n",
    "    def fit_train(self, model, loss_fn, dataloader, optimizer, trainData, trainLabel):\n",
    "        correct = 0\n",
    "        total = 0\n",
    "        running_loss = 0\n",
    "        model.train()\n",
    "\n",
    "        for train_data, train_label in dataloader:\n",
    "            # real train dataset\n",
    "            img = Variable(train_data.type(self.Tensor))\n",
    "            label = Variable(train_label.type(self.LongTensor))\n",
    "\n",
    "\n",
    "            # training model\n",
    "            features, y_pred = model(img)\n",
    "            # print(\"train outputs: \", outputs.shape, type(outputs))\n",
    "            # print(features.size())\n",
    "            loss = loss_fn(y_pred, label) \n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            with torch.no_grad():\n",
    "                y_pred = torch.argmax(y_pred, dim=1)\n",
    "                correct += (y_pred == label).sum().item()\n",
    "                total += label.size(0)\n",
    "                running_loss += loss.item()\n",
    "\n",
    "        epoch_train_loss = running_loss / len(dataloader.dataset)\n",
    "        epoch_train_acc = correct / total\n",
    "\n",
    "        return epoch_train_loss, epoch_train_acc\n",
    "    \n",
    "    \n",
    "    def train(self):\n",
    "        img, label, validate_data, validate_label, test_data, test_label = self.get_source_data()\n",
    "        # train dataset\n",
    "        img = torch.from_numpy(img)\n",
    "        label = torch.from_numpy(label - 1)\n",
    "        dataset = torch.utils.data.TensorDataset(img, label)\n",
    "        self.dataloader = torch.utils.data.DataLoader(dataset=dataset, batch_size=self.batch_size, shuffle=True)\n",
    "        # validate dataset\n",
    "        validate_data = torch.from_numpy(validate_data)\n",
    "        validate_label = torch.from_numpy(validate_label - 1)\n",
    "        validate_dataset = torch.utils.data.TensorDataset(validate_data, validate_label)\n",
    "\n",
    "        self.validate_dataloader = torch.utils.data.DataLoader(dataset=validate_dataset, batch_size=288, shuffle=False)\n",
    "        # test dataset\n",
    "        test_data = torch.from_numpy(test_data)\n",
    "        test_label = torch.from_numpy(test_label - 1)\n",
    "        test_dataset = torch.utils.data.TensorDataset(test_data, test_label)\n",
    "        self.test_dataloader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=288, shuffle=False)\n",
    "\n",
    "        # Optimizers\n",
    "        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.lr, betas=(self.b1, self.b2))\n",
    "\n",
    "        test_data = Variable(test_data.type(self.Tensor))\n",
    "        test_label = Variable(test_label.type(self.LongTensor))\n",
    "        best_epoch = 0\n",
    "        num = 0\n",
    "        min_loss = 100\n",
    "        # recording train_acc, train_loss, test_acc, test_loss\n",
    "        result_process = []\n",
    "        # Train the CTNet model\n",
    "        for e in range(self.n_epochs):\n",
    "            epoch_process = {}\n",
    "            epoch_process['epoch'] = e\n",
    "            # train model\n",
    "            self.model.train()\n",
    "            train_loss, train_acc = self.fit_train(self.model, self.criterion_cls, self.dataloader, self.optimizer, self.allData, self.allLabel)\n",
    "            epoch_process['train_acc'] = train_acc\n",
    "            epoch_process['train_loss'] = train_loss\n",
    "            \n",
    "            # validate model\n",
    "            (validate_loss, \n",
    "             validate_acc, \n",
    "             y_list, \n",
    "             y_pred_list) = self.fit_test(self.model, self.criterion_cls, self.validate_dataloader)\n",
    "            epoch_process['val_acc'] = validate_acc                \n",
    "            epoch_process['val_loss'] = validate_loss\n",
    "\n",
    "#             train_pred = torch.max(outputs, 1)[1]\n",
    "#             train_acc = float((train_pred == label).cpu().numpy().astype(int).sum()) / float(label.size(0))\n",
    "            num = num + 1\n",
    "\n",
    "            if min_loss>validate_loss:\n",
    "                min_loss = validate_loss\n",
    "                best_epoch = e\n",
    "                epoch_process['epoch'] = e\n",
    "                torch.save(self.model, self.model_filename)\n",
    "\n",
    "                (test_loss, \n",
    "                 test_acc, \n",
    "                 y_list, \n",
    "                 y_pred_list) = self.fit_test(self.model, self.criterion_cls, self.test_dataloader)\n",
    "                epoch_process['test_acc'] = test_acc                \n",
    "                epoch_process['test_loss'] = test_loss\n",
    "                print(\"{}_{} train_acc: {:.4f} train_loss: {:.6f}\\tval_acc: {:.6f} val_loss: {:.9f}, acc:{:.6f}\".format(self.nSub,\n",
    "                                                                                       epoch_process['epoch'],\n",
    "                                                                                       epoch_process['train_acc'],\n",
    "                                                                                       epoch_process['train_loss'],\n",
    "                                                                                       epoch_process['val_acc'],\n",
    "                                                                                       epoch_process['val_loss'],\n",
    "                                                                                       epoch_process['test_acc']\n",
    "                                                                                    ))\n",
    "            \n",
    "                \n",
    "            result_process.append(epoch_process)  \n",
    "        \n",
    "        # load model for test\n",
    "        self.model.eval()\n",
    "        self.model = torch.load(self.model_filename).cuda()\n",
    "        # test model\n",
    "        (test_loss, \n",
    "         test_acc, \n",
    "         y_list, \n",
    "         y_pred_list) = self.fit_test(self.model, self.criterion_cls, self.test_dataloader)\n",
    "\n",
    "        print(\"epoch: \", best_epoch, '\\tThe test accuracy is:', test_acc)\n",
    "\n",
    "\n",
    "        df_process = pd.DataFrame(result_process)\n",
    "\n",
    "        return test_acc, torch.tensor(y_list), torch.tensor(y_pred_list), df_process, best_epoch\n",
    "        # writer.close()\n",
    "        \n",
    "\n",
    "\n",
    "\n",
    "def main(dirs,                \n",
    "         evaluate_mode = 'subject-dependent', # LOSO or not\n",
    "         heads=8,             # heads of MHA\n",
    "         emb_size=48,         # token embding dim\n",
    "         depth=3,             # Transformer encoder depth\n",
    "         dataset_type='A',    # A->'BCI IV2a', B->'BCI IV2b'\n",
    "         eeg1_f1=20,          # features of temporal conv\n",
    "         eeg1_kernel_size=64, # kernel size of temporal conv\n",
    "         eeg1_D=2,            # depth-wise conv \n",
    "         eeg1_pooling_size1=8,# p1\n",
    "         eeg1_pooling_size2=8,# p2\n",
    "         eeg1_dropout_rate=0.3,\n",
    "         flatten_eeg1=600,   \n",
    "         validate_ratio = 0.2,\n",
    "         batch_size = 72\n",
    "         ):\n",
    "\n",
    "    if not os.path.exists(dirs):\n",
    "        os.makedirs(dirs)\n",
    "\n",
    "    result_write_metric = ExcelWriter(dirs+\"/result_metric.xlsx\")\n",
    "    \n",
    "    result_metric_dict = {}\n",
    "    y_true_pred_dict = { }\n",
    "\n",
    "    process_write = ExcelWriter(dirs+\"/process_train.xlsx\")\n",
    "    pred_true_write = ExcelWriter(dirs+\"/pred_true.xlsx\")\n",
    "    subjects_result = []\n",
    "    best_epochs = []\n",
    "    \n",
    "    for i in range(0, N_SUBJECT):      \n",
    "        \n",
    "        starttime = datetime.datetime.now()\n",
    "        seed_n = np.random.randint(2024)\n",
    "        print('seed is ' + str(seed_n))\n",
    "        random.seed(seed_n)\n",
    "        np.random.seed(seed_n)\n",
    "        torch.manual_seed(seed_n)\n",
    "        torch.cuda.manual_seed(seed_n)\n",
    "        torch.cuda.manual_seed_all(seed_n)\n",
    "        index_round =0\n",
    "        print('Subject %d' % (i+1))\n",
    "        exp = ExP(i + 1, DATA_DIR, dirs, EPOCHS, N_AUG, N_SEG, gpus, \n",
    "                  evaluate_mode = evaluate_mode,\n",
    "                  heads=heads, \n",
    "                  emb_size=emb_size,\n",
    "                  depth=depth, \n",
    "                  dataset_type=dataset_type,\n",
    "                  eeg1_f1 = eeg1_f1,\n",
    "                  eeg1_kernel_size = eeg1_kernel_size,\n",
    "                  eeg1_D = eeg1_D,\n",
    "                  eeg1_pooling_size1 = eeg1_pooling_size1,\n",
    "                  eeg1_pooling_size2 = eeg1_pooling_size2,\n",
    "                  eeg1_dropout_rate = eeg1_dropout_rate,\n",
    "                  flatten_eeg1 = flatten_eeg1,  \n",
    "                  validate_ratio = validate_ratio,\n",
    "                  batch_size = batch_size \n",
    "                  )\n",
    "\n",
    "        testAcc, Y_true, Y_pred, df_process, best_epoch = exp.train()\n",
    "        true_cpu = Y_true.cpu().numpy().astype(int)\n",
    "        pred_cpu = Y_pred.cpu().numpy().astype(int)\n",
    "        df_pred_true = pd.DataFrame({'pred': pred_cpu, 'true': true_cpu})\n",
    "        df_pred_true.to_excel(pred_true_write, sheet_name=str(i+1))\n",
    "        y_true_pred_dict[i] = df_pred_true\n",
    "\n",
    "        accuracy, precison, recall, f1, kappa = calMetrics(true_cpu, pred_cpu)\n",
    "        subject_result = {'accuray': accuracy*100,\n",
    "                          'precision': precison*100,\n",
    "                          'recall': recall*100,\n",
    "                          'f1': f1*100, \n",
    "                          'kappa': kappa*100\n",
    "                          }\n",
    "        subjects_result.append(subject_result)\n",
    "        df_process.to_excel(process_write, sheet_name=str(i+1))\n",
    "        best_epochs.append(best_epoch)\n",
    "    \n",
    "        print(' THE BEST ACCURACY IS ' + str(testAcc) + \"\\tkappa is \" + str(kappa) )\n",
    "    \n",
    "\n",
    "        endtime = datetime.datetime.now()\n",
    "        print('subject %d duration: '%(i+1) + str(endtime - starttime))\n",
    "\n",
    "        if i == 0:\n",
    "            yt = Y_true\n",
    "            yp = Y_pred\n",
    "        else:\n",
    "            yt = torch.cat((yt, Y_true))\n",
    "            yp = torch.cat((yp, Y_pred))\n",
    "                \n",
    "        df_result = pd.DataFrame(subjects_result)\n",
    "    process_write.close()\n",
    "    pred_true_write.close()\n",
    "\n",
    "\n",
    "    print('**The average Best accuracy is: ' + str(df_result['accuray'].mean()) + \"kappa is: \" + str(df_result['kappa'].mean()) + \"\\n\" )\n",
    "    print(\"best epochs: \", best_epochs)\n",
    "    #df_result.to_excel(result_write_metric, index=False)\n",
    "    result_metric_dict = df_result\n",
    "\n",
    "    mean = df_result.mean(axis=0)\n",
    "    mean.name = 'mean'\n",
    "    std = df_result.std(axis=0)\n",
    "    std.name = 'std'\n",
    "    df_result = pd.concat([df_result, pd.DataFrame(mean).T, pd.DataFrame(std).T])\n",
    "    \n",
    "    df_result.to_excel(result_write_metric, index=False)\n",
    "    print('-'*9, ' all result ', '-'*9)\n",
    "    print(df_result)\n",
    "    \n",
    "    print(\"*\"*40)\n",
    "\n",
    "    result_write_metric.close()\n",
    "\n",
    "    \n",
    "    return result_metric_dict\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    #----------------------------------------\n",
    "    DATA_DIR = r'../mymat_raw/'\n",
    "    EVALUATE_MODE = 'LOSO' # leaving one subject out subject-dependent  subject-indenpedent\n",
    "\n",
    "    N_SUBJECT = 9       # BCI \n",
    "    N_AUG = 3           # data augmentation times for benerating artificial training data set\n",
    "    N_SEG = 8           # segmentation times for S&R\n",
    "\n",
    "    EPOCHS = 600\n",
    "    EMB_DIM = 16\n",
    "    HEADS = 2\n",
    "    DEPTH = 6\n",
    "    TYPE = 'B'\n",
    "    validate_ratio = 0.3 # split raw train dataset into real train dataset and validate dataset\n",
    "    BATCH_SIZE = 512\n",
    "    EEGNet1_F1 = 8\n",
    "    EEGNet1_KERNEL_SIZE=64\n",
    "    EEGNet1_D=2\n",
    "    EEGNet1_POOL_SIZE1 = 8\n",
    "    EEGNet1_POOL_SIZE2 = 8\n",
    "    FLATTEN_EEGNet1 = 240\n",
    "\n",
    "    if EVALUATE_MODE!='LOSO':\n",
    "        EEGNet1_DROPOUT_RATE = 0.5\n",
    "    else:\n",
    "        EEGNet1_DROPOUT_RATE = 0.25    \n",
    "\n",
    "    \n",
    "    parameters_list = [0, 1, 2]\n",
    "    for i in parameters_list:\n",
    "        number_class, number_channel = numberClassChannel(TYPE)\n",
    "        RESULT_NAME = \"Loso_{}_heads_{}_depth_{}_{}\".format(TYPE, HEADS, DEPTH, i)\n",
    "    \n",
    "        sModel = EEGTransformer(\n",
    "            heads=HEADS, \n",
    "            emb_size=EMB_DIM,\n",
    "            depth=DEPTH, \n",
    "            database_type=TYPE,\n",
    "            eeg1_f1=EEGNet1_F1, \n",
    "            eeg1_D=EEGNet1_D,\n",
    "            eeg1_kernel_size=EEGNet1_KERNEL_SIZE,\n",
    "            eeg1_pooling_size1 = EEGNet1_POOL_SIZE1,\n",
    "            eeg1_pooling_size2 = EEGNet1_POOL_SIZE2,\n",
    "            eeg1_dropout_rate = EEGNet1_DROPOUT_RATE,\n",
    "            eeg1_number_channel = number_channel,\n",
    "            flatten_eeg1 = FLATTEN_EEGNet1,  \n",
    "            ).cuda()\n",
    "        summary(sModel, (1, number_channel, 1000)) \n",
    "    \n",
    "        print(time.asctime(time.localtime(time.time())))\n",
    "        \n",
    "        result = main(RESULT_NAME,\n",
    "                        evaluate_mode = EVALUATE_MODE,\n",
    "                        heads=HEADS, \n",
    "                        emb_size=EMB_DIM,\n",
    "                        depth=DEPTH, \n",
    "                        dataset_type=TYPE,\n",
    "                        eeg1_f1 = EEGNet1_F1,\n",
    "                        eeg1_kernel_size = EEGNet1_KERNEL_SIZE,\n",
    "                        eeg1_D = EEGNet1_D,\n",
    "                        eeg1_pooling_size1 = EEGNet1_POOL_SIZE1,\n",
    "                        eeg1_pooling_size2 = EEGNet1_POOL_SIZE2,\n",
    "                        eeg1_dropout_rate = EEGNet1_DROPOUT_RATE,\n",
    "                        flatten_eeg1 = FLATTEN_EEGNet1,\n",
    "                        validate_ratio = validate_ratio,\n",
    "                        batch_size = BATCH_SIZE\n",
    "                      )\n",
    "        print(time.asctime(time.localtime(time.time())))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}