Merge pull request #36 from neurodata/sailing

Switch from accuracy to Cohen's Kappa for vision & auditory data
neurodata · Sep 23, 2021 · 97ae251 · 97ae251
2 parents 98a8bb3 + 1dd00a9
commit 97ae251
Show file tree

Hide file tree

Showing 309 changed files with 504 additions and 859 deletions.
diff --git a/audio/3_class/cnn32.txt → benchmark_audition/3_class/cnn32.txt b/audio/3_class/cnn32.txt → benchmark_audition/3_class/cnn32.txt
diff --git a/audio/3_class/cnn32_2l.txt → benchmark_audition/3_class/cnn32_2l.txt b/audio/3_class/cnn32_2l.txt → benchmark_audition/3_class/cnn32_2l.txt
diff --git a/audio/3_class/cnn32_2l_test_time.txt → ...k_audition/3_class/cnn32_2l_test_time.txt b/audio/3_class/cnn32_2l_test_time.txt → ...k_audition/3_class/cnn32_2l_test_time.txt
diff --git a/audio/3_class/cnn32_2l_train_time.txt → ..._audition/3_class/cnn32_2l_train_time.txt b/audio/3_class/cnn32_2l_train_time.txt → ..._audition/3_class/cnn32_2l_train_time.txt
diff --git a/audio/3_class/cnn32_5l.txt → benchmark_audition/3_class/cnn32_5l.txt b/audio/3_class/cnn32_5l.txt → benchmark_audition/3_class/cnn32_5l.txt
diff --git a/audio/3_class/cnn32_5l_test_time.txt → ...k_audition/3_class/cnn32_5l_test_time.txt b/audio/3_class/cnn32_5l_test_time.txt → ...k_audition/3_class/cnn32_5l_test_time.txt
diff --git a/audio/3_class/cnn32_5l_train_time.txt → ..._audition/3_class/cnn32_5l_train_time.txt b/audio/3_class/cnn32_5l_train_time.txt → ..._audition/3_class/cnn32_5l_train_time.txt
diff --git a/audio/3_class/cnn32_test_time.txt → ...mark_audition/3_class/cnn32_test_time.txt b/audio/3_class/cnn32_test_time.txt → ...mark_audition/3_class/cnn32_test_time.txt
diff --git a/audio/3_class/cnn32_train_time.txt → ...ark_audition/3_class/cnn32_train_time.txt b/audio/3_class/cnn32_train_time.txt → ...ark_audition/3_class/cnn32_train_time.txt
diff --git a/audio/3_class/naive_rf.txt → benchmark_audition/3_class/naive_rf.txt b/audio/3_class/naive_rf.txt → benchmark_audition/3_class/naive_rf.txt
diff --git a/audio/3_class/naive_rf_test_time.txt → ...k_audition/3_class/naive_rf_test_time.txt b/audio/3_class/naive_rf_test_time.txt → ...k_audition/3_class/naive_rf_test_time.txt
diff --git a/audio/3_class/naive_rf_train_time.txt → ..._audition/3_class/naive_rf_train_time.txt b/audio/3_class/naive_rf_train_time.txt → ..._audition/3_class/naive_rf_train_time.txt
diff --git a/audio/3_class/resnet18.txt → benchmark_audition/3_class/resnet18.txt b/audio/3_class/resnet18.txt → benchmark_audition/3_class/resnet18.txt
diff --git a/audio/3_class/resnet18_test_time.txt → ...k_audition/3_class/resnet18_test_time.txt b/audio/3_class/resnet18_test_time.txt → ...k_audition/3_class/resnet18_test_time.txt
diff --git a/audio/3_class/resnet18_train_time.txt → ..._audition/3_class/resnet18_train_time.txt b/audio/3_class/resnet18_train_time.txt → ..._audition/3_class/resnet18_train_time.txt
diff --git a/audio/3_class/svm.txt → benchmark_audition/3_class/svm.txt b/audio/3_class/svm.txt → benchmark_audition/3_class/svm.txt
diff --git a/audio/3_class/svm_test_time.txt → benchmark_audition/3_class/svm_test_time.txt b/audio/3_class/svm_test_time.txt → benchmark_audition/3_class/svm_test_time.txt
diff --git a/audio/3_class/svm_train_time.txt → ...hmark_audition/3_class/svm_train_time.txt b/audio/3_class/svm_train_time.txt → ...hmark_audition/3_class/svm_train_time.txt
diff --git a/audio/3_class_mel/cnn32.txt → benchmark_audition/3_class_mel/cnn32.txt b/audio/3_class_mel/cnn32.txt → benchmark_audition/3_class_mel/cnn32.txt
diff --git a/audio/3_class_mel/cnn32_2l.txt → benchmark_audition/3_class_mel/cnn32_2l.txt b/audio/3_class_mel/cnn32_2l.txt → benchmark_audition/3_class_mel/cnn32_2l.txt
diff --git a/audio/3_class_mel/cnn32_2l_test_time.txt → ...dition/3_class_mel/cnn32_2l_test_time.txt b/audio/3_class_mel/cnn32_2l_test_time.txt → ...dition/3_class_mel/cnn32_2l_test_time.txt
diff --git a/audio/3_class_mel/cnn32_2l_train_time.txt → ...ition/3_class_mel/cnn32_2l_train_time.txt b/audio/3_class_mel/cnn32_2l_train_time.txt → ...ition/3_class_mel/cnn32_2l_train_time.txt
diff --git a/audio/3_class_mel/cnn32_5l.txt → benchmark_audition/3_class_mel/cnn32_5l.txt b/audio/3_class_mel/cnn32_5l.txt → benchmark_audition/3_class_mel/cnn32_5l.txt
diff --git a/audio/3_class_mel/cnn32_5l_test_time.txt → ...dition/3_class_mel/cnn32_5l_test_time.txt b/audio/3_class_mel/cnn32_5l_test_time.txt → ...dition/3_class_mel/cnn32_5l_test_time.txt
diff --git a/audio/3_class_mel/cnn32_5l_train_time.txt → ...ition/3_class_mel/cnn32_5l_train_time.txt b/audio/3_class_mel/cnn32_5l_train_time.txt → ...ition/3_class_mel/cnn32_5l_train_time.txt
diff --git a/audio/3_class_mel/cnn32_test_time.txt → ..._audition/3_class_mel/cnn32_test_time.txt b/audio/3_class_mel/cnn32_test_time.txt → ..._audition/3_class_mel/cnn32_test_time.txt
diff --git a/audio/3_class_mel/cnn32_train_time.txt → ...audition/3_class_mel/cnn32_train_time.txt b/audio/3_class_mel/cnn32_train_time.txt → ...audition/3_class_mel/cnn32_train_time.txt
diff --git a/audio/3_class_mel/naive_rf.txt → benchmark_audition/3_class_mel/naive_rf.txt b/audio/3_class_mel/naive_rf.txt → benchmark_audition/3_class_mel/naive_rf.txt
diff --git a/audio/3_class_mel/naive_rf_test_time.txt → ...dition/3_class_mel/naive_rf_test_time.txt b/audio/3_class_mel/naive_rf_test_time.txt → ...dition/3_class_mel/naive_rf_test_time.txt
diff --git a/audio/3_class_mel/naive_rf_train_time.txt → ...ition/3_class_mel/naive_rf_train_time.txt b/audio/3_class_mel/naive_rf_train_time.txt → ...ition/3_class_mel/naive_rf_train_time.txt
diff --git a/audio/3_class_mel/resnet18.txt → benchmark_audition/3_class_mel/resnet18.txt b/audio/3_class_mel/resnet18.txt → benchmark_audition/3_class_mel/resnet18.txt
diff --git a/audio/3_class_mel/resnet18_test_time.txt → ...dition/3_class_mel/resnet18_test_time.txt b/audio/3_class_mel/resnet18_test_time.txt → ...dition/3_class_mel/resnet18_test_time.txt
diff --git a/audio/3_class_mel/resnet18_train_time.txt → ...ition/3_class_mel/resnet18_train_time.txt b/audio/3_class_mel/resnet18_train_time.txt → ...ition/3_class_mel/resnet18_train_time.txt
diff --git a/audio/3_class_mel/svm.txt → benchmark_audition/3_class_mel/svm.txt b/audio/3_class_mel/svm.txt → benchmark_audition/3_class_mel/svm.txt
diff --git a/audio/3_class_mel/svm_test_time.txt → ...rk_audition/3_class_mel/svm_test_time.txt b/audio/3_class_mel/svm_test_time.txt → ...rk_audition/3_class_mel/svm_test_time.txt
diff --git a/audio/3_class_mel/svm_train_time.txt → ...k_audition/3_class_mel/svm_train_time.txt b/audio/3_class_mel/svm_train_time.txt → ...k_audition/3_class_mel/svm_train_time.txt
diff --git a/audio/3_class_mfcc/cnn32.txt → benchmark_audition/3_class_mfcc/cnn32.txt b/audio/3_class_mfcc/cnn32.txt → benchmark_audition/3_class_mfcc/cnn32.txt
diff --git a/audio/3_class_mfcc/cnn32_2l.txt → benchmark_audition/3_class_mfcc/cnn32_2l.txt b/audio/3_class_mfcc/cnn32_2l.txt → benchmark_audition/3_class_mfcc/cnn32_2l.txt
diff --git a/audio/3_class_mfcc/cnn32_2l_test_time.txt → ...ition/3_class_mfcc/cnn32_2l_test_time.txt b/audio/3_class_mfcc/cnn32_2l_test_time.txt → ...ition/3_class_mfcc/cnn32_2l_test_time.txt
diff --git a/audio/3_class_mfcc/cnn32_2l_train_time.txt → ...tion/3_class_mfcc/cnn32_2l_train_time.txt b/audio/3_class_mfcc/cnn32_2l_train_time.txt → ...tion/3_class_mfcc/cnn32_2l_train_time.txt
diff --git a/audio/3_class_mfcc/cnn32_5l.txt → benchmark_audition/3_class_mfcc/cnn32_5l.txt b/audio/3_class_mfcc/cnn32_5l.txt → benchmark_audition/3_class_mfcc/cnn32_5l.txt
diff --git a/audio/3_class_mfcc/cnn32_5l_test_time.txt → ...ition/3_class_mfcc/cnn32_5l_test_time.txt b/audio/3_class_mfcc/cnn32_5l_test_time.txt → ...ition/3_class_mfcc/cnn32_5l_test_time.txt
diff --git a/audio/3_class_mfcc/cnn32_5l_train_time.txt → ...tion/3_class_mfcc/cnn32_5l_train_time.txt b/audio/3_class_mfcc/cnn32_5l_train_time.txt → ...tion/3_class_mfcc/cnn32_5l_train_time.txt
diff --git a/audio/3_class_mfcc/cnn32_test_time.txt → ...audition/3_class_mfcc/cnn32_test_time.txt b/audio/3_class_mfcc/cnn32_test_time.txt → ...audition/3_class_mfcc/cnn32_test_time.txt
diff --git a/audio/3_class_mfcc/cnn32_train_time.txt → ...udition/3_class_mfcc/cnn32_train_time.txt b/audio/3_class_mfcc/cnn32_train_time.txt → ...udition/3_class_mfcc/cnn32_train_time.txt
diff --git a/audio/3_class_mfcc/naive_rf.txt → benchmark_audition/3_class_mfcc/naive_rf.txt b/audio/3_class_mfcc/naive_rf.txt → benchmark_audition/3_class_mfcc/naive_rf.txt
diff --git a/audio/3_class_mfcc/naive_rf_test_time.txt → ...ition/3_class_mfcc/naive_rf_test_time.txt b/audio/3_class_mfcc/naive_rf_test_time.txt → ...ition/3_class_mfcc/naive_rf_test_time.txt
diff --git a/audio/3_class_mfcc/naive_rf_train_time.txt → ...tion/3_class_mfcc/naive_rf_train_time.txt b/audio/3_class_mfcc/naive_rf_train_time.txt → ...tion/3_class_mfcc/naive_rf_train_time.txt
diff --git a/audio/3_class_mfcc/resnet18.txt → benchmark_audition/3_class_mfcc/resnet18.txt b/audio/3_class_mfcc/resnet18.txt → benchmark_audition/3_class_mfcc/resnet18.txt
diff --git a/audio/3_class_mfcc/resnet18_test_time.txt → ...ition/3_class_mfcc/resnet18_test_time.txt b/audio/3_class_mfcc/resnet18_test_time.txt → ...ition/3_class_mfcc/resnet18_test_time.txt
diff --git a/audio/3_class_mfcc/resnet18_train_time.txt → ...tion/3_class_mfcc/resnet18_train_time.txt b/audio/3_class_mfcc/resnet18_train_time.txt → ...tion/3_class_mfcc/resnet18_train_time.txt
diff --git a/audio/3_class_mfcc/svm.txt → benchmark_audition/3_class_mfcc/svm.txt b/audio/3_class_mfcc/svm.txt → benchmark_audition/3_class_mfcc/svm.txt
diff --git a/audio/3_class_mfcc/svm_test_time.txt → ...k_audition/3_class_mfcc/svm_test_time.txt b/audio/3_class_mfcc/svm_test_time.txt → ...k_audition/3_class_mfcc/svm_test_time.txt
diff --git a/audio/3_class_mfcc/svm_train_time.txt → ..._audition/3_class_mfcc/svm_train_time.txt b/audio/3_class_mfcc/svm_train_time.txt → ..._audition/3_class_mfcc/svm_train_time.txt
diff --git a/audio/8_class/cnn32.txt → benchmark_audition/8_class/cnn32.txt b/audio/8_class/cnn32.txt → benchmark_audition/8_class/cnn32.txt
diff --git a/audio/8_class/cnn32_2l.txt → benchmark_audition/8_class/cnn32_2l.txt b/audio/8_class/cnn32_2l.txt → benchmark_audition/8_class/cnn32_2l.txt
diff --git a/audio/8_class/cnn32_2l_test_time.txt → ...k_audition/8_class/cnn32_2l_test_time.txt b/audio/8_class/cnn32_2l_test_time.txt → ...k_audition/8_class/cnn32_2l_test_time.txt
diff --git a/audio/8_class/cnn32_2l_train_time.txt → ..._audition/8_class/cnn32_2l_train_time.txt b/audio/8_class/cnn32_2l_train_time.txt → ..._audition/8_class/cnn32_2l_train_time.txt
diff --git a/audio/8_class/cnn32_5l.txt → benchmark_audition/8_class/cnn32_5l.txt b/audio/8_class/cnn32_5l.txt → benchmark_audition/8_class/cnn32_5l.txt
diff --git a/audio/8_class/cnn32_5l_test_time.txt → ...k_audition/8_class/cnn32_5l_test_time.txt b/audio/8_class/cnn32_5l_test_time.txt → ...k_audition/8_class/cnn32_5l_test_time.txt
diff --git a/audio/8_class/cnn32_5l_train_time.txt → ..._audition/8_class/cnn32_5l_train_time.txt b/audio/8_class/cnn32_5l_train_time.txt → ..._audition/8_class/cnn32_5l_train_time.txt
diff --git a/audio/8_class/cnn32_test_time.txt → ...mark_audition/8_class/cnn32_test_time.txt b/audio/8_class/cnn32_test_time.txt → ...mark_audition/8_class/cnn32_test_time.txt
diff --git a/audio/8_class/cnn32_train_time.txt → ...ark_audition/8_class/cnn32_train_time.txt b/audio/8_class/cnn32_train_time.txt → ...ark_audition/8_class/cnn32_train_time.txt
diff --git a/audio/8_class/naive_rf.txt → benchmark_audition/8_class/naive_rf.txt b/audio/8_class/naive_rf.txt → benchmark_audition/8_class/naive_rf.txt
diff --git a/audio/8_class/naive_rf_test_time.txt → ...k_audition/8_class/naive_rf_test_time.txt b/audio/8_class/naive_rf_test_time.txt → ...k_audition/8_class/naive_rf_test_time.txt
diff --git a/audio/8_class/naive_rf_train_time.txt → ..._audition/8_class/naive_rf_train_time.txt b/audio/8_class/naive_rf_train_time.txt → ..._audition/8_class/naive_rf_train_time.txt
diff --git a/audio/8_class/resnet18.txt → benchmark_audition/8_class/resnet18.txt b/audio/8_class/resnet18.txt → benchmark_audition/8_class/resnet18.txt
diff --git a/audio/8_class/resnet18_test_time.txt → ...k_audition/8_class/resnet18_test_time.txt b/audio/8_class/resnet18_test_time.txt → ...k_audition/8_class/resnet18_test_time.txt
diff --git a/audio/8_class/resnet18_train_time.txt → ..._audition/8_class/resnet18_train_time.txt b/audio/8_class/resnet18_train_time.txt → ..._audition/8_class/resnet18_train_time.txt
diff --git a/audio/8_class/svm.txt → benchmark_audition/8_class/svm.txt b/audio/8_class/svm.txt → benchmark_audition/8_class/svm.txt
diff --git a/audio/8_class/svm_test_time.txt → benchmark_audition/8_class/svm_test_time.txt b/audio/8_class/svm_test_time.txt → benchmark_audition/8_class/svm_test_time.txt
diff --git a/audio/8_class/svm_train_time.txt → ...hmark_audition/8_class/svm_train_time.txt b/audio/8_class/svm_train_time.txt → ...hmark_audition/8_class/svm_train_time.txt
diff --git a/audio/8_class_mel/cnn32.txt → benchmark_audition/8_class_mel/cnn32.txt b/audio/8_class_mel/cnn32.txt → benchmark_audition/8_class_mel/cnn32.txt
diff --git a/audio/8_class_mel/cnn32_2l.txt → benchmark_audition/8_class_mel/cnn32_2l.txt b/audio/8_class_mel/cnn32_2l.txt → benchmark_audition/8_class_mel/cnn32_2l.txt
diff --git a/audio/8_class_mel/cnn32_2l_test_time.txt → ...dition/8_class_mel/cnn32_2l_test_time.txt b/audio/8_class_mel/cnn32_2l_test_time.txt → ...dition/8_class_mel/cnn32_2l_test_time.txt
diff --git a/audio/8_class_mel/cnn32_2l_train_time.txt → ...ition/8_class_mel/cnn32_2l_train_time.txt b/audio/8_class_mel/cnn32_2l_train_time.txt → ...ition/8_class_mel/cnn32_2l_train_time.txt
diff --git a/audio/8_class_mel/cnn32_5l.txt → benchmark_audition/8_class_mel/cnn32_5l.txt b/audio/8_class_mel/cnn32_5l.txt → benchmark_audition/8_class_mel/cnn32_5l.txt
diff --git a/audio/8_class_mel/cnn32_5l_test_time.txt → ...dition/8_class_mel/cnn32_5l_test_time.txt b/audio/8_class_mel/cnn32_5l_test_time.txt → ...dition/8_class_mel/cnn32_5l_test_time.txt
diff --git a/audio/8_class_mel/cnn32_5l_train_time.txt → ...ition/8_class_mel/cnn32_5l_train_time.txt b/audio/8_class_mel/cnn32_5l_train_time.txt → ...ition/8_class_mel/cnn32_5l_train_time.txt
diff --git a/audio/8_class_mel/cnn32_test_time.txt → ..._audition/8_class_mel/cnn32_test_time.txt b/audio/8_class_mel/cnn32_test_time.txt → ..._audition/8_class_mel/cnn32_test_time.txt
diff --git a/audio/8_class_mel/cnn32_train_time.txt → ...audition/8_class_mel/cnn32_train_time.txt b/audio/8_class_mel/cnn32_train_time.txt → ...audition/8_class_mel/cnn32_train_time.txt
diff --git a/audio/8_class_mel/naive_rf.txt → benchmark_audition/8_class_mel/naive_rf.txt b/audio/8_class_mel/naive_rf.txt → benchmark_audition/8_class_mel/naive_rf.txt
diff --git a/audio/8_class_mel/naive_rf_test_time.txt → ...dition/8_class_mel/naive_rf_test_time.txt b/audio/8_class_mel/naive_rf_test_time.txt → ...dition/8_class_mel/naive_rf_test_time.txt
diff --git a/audio/8_class_mel/naive_rf_train_time.txt → ...ition/8_class_mel/naive_rf_train_time.txt b/audio/8_class_mel/naive_rf_train_time.txt → ...ition/8_class_mel/naive_rf_train_time.txt
diff --git a/audio/8_class_mel/resnet18.txt → benchmark_audition/8_class_mel/resnet18.txt b/audio/8_class_mel/resnet18.txt → benchmark_audition/8_class_mel/resnet18.txt
diff --git a/audio/8_class_mel/resnet18_test_time.txt → ...dition/8_class_mel/resnet18_test_time.txt b/audio/8_class_mel/resnet18_test_time.txt → ...dition/8_class_mel/resnet18_test_time.txt
diff --git a/audio/8_class_mel/resnet18_train_time.txt → ...ition/8_class_mel/resnet18_train_time.txt b/audio/8_class_mel/resnet18_train_time.txt → ...ition/8_class_mel/resnet18_train_time.txt
diff --git a/audio/8_class_mel/svm.txt → benchmark_audition/8_class_mel/svm.txt b/audio/8_class_mel/svm.txt → benchmark_audition/8_class_mel/svm.txt
diff --git a/audio/8_class_mel/svm_test_time.txt → ...rk_audition/8_class_mel/svm_test_time.txt b/audio/8_class_mel/svm_test_time.txt → ...rk_audition/8_class_mel/svm_test_time.txt
diff --git a/audio/8_class_mel/svm_train_time.txt → ...k_audition/8_class_mel/svm_train_time.txt b/audio/8_class_mel/svm_train_time.txt → ...k_audition/8_class_mel/svm_train_time.txt
diff --git a/audio/8_class_mfcc/cnn32.txt → benchmark_audition/8_class_mfcc/cnn32.txt b/audio/8_class_mfcc/cnn32.txt → benchmark_audition/8_class_mfcc/cnn32.txt
diff --git a/audio/8_class_mfcc/cnn32_2l.txt → benchmark_audition/8_class_mfcc/cnn32_2l.txt b/audio/8_class_mfcc/cnn32_2l.txt → benchmark_audition/8_class_mfcc/cnn32_2l.txt
diff --git a/audio/8_class_mfcc/cnn32_2l_test_time.txt → ...ition/8_class_mfcc/cnn32_2l_test_time.txt b/audio/8_class_mfcc/cnn32_2l_test_time.txt → ...ition/8_class_mfcc/cnn32_2l_test_time.txt
diff --git a/audio/8_class_mfcc/cnn32_2l_train_time.txt → ...tion/8_class_mfcc/cnn32_2l_train_time.txt b/audio/8_class_mfcc/cnn32_2l_train_time.txt → ...tion/8_class_mfcc/cnn32_2l_train_time.txt
diff --git a/audio/8_class_mfcc/cnn32_5l.txt → benchmark_audition/8_class_mfcc/cnn32_5l.txt b/audio/8_class_mfcc/cnn32_5l.txt → benchmark_audition/8_class_mfcc/cnn32_5l.txt
diff --git a/audio/8_class_mfcc/cnn32_5l_test_time.txt → ...ition/8_class_mfcc/cnn32_5l_test_time.txt b/audio/8_class_mfcc/cnn32_5l_test_time.txt → ...ition/8_class_mfcc/cnn32_5l_test_time.txt
diff --git a/audio/8_class_mfcc/cnn32_5l_train_time.txt → ...tion/8_class_mfcc/cnn32_5l_train_time.txt b/audio/8_class_mfcc/cnn32_5l_train_time.txt → ...tion/8_class_mfcc/cnn32_5l_train_time.txt
diff --git a/audio/8_class_mfcc/cnn32_test_time.txt → ...audition/8_class_mfcc/cnn32_test_time.txt b/audio/8_class_mfcc/cnn32_test_time.txt → ...audition/8_class_mfcc/cnn32_test_time.txt
diff --git a/audio/8_class_mfcc/cnn32_train_time.txt → ...udition/8_class_mfcc/cnn32_train_time.txt b/audio/8_class_mfcc/cnn32_train_time.txt → ...udition/8_class_mfcc/cnn32_train_time.txt
diff --git a/audio/8_class_mfcc/naive_rf.txt → benchmark_audition/8_class_mfcc/naive_rf.txt b/audio/8_class_mfcc/naive_rf.txt → benchmark_audition/8_class_mfcc/naive_rf.txt
diff --git a/audio/8_class_mfcc/naive_rf_test_time.txt → ...ition/8_class_mfcc/naive_rf_test_time.txt b/audio/8_class_mfcc/naive_rf_test_time.txt → ...ition/8_class_mfcc/naive_rf_test_time.txt
diff --git a/audio/8_class_mfcc/naive_rf_train_time.txt → ...tion/8_class_mfcc/naive_rf_train_time.txt b/audio/8_class_mfcc/naive_rf_train_time.txt → ...tion/8_class_mfcc/naive_rf_train_time.txt
diff --git a/audio/8_class_mfcc/resnet18.txt → benchmark_audition/8_class_mfcc/resnet18.txt b/audio/8_class_mfcc/resnet18.txt → benchmark_audition/8_class_mfcc/resnet18.txt
diff --git a/audio/8_class_mfcc/resnet18_test_time.txt → ...ition/8_class_mfcc/resnet18_test_time.txt b/audio/8_class_mfcc/resnet18_test_time.txt → ...ition/8_class_mfcc/resnet18_test_time.txt
diff --git a/audio/8_class_mfcc/resnet18_train_time.txt → ...tion/8_class_mfcc/resnet18_train_time.txt b/audio/8_class_mfcc/resnet18_train_time.txt → ...tion/8_class_mfcc/resnet18_train_time.txt
diff --git a/audio/8_class_mfcc/svm.txt → benchmark_audition/8_class_mfcc/svm.txt b/audio/8_class_mfcc/svm.txt → benchmark_audition/8_class_mfcc/svm.txt
diff --git a/audio/8_class_mfcc/svm_test_time.txt → ...k_audition/8_class_mfcc/svm_test_time.txt b/audio/8_class_mfcc/svm_test_time.txt → ...k_audition/8_class_mfcc/svm_test_time.txt
diff --git a/audio/8_class_mfcc/svm_train_time.txt → ..._audition/8_class_mfcc/svm_train_time.txt b/audio/8_class_mfcc/svm_train_time.txt → ..._audition/8_class_mfcc/svm_train_time.txt
diff --git a/audio/audio_toolbox.py → benchmark_audition/audio_toolbox.py b/audio/audio_toolbox.py → benchmark_audition/audio_toolbox.py
@@ -1,10 +1,10 @@
 """
-Coauthors: Yu-Chung Peng
-           Haoyin Xu
+Coauthors: Haoyin Xu
+           Yu-Chung Peng
            Madi Kusmanov
 """
 import numpy as np
-from sklearn.metrics import accuracy_score
+from sklearn.metrics import cohen_kappa_score
 import time
 import torch
 import os
@@ -206,7 +206,7 @@ def run_rf_image_set(
     end_time = time.perf_counter()
     test_time = end_time - start_time
 
-    return accuracy_score(test_labels, test_preds), train_time, test_time
+    return cohen_kappa_score(test_labels, test_preds), train_time, test_time
 
 
 def run_dn_image_es(
@@ -236,7 +236,7 @@ def run_dn_image_es(
     flag = 0
     start_time = time.perf_counter()
     for epoch in range(epochs):  # loop over the dataset multiple times
-
+        model.train()
         for i in range(0, len(train_data), batch):
             # get the inputs
             inputs = train_data[i : i + batch].to(dev)
@@ -251,6 +251,7 @@ def run_dn_image_es(
             optimizer.step()
 
         # test generalization error for early stopping
+        model.eval()
         cur_loss = 0
         with torch.no_grad():
             for i in range(0, len(valid_data), batch):
@@ -273,22 +274,25 @@ def run_dn_image_es(
                 break
     end_time = time.perf_counter()
     train_time = end_time - start_time
+
     # test the model
-    correct = torch.tensor(0).to(dev)
-    total = torch.tensor(0).to(dev)
+    model.eval()
     start_time = time.perf_counter()
+    test_preds = []
+    test_labels = []
     with torch.no_grad():
         for i in range(0, len(test_data), batch):
             inputs = test_data[i : i + batch].to(dev)
             labels = test_labels[i : i + batch].to(dev)
+            test_labels = np.concatenate((test_labels, labels.tolist()))
+
             outputs = model(inputs)
             _, predicted = torch.max(outputs.data, 1)
-            total += labels.size(0)
-            correct += (predicted == labels.view(-1)).sum().item()
+            test_preds = np.concatenate((test_preds, predicted.tolist()))
+
     end_time = time.perf_counter()
     test_time = end_time - start_time
-    accuracy = float(correct) / float(total)
-    return accuracy, train_time, test_time
+    return cohen_kappa_score(test_preds, test_labels), train_time, test_time
 
 
 def prepare_data(

diff --git a/audio/fsdd.py → benchmark_audition/fsdd.py b/audio/fsdd.py → benchmark_audition/fsdd.py
@@ -1,6 +1,6 @@
 """
-Coauthors: Yu-Chung Peng
-           Haoyin Xu
+Coauthors: Haoyin Xu
+           Yu-Chung Peng
            Madi Kusmanov
 """
 from audio_toolbox import *

diff --git a/audio/fsdd_figures.ipynb → benchmark_audition/fsdd_figures.ipynb b/audio/fsdd_figures.ipynb → benchmark_audition/fsdd_figures.ipynb
diff --git a/tabular/cc18_figures.ipynb → benchmark_tabular/cc18_figures.ipynb b/tabular/cc18_figures.ipynb → benchmark_tabular/cc18_figures.ipynb
diff --git a/tabular/cc18_hyperparameter.py → benchmark_tabular/cc18_hyperparameter.py b/tabular/cc18_hyperparameter.py → benchmark_tabular/cc18_hyperparameter.py
diff --git a/tabular/cc18_kappa_ece.py → benchmark_tabular/cc18_kappa_ece.py b/tabular/cc18_kappa_ece.py → benchmark_tabular/cc18_kappa_ece.py
@@ -1,5 +1,6 @@
 """
-Author: Michael Ainsworth
+Coauthors: Michael Ainsworth
+           Jayanta Dey
 """
 
 import numpy as np

diff --git a/tabular/cc18_times.py → benchmark_tabular/cc18_times.py b/tabular/cc18_times.py → benchmark_tabular/cc18_times.py
diff --git a/tabular/metrics/cc18_all_parameters.txt → ...k_tabular/metrics/cc18_all_parameters.txt b/tabular/metrics/cc18_all_parameters.txt → ...k_tabular/metrics/cc18_all_parameters.txt
diff --git a/tabular/metrics/cc18_sample_sizes.txt → ...ark_tabular/metrics/cc18_sample_sizes.txt b/tabular/metrics/cc18_sample_sizes.txt → ...ark_tabular/metrics/cc18_sample_sizes.txt
diff --git a/tabular/results/cc18_dn_ece.txt → benchmark_tabular/results/cc18_dn_ece.txt b/tabular/results/cc18_dn_ece.txt → benchmark_tabular/results/cc18_dn_ece.txt
diff --git a/tabular/results/cc18_dn_kappa.txt → benchmark_tabular/results/cc18_dn_kappa.txt b/tabular/results/cc18_dn_kappa.txt → benchmark_tabular/results/cc18_dn_kappa.txt
diff --git a/tabular/results/cc18_dn_times_test.txt → ...rk_tabular/results/cc18_dn_times_test.txt b/tabular/results/cc18_dn_times_test.txt → ...rk_tabular/results/cc18_dn_times_test.txt
diff --git a/tabular/results/cc18_dn_times_train.txt → ...k_tabular/results/cc18_dn_times_train.txt b/tabular/results/cc18_dn_times_train.txt → ...k_tabular/results/cc18_dn_times_train.txt
diff --git a/tabular/results/cc18_rf_ece.txt → benchmark_tabular/results/cc18_rf_ece.txt b/tabular/results/cc18_rf_ece.txt → benchmark_tabular/results/cc18_rf_ece.txt
diff --git a/tabular/results/cc18_rf_kappa.txt → benchmark_tabular/results/cc18_rf_kappa.txt b/tabular/results/cc18_rf_kappa.txt → benchmark_tabular/results/cc18_rf_kappa.txt
diff --git a/tabular/results/cc18_rf_times_test.txt → ...rk_tabular/results/cc18_rf_times_test.txt b/tabular/results/cc18_rf_times_test.txt → ...rk_tabular/results/cc18_rf_times_test.txt
diff --git a/tabular/results/cc18_rf_times_train.txt → ...k_tabular/results/cc18_rf_times_train.txt b/tabular/results/cc18_rf_times_train.txt → ...k_tabular/results/cc18_rf_times_train.txt
diff --git a/image/3_class/cnn32.txt → benchmark_vision/3_class/cnn32.txt b/image/3_class/cnn32.txt → benchmark_vision/3_class/cnn32.txt
diff --git a/image/3_class/cnn32_2l.txt → benchmark_vision/3_class/cnn32_2l.txt b/image/3_class/cnn32_2l.txt → benchmark_vision/3_class/cnn32_2l.txt
diff --git a/image/3_class/cnn32_2l_sc.txt → benchmark_vision/3_class/cnn32_2l_sc.txt b/image/3_class/cnn32_2l_sc.txt → benchmark_vision/3_class/cnn32_2l_sc.txt
diff --git a/image/3_class/cnn32_2l_st.txt → benchmark_vision/3_class/cnn32_2l_st.txt b/image/3_class/cnn32_2l_st.txt → benchmark_vision/3_class/cnn32_2l_st.txt
diff --git a/image/3_class/cnn32_2l_test_time.txt → ...ark_vision/3_class/cnn32_2l_test_time.txt b/image/3_class/cnn32_2l_test_time.txt → ...ark_vision/3_class/cnn32_2l_test_time.txt
diff --git a/image/3_class/cnn32_2l_test_time_sc.txt → ..._vision/3_class/cnn32_2l_test_time_sc.txt b/image/3_class/cnn32_2l_test_time_sc.txt → ..._vision/3_class/cnn32_2l_test_time_sc.txt
diff --git a/image/3_class/cnn32_2l_test_time_st.txt → ..._vision/3_class/cnn32_2l_test_time_st.txt b/image/3_class/cnn32_2l_test_time_st.txt → ..._vision/3_class/cnn32_2l_test_time_st.txt
diff --git a/image/3_class/cnn32_2l_train_time.txt → ...rk_vision/3_class/cnn32_2l_train_time.txt b/image/3_class/cnn32_2l_train_time.txt → ...rk_vision/3_class/cnn32_2l_train_time.txt
diff --git a/image/3_class/cnn32_2l_train_time_sc.txt → ...vision/3_class/cnn32_2l_train_time_sc.txt b/image/3_class/cnn32_2l_train_time_sc.txt → ...vision/3_class/cnn32_2l_train_time_sc.txt
diff --git a/image/3_class/cnn32_2l_train_time_st.txt → ...vision/3_class/cnn32_2l_train_time_st.txt b/image/3_class/cnn32_2l_train_time_st.txt → ...vision/3_class/cnn32_2l_train_time_st.txt
diff --git a/image/3_class/cnn32_5l.txt → benchmark_vision/3_class/cnn32_5l.txt b/image/3_class/cnn32_5l.txt → benchmark_vision/3_class/cnn32_5l.txt
diff --git a/image/3_class/cnn32_5l_sc.txt → benchmark_vision/3_class/cnn32_5l_sc.txt b/image/3_class/cnn32_5l_sc.txt → benchmark_vision/3_class/cnn32_5l_sc.txt
diff --git a/image/3_class/cnn32_5l_st.txt → benchmark_vision/3_class/cnn32_5l_st.txt b/image/3_class/cnn32_5l_st.txt → benchmark_vision/3_class/cnn32_5l_st.txt
diff --git a/image/3_class/cnn32_5l_test_time.txt → ...ark_vision/3_class/cnn32_5l_test_time.txt b/image/3_class/cnn32_5l_test_time.txt → ...ark_vision/3_class/cnn32_5l_test_time.txt
diff --git a/image/3_class/cnn32_5l_test_time_sc.txt → ..._vision/3_class/cnn32_5l_test_time_sc.txt b/image/3_class/cnn32_5l_test_time_sc.txt → ..._vision/3_class/cnn32_5l_test_time_sc.txt
diff --git a/image/3_class/cnn32_5l_test_time_st.txt → ..._vision/3_class/cnn32_5l_test_time_st.txt b/image/3_class/cnn32_5l_test_time_st.txt → ..._vision/3_class/cnn32_5l_test_time_st.txt
diff --git a/image/3_class/cnn32_5l_train_time.txt → ...rk_vision/3_class/cnn32_5l_train_time.txt b/image/3_class/cnn32_5l_train_time.txt → ...rk_vision/3_class/cnn32_5l_train_time.txt
diff --git a/image/3_class/cnn32_5l_train_time_sc.txt → ...vision/3_class/cnn32_5l_train_time_sc.txt b/image/3_class/cnn32_5l_train_time_sc.txt → ...vision/3_class/cnn32_5l_train_time_sc.txt
diff --git a/image/3_class/cnn32_5l_train_time_st.txt → ...vision/3_class/cnn32_5l_train_time_st.txt b/image/3_class/cnn32_5l_train_time_st.txt → ...vision/3_class/cnn32_5l_train_time_st.txt
diff --git a/image/3_class/cnn32_sc.txt → benchmark_vision/3_class/cnn32_sc.txt b/image/3_class/cnn32_sc.txt → benchmark_vision/3_class/cnn32_sc.txt
diff --git a/image/3_class/cnn32_st.txt → benchmark_vision/3_class/cnn32_st.txt b/image/3_class/cnn32_st.txt → benchmark_vision/3_class/cnn32_st.txt
diff --git a/image/3_class/cnn32_test_time.txt → benchmark_vision/3_class/cnn32_test_time.txt b/image/3_class/cnn32_test_time.txt → benchmark_vision/3_class/cnn32_test_time.txt
diff --git a/image/3_class/cnn32_test_time_sc.txt → ...ark_vision/3_class/cnn32_test_time_sc.txt b/image/3_class/cnn32_test_time_sc.txt → ...ark_vision/3_class/cnn32_test_time_sc.txt
diff --git a/image/3_class/cnn32_test_time_st.txt → ...ark_vision/3_class/cnn32_test_time_st.txt b/image/3_class/cnn32_test_time_st.txt → ...ark_vision/3_class/cnn32_test_time_st.txt
diff --git a/image/3_class/cnn32_train_time.txt → ...hmark_vision/3_class/cnn32_train_time.txt b/image/3_class/cnn32_train_time.txt → ...hmark_vision/3_class/cnn32_train_time.txt
diff --git a/image/3_class/cnn32_train_time_sc.txt → ...rk_vision/3_class/cnn32_train_time_sc.txt b/image/3_class/cnn32_train_time_sc.txt → ...rk_vision/3_class/cnn32_train_time_sc.txt
diff --git a/image/3_class/cnn32_train_time_st.txt → ...rk_vision/3_class/cnn32_train_time_st.txt b/image/3_class/cnn32_train_time_st.txt → ...rk_vision/3_class/cnn32_train_time_st.txt
diff --git a/image/3_class/naive_rf.txt → benchmark_vision/3_class/naive_rf.txt b/image/3_class/naive_rf.txt → benchmark_vision/3_class/naive_rf.txt
diff --git a/image/3_class/naive_rf_test_time.txt → ...ark_vision/3_class/naive_rf_test_time.txt b/image/3_class/naive_rf_test_time.txt → ...ark_vision/3_class/naive_rf_test_time.txt
diff --git a/image/3_class/naive_rf_test_time_lc.txt → ..._vision/3_class/naive_rf_test_time_lc.txt b/image/3_class/naive_rf_test_time_lc.txt → ..._vision/3_class/naive_rf_test_time_lc.txt
diff --git a/image/3_class/naive_rf_train_time.txt → ...rk_vision/3_class/naive_rf_train_time.txt b/image/3_class/naive_rf_train_time.txt → ...rk_vision/3_class/naive_rf_train_time.txt
diff --git a/image/3_class/naive_rf_train_time_lc.txt → ...vision/3_class/naive_rf_train_time_lc.txt b/image/3_class/naive_rf_train_time_lc.txt → ...vision/3_class/naive_rf_train_time_lc.txt
diff --git a/image/3_class/resnet18.txt → benchmark_vision/3_class/resnet18.txt b/image/3_class/resnet18.txt → benchmark_vision/3_class/resnet18.txt
diff --git a/image/3_class/resnet18_sc.txt → benchmark_vision/3_class/resnet18_sc.txt b/image/3_class/resnet18_sc.txt → benchmark_vision/3_class/resnet18_sc.txt
diff --git a/image/3_class/resnet18_st.txt → benchmark_vision/3_class/resnet18_st.txt b/image/3_class/resnet18_st.txt → benchmark_vision/3_class/resnet18_st.txt
diff --git a/image/3_class/resnet18_test_time.txt → ...ark_vision/3_class/resnet18_test_time.txt b/image/3_class/resnet18_test_time.txt → ...ark_vision/3_class/resnet18_test_time.txt
diff --git a/image/3_class/resnet18_test_time_sc.txt → ..._vision/3_class/resnet18_test_time_sc.txt b/image/3_class/resnet18_test_time_sc.txt → ..._vision/3_class/resnet18_test_time_sc.txt
diff --git a/image/3_class/resnet18_test_time_st.txt → ..._vision/3_class/resnet18_test_time_st.txt b/image/3_class/resnet18_test_time_st.txt → ..._vision/3_class/resnet18_test_time_st.txt
diff --git a/image/3_class/resnet18_train_time.txt → ...rk_vision/3_class/resnet18_train_time.txt b/image/3_class/resnet18_train_time.txt → ...rk_vision/3_class/resnet18_train_time.txt
diff --git a/image/3_class/resnet18_train_time_sc.txt → ...vision/3_class/resnet18_train_time_sc.txt b/image/3_class/resnet18_train_time_sc.txt → ...vision/3_class/resnet18_train_time_sc.txt
diff --git a/image/3_class/resnet18_train_time_st.txt → ...vision/3_class/resnet18_train_time_st.txt b/image/3_class/resnet18_train_time_st.txt → ...vision/3_class/resnet18_train_time_st.txt
diff --git a/image/3_class/svhn_cnn32.txt → benchmark_vision/3_class/svhn_cnn32.txt b/image/3_class/svhn_cnn32.txt → benchmark_vision/3_class/svhn_cnn32.txt
diff --git a/image/3_class/svhn_cnn32_2l.txt → benchmark_vision/3_class/svhn_cnn32_2l.txt b/image/3_class/svhn_cnn32_2l.txt → benchmark_vision/3_class/svhn_cnn32_2l.txt
diff --git a/image/3_class/svhn_cnn32_2l_test_time.txt → ...ision/3_class/svhn_cnn32_2l_test_time.txt b/image/3_class/svhn_cnn32_2l_test_time.txt → ...ision/3_class/svhn_cnn32_2l_test_time.txt
diff --git a/image/3_class/svhn_cnn32_2l_train_time.txt → ...sion/3_class/svhn_cnn32_2l_train_time.txt b/image/3_class/svhn_cnn32_2l_train_time.txt → ...sion/3_class/svhn_cnn32_2l_train_time.txt
diff --git a/image/3_class/svhn_cnn32_5l.txt → benchmark_vision/3_class/svhn_cnn32_5l.txt b/image/3_class/svhn_cnn32_5l.txt → benchmark_vision/3_class/svhn_cnn32_5l.txt
diff --git a/image/3_class/svhn_cnn32_5l_test_time.txt → ...ision/3_class/svhn_cnn32_5l_test_time.txt b/image/3_class/svhn_cnn32_5l_test_time.txt → ...ision/3_class/svhn_cnn32_5l_test_time.txt
diff --git a/image/3_class/svhn_cnn32_5l_train_time.txt → ...sion/3_class/svhn_cnn32_5l_train_time.txt b/image/3_class/svhn_cnn32_5l_train_time.txt → ...sion/3_class/svhn_cnn32_5l_train_time.txt
diff --git a/image/3_class/svhn_cnn32_test_time.txt → ...k_vision/3_class/svhn_cnn32_test_time.txt b/image/3_class/svhn_cnn32_test_time.txt → ...k_vision/3_class/svhn_cnn32_test_time.txt
diff --git a/image/3_class/svhn_cnn32_train_time.txt → ..._vision/3_class/svhn_cnn32_train_time.txt b/image/3_class/svhn_cnn32_train_time.txt → ..._vision/3_class/svhn_cnn32_train_time.txt
diff --git a/image/3_class/svhn_naive_rf.txt → benchmark_vision/3_class/svhn_naive_rf.txt b/image/3_class/svhn_naive_rf.txt → benchmark_vision/3_class/svhn_naive_rf.txt
diff --git a/image/3_class/svhn_naive_rf_test_time.txt → ...ision/3_class/svhn_naive_rf_test_time.txt b/image/3_class/svhn_naive_rf_test_time.txt → ...ision/3_class/svhn_naive_rf_test_time.txt
diff --git a/image/3_class/svhn_naive_rf_train_time.txt → ...sion/3_class/svhn_naive_rf_train_time.txt b/image/3_class/svhn_naive_rf_train_time.txt → ...sion/3_class/svhn_naive_rf_train_time.txt
diff --git a/image/3_class/svhn_resnet18.txt → benchmark_vision/3_class/svhn_resnet18.txt b/image/3_class/svhn_resnet18.txt → benchmark_vision/3_class/svhn_resnet18.txt
diff --git a/image/3_class/svhn_resnet18_test_time.txt → ...ision/3_class/svhn_resnet18_test_time.txt b/image/3_class/svhn_resnet18_test_time.txt → ...ision/3_class/svhn_resnet18_test_time.txt
diff --git a/image/3_class/svhn_resnet18_train_time.txt → ...sion/3_class/svhn_resnet18_train_time.txt b/image/3_class/svhn_resnet18_train_time.txt → ...sion/3_class/svhn_resnet18_train_time.txt
diff --git a/image/3_class/svhn_svm.txt → benchmark_vision/3_class/svhn_svm.txt b/image/3_class/svhn_svm.txt → benchmark_vision/3_class/svhn_svm.txt
diff --git a/image/3_class/svhn_svm_test_time.txt → ...ark_vision/3_class/svhn_svm_test_time.txt b/image/3_class/svhn_svm_test_time.txt → ...ark_vision/3_class/svhn_svm_test_time.txt
diff --git a/image/3_class/svhn_svm_train_time.txt → ...rk_vision/3_class/svhn_svm_train_time.txt b/image/3_class/svhn_svm_train_time.txt → ...rk_vision/3_class/svhn_svm_train_time.txt
diff --git a/image/3_class/svm.txt → benchmark_vision/3_class/svm.txt b/image/3_class/svm.txt → benchmark_vision/3_class/svm.txt
diff --git a/image/3_class/svm_test_time.txt → benchmark_vision/3_class/svm_test_time.txt b/image/3_class/svm_test_time.txt → benchmark_vision/3_class/svm_test_time.txt
diff --git a/image/3_class/svm_test_time_lc.txt → ...hmark_vision/3_class/svm_test_time_lc.txt b/image/3_class/svm_test_time_lc.txt → ...hmark_vision/3_class/svm_test_time_lc.txt
diff --git a/image/3_class/svm_train_time.txt → benchmark_vision/3_class/svm_train_time.txt b/image/3_class/svm_train_time.txt → benchmark_vision/3_class/svm_train_time.txt
diff --git a/image/3_class/svm_train_time_lc.txt → ...mark_vision/3_class/svm_train_time_lc.txt b/image/3_class/svm_train_time_lc.txt → ...mark_vision/3_class/svm_train_time_lc.txt
diff --git a/image/8_class/cnn32.txt → benchmark_vision/8_class/cnn32.txt b/image/8_class/cnn32.txt → benchmark_vision/8_class/cnn32.txt
diff --git a/image/8_class/cnn32_2l.txt → benchmark_vision/8_class/cnn32_2l.txt b/image/8_class/cnn32_2l.txt → benchmark_vision/8_class/cnn32_2l.txt
diff --git a/image/8_class/cnn32_2l_sc.txt → benchmark_vision/8_class/cnn32_2l_sc.txt b/image/8_class/cnn32_2l_sc.txt → benchmark_vision/8_class/cnn32_2l_sc.txt
diff --git a/image/8_class/cnn32_2l_st.txt → benchmark_vision/8_class/cnn32_2l_st.txt b/image/8_class/cnn32_2l_st.txt → benchmark_vision/8_class/cnn32_2l_st.txt
diff --git a/image/8_class/cnn32_2l_test_time.txt → ...ark_vision/8_class/cnn32_2l_test_time.txt b/image/8_class/cnn32_2l_test_time.txt → ...ark_vision/8_class/cnn32_2l_test_time.txt
diff --git a/image/8_class/cnn32_2l_test_time_sc.txt → ..._vision/8_class/cnn32_2l_test_time_sc.txt b/image/8_class/cnn32_2l_test_time_sc.txt → ..._vision/8_class/cnn32_2l_test_time_sc.txt
diff --git a/image/8_class/cnn32_2l_test_time_st.txt → ..._vision/8_class/cnn32_2l_test_time_st.txt b/image/8_class/cnn32_2l_test_time_st.txt → ..._vision/8_class/cnn32_2l_test_time_st.txt
diff --git a/image/8_class/cnn32_2l_train_time.txt → ...rk_vision/8_class/cnn32_2l_train_time.txt b/image/8_class/cnn32_2l_train_time.txt → ...rk_vision/8_class/cnn32_2l_train_time.txt
diff --git a/image/8_class/cnn32_2l_train_time_sc.txt → ...vision/8_class/cnn32_2l_train_time_sc.txt b/image/8_class/cnn32_2l_train_time_sc.txt → ...vision/8_class/cnn32_2l_train_time_sc.txt
diff --git a/image/8_class/cnn32_2l_train_time_st.txt → ...vision/8_class/cnn32_2l_train_time_st.txt b/image/8_class/cnn32_2l_train_time_st.txt → ...vision/8_class/cnn32_2l_train_time_st.txt
diff --git a/image/8_class/cnn32_5l.txt → benchmark_vision/8_class/cnn32_5l.txt b/image/8_class/cnn32_5l.txt → benchmark_vision/8_class/cnn32_5l.txt
diff --git a/image/8_class/cnn32_5l_sc.txt → benchmark_vision/8_class/cnn32_5l_sc.txt b/image/8_class/cnn32_5l_sc.txt → benchmark_vision/8_class/cnn32_5l_sc.txt
diff --git a/image/8_class/cnn32_5l_st.txt → benchmark_vision/8_class/cnn32_5l_st.txt b/image/8_class/cnn32_5l_st.txt → benchmark_vision/8_class/cnn32_5l_st.txt
diff --git a/image/8_class/cnn32_5l_test_time.txt → ...ark_vision/8_class/cnn32_5l_test_time.txt b/image/8_class/cnn32_5l_test_time.txt → ...ark_vision/8_class/cnn32_5l_test_time.txt
diff --git a/image/8_class/cnn32_5l_test_time_sc.txt → ..._vision/8_class/cnn32_5l_test_time_sc.txt b/image/8_class/cnn32_5l_test_time_sc.txt → ..._vision/8_class/cnn32_5l_test_time_sc.txt
diff --git a/image/8_class/cnn32_5l_test_time_st.txt → ..._vision/8_class/cnn32_5l_test_time_st.txt b/image/8_class/cnn32_5l_test_time_st.txt → ..._vision/8_class/cnn32_5l_test_time_st.txt
diff --git a/image/8_class/cnn32_5l_train_time.txt → ...rk_vision/8_class/cnn32_5l_train_time.txt b/image/8_class/cnn32_5l_train_time.txt → ...rk_vision/8_class/cnn32_5l_train_time.txt
diff --git a/image/8_class/cnn32_5l_train_time_sc.txt → ...vision/8_class/cnn32_5l_train_time_sc.txt b/image/8_class/cnn32_5l_train_time_sc.txt → ...vision/8_class/cnn32_5l_train_time_sc.txt
diff --git a/image/8_class/cnn32_5l_train_time_st.txt → ...vision/8_class/cnn32_5l_train_time_st.txt b/image/8_class/cnn32_5l_train_time_st.txt → ...vision/8_class/cnn32_5l_train_time_st.txt
diff --git a/image/8_class/cnn32_sc.txt → benchmark_vision/8_class/cnn32_sc.txt b/image/8_class/cnn32_sc.txt → benchmark_vision/8_class/cnn32_sc.txt
diff --git a/image/8_class/cnn32_st.txt → benchmark_vision/8_class/cnn32_st.txt b/image/8_class/cnn32_st.txt → benchmark_vision/8_class/cnn32_st.txt
diff --git a/image/8_class/cnn32_test_time.txt → benchmark_vision/8_class/cnn32_test_time.txt b/image/8_class/cnn32_test_time.txt → benchmark_vision/8_class/cnn32_test_time.txt
diff --git a/image/8_class/cnn32_test_time_sc.txt → ...ark_vision/8_class/cnn32_test_time_sc.txt b/image/8_class/cnn32_test_time_sc.txt → ...ark_vision/8_class/cnn32_test_time_sc.txt
diff --git a/image/8_class/cnn32_test_time_st.txt → ...ark_vision/8_class/cnn32_test_time_st.txt b/image/8_class/cnn32_test_time_st.txt → ...ark_vision/8_class/cnn32_test_time_st.txt
diff --git a/image/8_class/cnn32_train_time.txt → ...hmark_vision/8_class/cnn32_train_time.txt b/image/8_class/cnn32_train_time.txt → ...hmark_vision/8_class/cnn32_train_time.txt
diff --git a/image/8_class/cnn32_train_time_sc.txt → ...rk_vision/8_class/cnn32_train_time_sc.txt b/image/8_class/cnn32_train_time_sc.txt → ...rk_vision/8_class/cnn32_train_time_sc.txt
diff --git a/image/8_class/cnn32_train_time_st.txt → ...rk_vision/8_class/cnn32_train_time_st.txt b/image/8_class/cnn32_train_time_st.txt → ...rk_vision/8_class/cnn32_train_time_st.txt
diff --git a/image/8_class/naive_rf.txt → benchmark_vision/8_class/naive_rf.txt b/image/8_class/naive_rf.txt → benchmark_vision/8_class/naive_rf.txt
diff --git a/image/8_class/naive_rf_test_time.txt → ...ark_vision/8_class/naive_rf_test_time.txt b/image/8_class/naive_rf_test_time.txt → ...ark_vision/8_class/naive_rf_test_time.txt
diff --git a/image/8_class/naive_rf_test_time_lc.txt → ..._vision/8_class/naive_rf_test_time_lc.txt b/image/8_class/naive_rf_test_time_lc.txt → ..._vision/8_class/naive_rf_test_time_lc.txt
diff --git a/image/8_class/naive_rf_train_time.txt → ...rk_vision/8_class/naive_rf_train_time.txt b/image/8_class/naive_rf_train_time.txt → ...rk_vision/8_class/naive_rf_train_time.txt
diff --git a/image/8_class/naive_rf_train_time_lc.txt → ...vision/8_class/naive_rf_train_time_lc.txt b/image/8_class/naive_rf_train_time_lc.txt → ...vision/8_class/naive_rf_train_time_lc.txt
diff --git a/image/8_class/resnet18.txt → benchmark_vision/8_class/resnet18.txt b/image/8_class/resnet18.txt → benchmark_vision/8_class/resnet18.txt
diff --git a/image/8_class/resnet18_sc.txt → benchmark_vision/8_class/resnet18_sc.txt b/image/8_class/resnet18_sc.txt → benchmark_vision/8_class/resnet18_sc.txt
diff --git a/image/8_class/resnet18_st.txt → benchmark_vision/8_class/resnet18_st.txt b/image/8_class/resnet18_st.txt → benchmark_vision/8_class/resnet18_st.txt
diff --git a/image/8_class/resnet18_test_time.txt → ...ark_vision/8_class/resnet18_test_time.txt b/image/8_class/resnet18_test_time.txt → ...ark_vision/8_class/resnet18_test_time.txt
diff --git a/image/8_class/resnet18_test_time_sc.txt → ..._vision/8_class/resnet18_test_time_sc.txt b/image/8_class/resnet18_test_time_sc.txt → ..._vision/8_class/resnet18_test_time_sc.txt
diff --git a/image/8_class/resnet18_test_time_st.txt → ..._vision/8_class/resnet18_test_time_st.txt b/image/8_class/resnet18_test_time_st.txt → ..._vision/8_class/resnet18_test_time_st.txt
diff --git a/image/8_class/resnet18_train_time.txt → ...rk_vision/8_class/resnet18_train_time.txt b/image/8_class/resnet18_train_time.txt → ...rk_vision/8_class/resnet18_train_time.txt
diff --git a/image/8_class/resnet18_train_time_sc.txt → ...vision/8_class/resnet18_train_time_sc.txt b/image/8_class/resnet18_train_time_sc.txt → ...vision/8_class/resnet18_train_time_sc.txt
diff --git a/image/8_class/resnet18_train_time_st.txt → ...vision/8_class/resnet18_train_time_st.txt b/image/8_class/resnet18_train_time_st.txt → ...vision/8_class/resnet18_train_time_st.txt
diff --git a/image/8_class/svhn_cnn32.txt → benchmark_vision/8_class/svhn_cnn32.txt b/image/8_class/svhn_cnn32.txt → benchmark_vision/8_class/svhn_cnn32.txt
diff --git a/image/8_class/svhn_cnn32_2l.txt → benchmark_vision/8_class/svhn_cnn32_2l.txt b/image/8_class/svhn_cnn32_2l.txt → benchmark_vision/8_class/svhn_cnn32_2l.txt
diff --git a/image/8_class/svhn_cnn32_2l_test_time.txt → ...ision/8_class/svhn_cnn32_2l_test_time.txt b/image/8_class/svhn_cnn32_2l_test_time.txt → ...ision/8_class/svhn_cnn32_2l_test_time.txt
diff --git a/image/8_class/svhn_cnn32_2l_train_time.txt → ...sion/8_class/svhn_cnn32_2l_train_time.txt b/image/8_class/svhn_cnn32_2l_train_time.txt → ...sion/8_class/svhn_cnn32_2l_train_time.txt
diff --git a/image/8_class/svhn_cnn32_5l.txt → benchmark_vision/8_class/svhn_cnn32_5l.txt b/image/8_class/svhn_cnn32_5l.txt → benchmark_vision/8_class/svhn_cnn32_5l.txt
diff --git a/image/8_class/svhn_cnn32_5l_test_time.txt → ...ision/8_class/svhn_cnn32_5l_test_time.txt b/image/8_class/svhn_cnn32_5l_test_time.txt → ...ision/8_class/svhn_cnn32_5l_test_time.txt
diff --git a/image/8_class/svhn_cnn32_5l_train_time.txt → ...sion/8_class/svhn_cnn32_5l_train_time.txt b/image/8_class/svhn_cnn32_5l_train_time.txt → ...sion/8_class/svhn_cnn32_5l_train_time.txt
diff --git a/image/8_class/svhn_cnn32_test_time.txt → ...k_vision/8_class/svhn_cnn32_test_time.txt b/image/8_class/svhn_cnn32_test_time.txt → ...k_vision/8_class/svhn_cnn32_test_time.txt
diff --git a/image/8_class/svhn_cnn32_train_time.txt → ..._vision/8_class/svhn_cnn32_train_time.txt b/image/8_class/svhn_cnn32_train_time.txt → ..._vision/8_class/svhn_cnn32_train_time.txt
diff --git a/image/8_class/svhn_naive_rf.txt → benchmark_vision/8_class/svhn_naive_rf.txt b/image/8_class/svhn_naive_rf.txt → benchmark_vision/8_class/svhn_naive_rf.txt
diff --git a/image/8_class/svhn_naive_rf_test_time.txt → ...ision/8_class/svhn_naive_rf_test_time.txt b/image/8_class/svhn_naive_rf_test_time.txt → ...ision/8_class/svhn_naive_rf_test_time.txt
diff --git a/image/8_class/svhn_naive_rf_train_time.txt → ...sion/8_class/svhn_naive_rf_train_time.txt b/image/8_class/svhn_naive_rf_train_time.txt → ...sion/8_class/svhn_naive_rf_train_time.txt
diff --git a/image/8_class/svhn_resnet18.txt → benchmark_vision/8_class/svhn_resnet18.txt b/image/8_class/svhn_resnet18.txt → benchmark_vision/8_class/svhn_resnet18.txt
diff --git a/image/8_class/svhn_resnet18_test_time.txt → ...ision/8_class/svhn_resnet18_test_time.txt b/image/8_class/svhn_resnet18_test_time.txt → ...ision/8_class/svhn_resnet18_test_time.txt
diff --git a/image/8_class/svhn_resnet18_train_time.txt → ...sion/8_class/svhn_resnet18_train_time.txt b/image/8_class/svhn_resnet18_train_time.txt → ...sion/8_class/svhn_resnet18_train_time.txt
diff --git a/image/8_class/svhn_svm.txt → benchmark_vision/8_class/svhn_svm.txt b/image/8_class/svhn_svm.txt → benchmark_vision/8_class/svhn_svm.txt
diff --git a/image/8_class/svhn_svm_test_time.txt → ...ark_vision/8_class/svhn_svm_test_time.txt b/image/8_class/svhn_svm_test_time.txt → ...ark_vision/8_class/svhn_svm_test_time.txt
diff --git a/image/8_class/svhn_svm_train_time.txt → ...rk_vision/8_class/svhn_svm_train_time.txt b/image/8_class/svhn_svm_train_time.txt → ...rk_vision/8_class/svhn_svm_train_time.txt
diff --git a/image/8_class/svm.txt → benchmark_vision/8_class/svm.txt b/image/8_class/svm.txt → benchmark_vision/8_class/svm.txt
diff --git a/image/8_class/svm_test_time.txt → benchmark_vision/8_class/svm_test_time.txt b/image/8_class/svm_test_time.txt → benchmark_vision/8_class/svm_test_time.txt
diff --git a/image/8_class/svm_test_time_lc.txt → ...hmark_vision/8_class/svm_test_time_lc.txt b/image/8_class/svm_test_time_lc.txt → ...hmark_vision/8_class/svm_test_time_lc.txt
diff --git a/image/8_class/svm_train_time.txt → benchmark_vision/8_class/svm_train_time.txt b/image/8_class/svm_train_time.txt → benchmark_vision/8_class/svm_train_time.txt
diff --git a/image/8_class/svm_train_time_lc.txt → ...mark_vision/8_class/svm_train_time_lc.txt b/image/8_class/svm_train_time_lc.txt → ...mark_vision/8_class/svm_train_time_lc.txt
diff --git a/image/90_class/cnn32.txt → benchmark_vision/90_class/cnn32.txt b/image/90_class/cnn32.txt → benchmark_vision/90_class/cnn32.txt
diff --git a/image/90_class/cnn32_2l.txt → benchmark_vision/90_class/cnn32_2l.txt b/image/90_class/cnn32_2l.txt → benchmark_vision/90_class/cnn32_2l.txt
diff --git a/image/90_class/cnn32_2l_sc.txt → benchmark_vision/90_class/cnn32_2l_sc.txt b/image/90_class/cnn32_2l_sc.txt → benchmark_vision/90_class/cnn32_2l_sc.txt
diff --git a/image/90_class/cnn32_2l_st.txt → benchmark_vision/90_class/cnn32_2l_st.txt b/image/90_class/cnn32_2l_st.txt → benchmark_vision/90_class/cnn32_2l_st.txt
diff --git a/image/90_class/cnn32_2l_test_time.txt → ...rk_vision/90_class/cnn32_2l_test_time.txt b/image/90_class/cnn32_2l_test_time.txt → ...rk_vision/90_class/cnn32_2l_test_time.txt
diff --git a/image/90_class/cnn32_2l_test_time_sc.txt → ...vision/90_class/cnn32_2l_test_time_sc.txt b/image/90_class/cnn32_2l_test_time_sc.txt → ...vision/90_class/cnn32_2l_test_time_sc.txt
diff --git a/image/90_class/cnn32_2l_test_time_st.txt → ...vision/90_class/cnn32_2l_test_time_st.txt b/image/90_class/cnn32_2l_test_time_st.txt → ...vision/90_class/cnn32_2l_test_time_st.txt
diff --git a/image/90_class/cnn32_2l_train_time.txt → ...k_vision/90_class/cnn32_2l_train_time.txt b/image/90_class/cnn32_2l_train_time.txt → ...k_vision/90_class/cnn32_2l_train_time.txt
diff --git a/image/90_class/cnn32_2l_train_time_sc.txt → ...ision/90_class/cnn32_2l_train_time_sc.txt b/image/90_class/cnn32_2l_train_time_sc.txt → ...ision/90_class/cnn32_2l_train_time_sc.txt
diff --git a/image/90_class/cnn32_2l_train_time_st.txt → ...ision/90_class/cnn32_2l_train_time_st.txt b/image/90_class/cnn32_2l_train_time_st.txt → ...ision/90_class/cnn32_2l_train_time_st.txt
diff --git a/image/90_class/cnn32_5l.txt → benchmark_vision/90_class/cnn32_5l.txt b/image/90_class/cnn32_5l.txt → benchmark_vision/90_class/cnn32_5l.txt
diff --git a/image/90_class/cnn32_5l_sc.txt → benchmark_vision/90_class/cnn32_5l_sc.txt b/image/90_class/cnn32_5l_sc.txt → benchmark_vision/90_class/cnn32_5l_sc.txt
diff --git a/image/90_class/cnn32_5l_st.txt → benchmark_vision/90_class/cnn32_5l_st.txt b/image/90_class/cnn32_5l_st.txt → benchmark_vision/90_class/cnn32_5l_st.txt
diff --git a/image/90_class/cnn32_5l_test_time.txt → ...rk_vision/90_class/cnn32_5l_test_time.txt b/image/90_class/cnn32_5l_test_time.txt → ...rk_vision/90_class/cnn32_5l_test_time.txt
diff --git a/image/90_class/cnn32_5l_test_time_sc.txt → ...vision/90_class/cnn32_5l_test_time_sc.txt b/image/90_class/cnn32_5l_test_time_sc.txt → ...vision/90_class/cnn32_5l_test_time_sc.txt
diff --git a/image/90_class/cnn32_5l_test_time_st.txt → ...vision/90_class/cnn32_5l_test_time_st.txt b/image/90_class/cnn32_5l_test_time_st.txt → ...vision/90_class/cnn32_5l_test_time_st.txt
diff --git a/image/90_class/cnn32_5l_train_time.txt → ...k_vision/90_class/cnn32_5l_train_time.txt b/image/90_class/cnn32_5l_train_time.txt → ...k_vision/90_class/cnn32_5l_train_time.txt
diff --git a/image/90_class/cnn32_5l_train_time_sc.txt → ...ision/90_class/cnn32_5l_train_time_sc.txt b/image/90_class/cnn32_5l_train_time_sc.txt → ...ision/90_class/cnn32_5l_train_time_sc.txt
diff --git a/image/90_class/cnn32_5l_train_time_st.txt → ...ision/90_class/cnn32_5l_train_time_st.txt b/image/90_class/cnn32_5l_train_time_st.txt → ...ision/90_class/cnn32_5l_train_time_st.txt
diff --git a/image/90_class/cnn32_sc.txt → benchmark_vision/90_class/cnn32_sc.txt b/image/90_class/cnn32_sc.txt → benchmark_vision/90_class/cnn32_sc.txt
diff --git a/image/90_class/cnn32_st.txt → benchmark_vision/90_class/cnn32_st.txt b/image/90_class/cnn32_st.txt → benchmark_vision/90_class/cnn32_st.txt
diff --git a/image/90_class/cnn32_test_time.txt → ...hmark_vision/90_class/cnn32_test_time.txt b/image/90_class/cnn32_test_time.txt → ...hmark_vision/90_class/cnn32_test_time.txt
diff --git a/image/90_class/cnn32_test_time_sc.txt → ...rk_vision/90_class/cnn32_test_time_sc.txt b/image/90_class/cnn32_test_time_sc.txt → ...rk_vision/90_class/cnn32_test_time_sc.txt
diff --git a/image/90_class/cnn32_test_time_st.txt → ...rk_vision/90_class/cnn32_test_time_st.txt b/image/90_class/cnn32_test_time_st.txt → ...rk_vision/90_class/cnn32_test_time_st.txt
diff --git a/image/90_class/cnn32_train_time.txt → ...mark_vision/90_class/cnn32_train_time.txt b/image/90_class/cnn32_train_time.txt → ...mark_vision/90_class/cnn32_train_time.txt
diff --git a/image/90_class/cnn32_train_time_sc.txt → ...k_vision/90_class/cnn32_train_time_sc.txt b/image/90_class/cnn32_train_time_sc.txt → ...k_vision/90_class/cnn32_train_time_sc.txt
diff --git a/image/90_class/cnn32_train_time_st.txt → ...k_vision/90_class/cnn32_train_time_st.txt b/image/90_class/cnn32_train_time_st.txt → ...k_vision/90_class/cnn32_train_time_st.txt
diff --git a/image/90_class/naive_rf.txt → benchmark_vision/90_class/naive_rf.txt b/image/90_class/naive_rf.txt → benchmark_vision/90_class/naive_rf.txt
diff --git a/image/90_class/naive_rf_test_time.txt → ...rk_vision/90_class/naive_rf_test_time.txt b/image/90_class/naive_rf_test_time.txt → ...rk_vision/90_class/naive_rf_test_time.txt
diff --git a/image/90_class/naive_rf_test_time_lc.txt → ...vision/90_class/naive_rf_test_time_lc.txt b/image/90_class/naive_rf_test_time_lc.txt → ...vision/90_class/naive_rf_test_time_lc.txt
diff --git a/image/90_class/naive_rf_train_time.txt → ...k_vision/90_class/naive_rf_train_time.txt b/image/90_class/naive_rf_train_time.txt → ...k_vision/90_class/naive_rf_train_time.txt
diff --git a/image/90_class/naive_rf_train_time_lc.txt → ...ision/90_class/naive_rf_train_time_lc.txt b/image/90_class/naive_rf_train_time_lc.txt → ...ision/90_class/naive_rf_train_time_lc.txt
diff --git a/image/90_class/resnet18.txt → benchmark_vision/90_class/resnet18.txt b/image/90_class/resnet18.txt → benchmark_vision/90_class/resnet18.txt
diff --git a/image/90_class/resnet18_sc.txt → benchmark_vision/90_class/resnet18_sc.txt b/image/90_class/resnet18_sc.txt → benchmark_vision/90_class/resnet18_sc.txt
diff --git a/image/90_class/resnet18_st.txt → benchmark_vision/90_class/resnet18_st.txt b/image/90_class/resnet18_st.txt → benchmark_vision/90_class/resnet18_st.txt
diff --git a/image/90_class/resnet18_test_time.txt → ...rk_vision/90_class/resnet18_test_time.txt b/image/90_class/resnet18_test_time.txt → ...rk_vision/90_class/resnet18_test_time.txt
diff --git a/image/90_class/resnet18_test_time_sc.txt → ...vision/90_class/resnet18_test_time_sc.txt b/image/90_class/resnet18_test_time_sc.txt → ...vision/90_class/resnet18_test_time_sc.txt
diff --git a/image/90_class/resnet18_test_time_st.txt → ...vision/90_class/resnet18_test_time_st.txt b/image/90_class/resnet18_test_time_st.txt → ...vision/90_class/resnet18_test_time_st.txt
diff --git a/image/90_class/resnet18_train_time.txt → ...k_vision/90_class/resnet18_train_time.txt b/image/90_class/resnet18_train_time.txt → ...k_vision/90_class/resnet18_train_time.txt
diff --git a/image/90_class/resnet18_train_time_sc.txt → ...ision/90_class/resnet18_train_time_sc.txt b/image/90_class/resnet18_train_time_sc.txt → ...ision/90_class/resnet18_train_time_sc.txt
diff --git a/image/90_class/resnet18_train_time_st.txt → ...ision/90_class/resnet18_train_time_st.txt b/image/90_class/resnet18_train_time_st.txt → ...ision/90_class/resnet18_train_time_st.txt
diff --git a/image/90_class/svm.txt → benchmark_vision/90_class/svm.txt b/image/90_class/svm.txt → benchmark_vision/90_class/svm.txt
diff --git a/image/90_class/svm_test_time.txt → benchmark_vision/90_class/svm_test_time.txt b/image/90_class/svm_test_time.txt → benchmark_vision/90_class/svm_test_time.txt
diff --git a/image/90_class/svm_test_time_lc.txt → ...mark_vision/90_class/svm_test_time_lc.txt b/image/90_class/svm_test_time_lc.txt → ...mark_vision/90_class/svm_test_time_lc.txt
diff --git a/image/90_class/svm_train_time.txt → benchmark_vision/90_class/svm_train_time.txt b/image/90_class/svm_train_time.txt → benchmark_vision/90_class/svm_train_time.txt
diff --git a/image/90_class/svm_train_time_lc.txt → ...ark_vision/90_class/svm_train_time_lc.txt b/image/90_class/svm_train_time_lc.txt → ...ark_vision/90_class/svm_train_time_lc.txt
diff --git a/image/cifar_10.py → benchmark_vision/cifar_10.py b/image/cifar_10.py → benchmark_vision/cifar_10.py
@@ -1,6 +1,6 @@
 """
-Coauthors: Yu-Chung Peng
-           Haoyin Xu
+Coauthors: Haoyin Xu
+           Yu-Chung Peng
 """
 from toolbox import *
 
@@ -14,50 +14,16 @@
 import torchvision.transforms as transforms
 
 
-# prepare CIFAR data
-def main():
-    # Example usage: python cifar_10.py -m 3
-    parser = argparse.ArgumentParser()
-    parser.add_argument("-m", help="class number")
-    args = parser.parse_args()
-    n_classes = int(args.m)
-    prefix = args.m + "_class/"
-
-    nums = list(range(10))
-    random.shuffle(nums)
-    classes_space = list(combinations_45(nums, n_classes))
-
-    # normalize
-    scale = np.mean(np.arange(0, 256))
-    normalize = lambda x: (x - scale) / scale
-
-    # train data
-    cifar_trainset = datasets.CIFAR10(
-        root="./", train=True, download=True, transform=None
-    )
-    cifar_train_images = normalize(cifar_trainset.data)
-    cifar_train_labels = np.array(cifar_trainset.targets)
-
-    # test data
-    cifar_testset = datasets.CIFAR10(
-        root="./", train=False, download=True, transform=None
-    )
-    cifar_test_images = normalize(cifar_testset.data)
-    cifar_test_labels = np.array(cifar_testset.targets)
-
-    cifar_train_images = cifar_train_images.reshape(-1, 32 * 32 * 3)
-    cifar_test_images = cifar_test_images.reshape(-1, 32 * 32 * 3)
-
+def run_naive_rf():
     naive_rf_acc_vs_n = list()
     naive_rf_train_time = list()
     naive_rf_test_time = list()
     for classes in classes_space:
 
-        # accuracy vs num training samples (naive_rf)
-        samples_space = np.geomspace(10, 10000, num=8, dtype=int)
+        # cohen_kappa vs num training samples (naive_rf)
         for samples in samples_space:
             RF = RandomForestClassifier(n_estimators=100, n_jobs=-1)
-            accuracy, train_time, test_time = run_rf_image_set(
+            cohen_kappa, train_time, test_time = run_rf_image_set(
                 RF,
                 cifar_train_images,
                 cifar_train_labels,
@@ -66,7 +32,7 @@ def main():
                 samples,
                 classes,
             )
-            naive_rf_acc_vs_n.append(accuracy)
+            naive_rf_acc_vs_n.append(cohen_kappa)
             naive_rf_train_time.append(train_time)
             naive_rf_test_time.append(test_time)
 
@@ -75,16 +41,17 @@ def main():
     write_result(prefix + "naive_rf_train_time.txt", naive_rf_train_time)
     write_result(prefix + "naive_rf_test_time.txt", naive_rf_test_time)
 
+
+def run_svm():
     svm_acc_vs_n = list()
     svm_train_time = list()
     svm_test_time = list()
     for classes in classes_space:
 
-        # accuracy vs num training samples (svm)
-        samples_space = np.geomspace(10, 10000, num=8, dtype=int)
+        # cohen_kappa vs num training samples (svm)
         for samples in samples_space:
             SVM = SVC()
-            accuracy, train_time, test_time = run_rf_image_set(
+            cohen_kappa, train_time, test_time = run_rf_image_set(
                 SVM,
                 cifar_train_images,
                 cifar_train_labels,
@@ -93,7 +60,7 @@ def main():
                 samples,
                 classes,
             )
-            svm_acc_vs_n.append(accuracy)
+            svm_acc_vs_n.append(cohen_kappa)
             svm_train_time.append(train_time)
             svm_test_time.append(test_time)
 
@@ -102,17 +69,14 @@ def main():
     write_result(prefix + "svm_train_time.txt", svm_train_time)
     write_result(prefix + "svm_test_time.txt", svm_test_time)
 
-    data_transforms = transforms.Compose(
-        [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]
-    )
 
+def run_cnn32():
     cnn32_acc_vs_n = list()
     cnn32_train_time = list()
     cnn32_test_time = list()
     for classes in classes_space:
 
-        # accuracy vs num training samples (cnn32)
-        samples_space = np.geomspace(10, 10000, num=8, dtype=int)
+        # cohen_kappa vs num training samples (cnn32)
         for samples in samples_space:
             # train data
             cifar_trainset = datasets.CIFAR10(
@@ -135,13 +99,13 @@ def main():
                 cifar_testset,
                 samples,
             )
-            accuracy, train_time, test_time = run_dn_image_es(
+            cohen_kappa, train_time, test_time = run_dn_image_es(
                 cnn32,
                 train_loader,
                 valid_loader,
                 test_loader,
             )
-            cnn32_acc_vs_n.append(accuracy)
+            cnn32_acc_vs_n.append(cohen_kappa)
             cnn32_train_time.append(train_time)
             cnn32_test_time.append(test_time)
 
@@ -150,13 +114,14 @@ def main():
     write_result(prefix + "cnn32_train_time.txt", cnn32_train_time)
     write_result(prefix + "cnn32_test_time.txt", cnn32_test_time)
 
+
+def run_cnn32_2l():
     cnn32_2l_acc_vs_n = list()
     cnn32_2l_train_time = list()
     cnn32_2l_test_time = list()
     for classes in classes_space:
 
-        # accuracy vs num training samples (cnn32_2l)
-        samples_space = np.geomspace(10, 10000, num=8, dtype=int)
+        # cohen_kappa vs num training samples (cnn32_2l)
         for samples in samples_space:
             # train data
             cifar_trainset = datasets.CIFAR10(
@@ -179,13 +144,13 @@ def main():
                 cifar_testset,
                 samples,
             )
-            accuracy, train_time, test_time = run_dn_image_es(
+            cohen_kappa, train_time, test_time = run_dn_image_es(
                 cnn32_2l,
                 train_loader,
                 valid_loader,
                 test_loader,
             )
-            cnn32_2l_acc_vs_n.append(accuracy)
+            cnn32_2l_acc_vs_n.append(cohen_kappa)
             cnn32_2l_train_time.append(train_time)
             cnn32_2l_test_time.append(test_time)
 
@@ -194,13 +159,14 @@ def main():
     write_result(prefix + "cnn32_2l_train_time.txt", cnn32_2l_train_time)
     write_result(prefix + "cnn32_2l_test_time.txt", cnn32_2l_test_time)
 
+
+def run_cnn32_5l():
     cnn32_5l_acc_vs_n = list()
     cnn32_5l_train_time = list()
     cnn32_5l_test_time = list()
     for classes in classes_space:
 
-        # accuracy vs num training samples (cnn32_5l)
-        samples_space = np.geomspace(10, 10000, num=8, dtype=int)
+        # cohen_kappa vs num training samples (cnn32_5l)
         for samples in samples_space:
             # train data
             cifar_trainset = datasets.CIFAR10(
@@ -223,13 +189,13 @@ def main():
                 cifar_testset,
                 samples,
             )
-            accuracy, train_time, test_time = run_dn_image_es(
+            cohen_kappa, train_time, test_time = run_dn_image_es(
                 cnn32_5l,
                 train_loader,
                 valid_loader,
                 test_loader,
             )
-            cnn32_5l_acc_vs_n.append(accuracy)
+            cnn32_5l_acc_vs_n.append(cohen_kappa)
             cnn32_5l_train_time.append(train_time)
             cnn32_5l_test_time.append(test_time)
 
@@ -238,21 +204,14 @@ def main():
     write_result(prefix + "cnn32_5l_train_time.txt", cnn32_5l_train_time)
     write_result(prefix + "cnn32_5l_test_time.txt", cnn32_5l_test_time)
 
-    # prepare CIFAR data
-    data_transforms = transforms.Compose(
-        [
-            transforms.ToTensor(),
-            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
-        ]
-    )
 
+def run_resnet18():
     resnet18_acc_vs_n = list()
     resnet18_train_time = list()
     resnet18_test_time = list()
     for classes in classes_space:
 
-        # accuracy vs num training samples (resnet18)
-        samples_space = np.geomspace(10, 10000, num=8, dtype=int)
+        # cohen_kappa vs num training samples (resnet18)
         for samples in samples_space:
             # train data
             cifar_trainset = datasets.CIFAR10(
@@ -277,13 +236,13 @@ def main():
                 cifar_testset,
                 samples,
             )
-            accuracy, train_time, test_time = run_dn_image_es(
+            cohen_kappa, train_time, test_time = run_dn_image_es(
                 res,
                 train_loader,
                 valid_loader,
                 test_loader,
             )
-            resnet18_acc_vs_n.append(accuracy)
+            resnet18_acc_vs_n.append(cohen_kappa)
             resnet18_train_time.append(train_time)
             resnet18_test_time.append(test_time)
 
@@ -295,4 +254,56 @@ def main():
 
 if __name__ == "__main__":
     torch.multiprocessing.freeze_support()
-    main()
+
+    # Example usage: python cifar_10.py -m 3
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-m", help="class number")
+    args = parser.parse_args()
+    n_classes = int(args.m)
+    prefix = args.m + "_class/"
+    samples_space = np.geomspace(10, 10000, num=8, dtype=int)
+
+    nums = list(range(10))
+    random.shuffle(nums)
+    classes_space = list(combinations_45(nums, n_classes))
+
+    # normalize
+    scale = np.mean(np.arange(0, 256))
+    normalize = lambda x: (x - scale) / scale
+
+    # train data
+    cifar_trainset = datasets.CIFAR10(
+        root="./", train=True, download=True, transform=None
+    )
+    cifar_train_images = normalize(cifar_trainset.data)
+    cifar_train_labels = np.array(cifar_trainset.targets)
+
+    # test data
+    cifar_testset = datasets.CIFAR10(
+        root="./", train=False, download=True, transform=None
+    )
+    cifar_test_images = normalize(cifar_testset.data)
+    cifar_test_labels = np.array(cifar_testset.targets)
+
+    cifar_train_images = cifar_train_images.reshape(-1, 32 * 32 * 3)
+    cifar_test_images = cifar_test_images.reshape(-1, 32 * 32 * 3)
+
+    run_naive_rf()
+    run_svm()
+
+    data_transforms = transforms.Compose(
+        [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]
+    )
+
+    run_cnn32()
+    run_cnn32_2l()
+    run_cnn32_5l()
+
+    data_transforms = transforms.Compose(
+        [
+            transforms.ToTensor(),
+            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
+        ]
+    )
+
+    run_resnet18()