From 5010952ea08e19cbbd97520a79d0b7014dac3246 Mon Sep 17 00:00:00 2001 From: Haoyin Xu Date: Thu, 23 Sep 2021 11:35:42 -0400 Subject: [PATCH 1/5] EHN switch from accuracy to cohen's kappa --- image/cifar_10.py | 153 ++++++++++++---------- image/cifar_100.py | 153 ++++++++++++---------- image/cifar_100_stc.py | 120 +++++++++-------- image/cifar_10_stc.py | 120 +++++++++-------- image/svhn.py | 143 ++++++++++++--------- image/svhn_toolbox.py | 286 ++--------------------------------------- image/toolbox.py | 201 ++++------------------------- 7 files changed, 408 insertions(+), 768 deletions(-) diff --git a/image/cifar_10.py b/image/cifar_10.py index 0b95e7c3..8e5b199c 100644 --- a/image/cifar_10.py +++ b/image/cifar_10.py @@ -1,6 +1,6 @@ """ -Coauthors: Yu-Chung Peng - Haoyin Xu +Coauthors: Haoyin Xu + Yu-Chung Peng """ from toolbox import * @@ -14,50 +14,16 @@ import torchvision.transforms as transforms -# prepare CIFAR data -def main(): - # Example usage: python cifar_10.py -m 3 - parser = argparse.ArgumentParser() - parser.add_argument("-m", help="class number") - args = parser.parse_args() - n_classes = int(args.m) - prefix = args.m + "_class/" - - nums = list(range(10)) - random.shuffle(nums) - classes_space = list(combinations_45(nums, n_classes)) - - # normalize - scale = np.mean(np.arange(0, 256)) - normalize = lambda x: (x - scale) / scale - - # train data - cifar_trainset = datasets.CIFAR10( - root="./", train=True, download=True, transform=None - ) - cifar_train_images = normalize(cifar_trainset.data) - cifar_train_labels = np.array(cifar_trainset.targets) - - # test data - cifar_testset = datasets.CIFAR10( - root="./", train=False, download=True, transform=None - ) - cifar_test_images = normalize(cifar_testset.data) - cifar_test_labels = np.array(cifar_testset.targets) - - cifar_train_images = cifar_train_images.reshape(-1, 32 * 32 * 3) - cifar_test_images = cifar_test_images.reshape(-1, 32 * 32 * 3) - +def run_naive_rf(): naive_rf_acc_vs_n = list() naive_rf_train_time = list() naive_rf_test_time = list() for classes in classes_space: - # accuracy vs num training samples (naive_rf) - samples_space = np.geomspace(10, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (naive_rf) for samples in samples_space: RF = RandomForestClassifier(n_estimators=100, n_jobs=-1) - accuracy, train_time, test_time = run_rf_image_set( + cohen_kappa, train_time, test_time = run_rf_image_set( RF, cifar_train_images, cifar_train_labels, @@ -66,7 +32,7 @@ def main(): samples, classes, ) - naive_rf_acc_vs_n.append(accuracy) + naive_rf_acc_vs_n.append(cohen_kappa) naive_rf_train_time.append(train_time) naive_rf_test_time.append(test_time) @@ -75,16 +41,17 @@ def main(): write_result(prefix + "naive_rf_train_time.txt", naive_rf_train_time) write_result(prefix + "naive_rf_test_time.txt", naive_rf_test_time) + +def run_svm(): svm_acc_vs_n = list() svm_train_time = list() svm_test_time = list() for classes in classes_space: - # accuracy vs num training samples (svm) - samples_space = np.geomspace(10, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (svm) for samples in samples_space: SVM = SVC() - accuracy, train_time, test_time = run_rf_image_set( + cohen_kappa, train_time, test_time = run_rf_image_set( SVM, cifar_train_images, cifar_train_labels, @@ -93,7 +60,7 @@ def main(): samples, classes, ) - svm_acc_vs_n.append(accuracy) + svm_acc_vs_n.append(cohen_kappa) svm_train_time.append(train_time) svm_test_time.append(test_time) @@ -102,17 +69,14 @@ def main(): write_result(prefix + "svm_train_time.txt", svm_train_time) write_result(prefix + "svm_test_time.txt", svm_test_time) - data_transforms = transforms.Compose( - [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] - ) +def run_cnn32(): cnn32_acc_vs_n = list() cnn32_train_time = list() cnn32_test_time = list() for classes in classes_space: - # accuracy vs num training samples (cnn32) - samples_space = np.geomspace(10, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (cnn32) for samples in samples_space: # train data cifar_trainset = datasets.CIFAR10( @@ -135,13 +99,13 @@ def main(): cifar_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_es( + cohen_kappa, train_time, test_time = run_dn_image_es( cnn32, train_loader, valid_loader, test_loader, ) - cnn32_acc_vs_n.append(accuracy) + cnn32_acc_vs_n.append(cohen_kappa) cnn32_train_time.append(train_time) cnn32_test_time.append(test_time) @@ -150,13 +114,14 @@ def main(): write_result(prefix + "cnn32_train_time.txt", cnn32_train_time) write_result(prefix + "cnn32_test_time.txt", cnn32_test_time) + +def run_cnn32_2l(): cnn32_2l_acc_vs_n = list() cnn32_2l_train_time = list() cnn32_2l_test_time = list() for classes in classes_space: - # accuracy vs num training samples (cnn32_2l) - samples_space = np.geomspace(10, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (cnn32_2l) for samples in samples_space: # train data cifar_trainset = datasets.CIFAR10( @@ -179,13 +144,13 @@ def main(): cifar_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_es( + cohen_kappa, train_time, test_time = run_dn_image_es( cnn32_2l, train_loader, valid_loader, test_loader, ) - cnn32_2l_acc_vs_n.append(accuracy) + cnn32_2l_acc_vs_n.append(cohen_kappa) cnn32_2l_train_time.append(train_time) cnn32_2l_test_time.append(test_time) @@ -194,13 +159,14 @@ def main(): write_result(prefix + "cnn32_2l_train_time.txt", cnn32_2l_train_time) write_result(prefix + "cnn32_2l_test_time.txt", cnn32_2l_test_time) + +def run_cnn32_5l(): cnn32_5l_acc_vs_n = list() cnn32_5l_train_time = list() cnn32_5l_test_time = list() for classes in classes_space: - # accuracy vs num training samples (cnn32_5l) - samples_space = np.geomspace(10, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (cnn32_5l) for samples in samples_space: # train data cifar_trainset = datasets.CIFAR10( @@ -223,13 +189,13 @@ def main(): cifar_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_es( + cohen_kappa, train_time, test_time = run_dn_image_es( cnn32_5l, train_loader, valid_loader, test_loader, ) - cnn32_5l_acc_vs_n.append(accuracy) + cnn32_5l_acc_vs_n.append(cohen_kappa) cnn32_5l_train_time.append(train_time) cnn32_5l_test_time.append(test_time) @@ -238,21 +204,14 @@ def main(): write_result(prefix + "cnn32_5l_train_time.txt", cnn32_5l_train_time) write_result(prefix + "cnn32_5l_test_time.txt", cnn32_5l_test_time) - # prepare CIFAR data - data_transforms = transforms.Compose( - [ - transforms.ToTensor(), - transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), - ] - ) +def run_resnet18(): resnet18_acc_vs_n = list() resnet18_train_time = list() resnet18_test_time = list() for classes in classes_space: - # accuracy vs num training samples (resnet18) - samples_space = np.geomspace(10, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (resnet18) for samples in samples_space: # train data cifar_trainset = datasets.CIFAR10( @@ -277,13 +236,13 @@ def main(): cifar_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_es( + cohen_kappa, train_time, test_time = run_dn_image_es( res, train_loader, valid_loader, test_loader, ) - resnet18_acc_vs_n.append(accuracy) + resnet18_acc_vs_n.append(cohen_kappa) resnet18_train_time.append(train_time) resnet18_test_time.append(test_time) @@ -295,4 +254,56 @@ def main(): if __name__ == "__main__": torch.multiprocessing.freeze_support() - main() + + # Example usage: python cifar_10.py -m 3 + parser = argparse.ArgumentParser() + parser.add_argument("-m", help="class number") + args = parser.parse_args() + n_classes = int(args.m) + prefix = args.m + "_class/" + samples_space = np.geomspace(10, 10000, num=8, dtype=int) + + nums = list(range(10)) + random.shuffle(nums) + classes_space = list(combinations_45(nums, n_classes)) + + # normalize + scale = np.mean(np.arange(0, 256)) + normalize = lambda x: (x - scale) / scale + + # train data + cifar_trainset = datasets.CIFAR10( + root="./", train=True, download=True, transform=None + ) + cifar_train_images = normalize(cifar_trainset.data) + cifar_train_labels = np.array(cifar_trainset.targets) + + # test data + cifar_testset = datasets.CIFAR10( + root="./", train=False, download=True, transform=None + ) + cifar_test_images = normalize(cifar_testset.data) + cifar_test_labels = np.array(cifar_testset.targets) + + cifar_train_images = cifar_train_images.reshape(-1, 32 * 32 * 3) + cifar_test_images = cifar_test_images.reshape(-1, 32 * 32 * 3) + + run_naive_rf() + run_svm() + + data_transforms = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] + ) + + run_cnn32() + run_cnn32_2l() + run_cnn32_5l() + + data_transforms = transforms.Compose( + [ + transforms.ToTensor(), + transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), + ] + ) + + run_resnet18() diff --git a/image/cifar_100.py b/image/cifar_100.py index 0e693f0b..b0d0d3df 100644 --- a/image/cifar_100.py +++ b/image/cifar_100.py @@ -1,6 +1,6 @@ """ -Coauthors: Yu-Chung Peng - Haoyin Xu +Coauthors: Haoyin Xu + Yu-Chung Peng """ from toolbox import * @@ -14,50 +14,16 @@ import torchvision.transforms as transforms -# prepare CIFAR data -def main(): - # Example usage: python cifar_100.py -m 90 - parser = argparse.ArgumentParser() - parser.add_argument("-m", help="class number") - args = parser.parse_args() - n_classes = int(args.m) - prefix = args.m + "_class/" - - nums = list(range(100)) - random.shuffle(nums) - classes_space = list(combinations_45(nums, n_classes)) - - # normalize - scale = np.mean(np.arange(0, 256)) - normalize = lambda x: (x - scale) / scale - - # train data - cifar_trainset = datasets.CIFAR100( - root="./", train=True, download=True, transform=None - ) - cifar_train_images = normalize(cifar_trainset.data) - cifar_train_labels = np.array(cifar_trainset.targets) - - # test data - cifar_testset = datasets.CIFAR100( - root="./", train=False, download=True, transform=None - ) - cifar_test_images = normalize(cifar_testset.data) - cifar_test_labels = np.array(cifar_testset.targets) - - cifar_train_images = cifar_train_images.reshape(-1, 32 * 32 * 3) - cifar_test_images = cifar_test_images.reshape(-1, 32 * 32 * 3) - +def run_naive_rf(): naive_rf_acc_vs_n = list() naive_rf_train_time = list() naive_rf_test_time = list() for classes in classes_space: - # accuracy vs num training samples (naive_rf) - samples_space = np.geomspace(100, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (naive_rf) for samples in samples_space: RF = RandomForestClassifier(n_estimators=100, n_jobs=-1) - accuracy, train_time, test_time = run_rf_image_set( + cohen_kappa, train_time, test_time = run_rf_image_set( RF, cifar_train_images, cifar_train_labels, @@ -66,7 +32,7 @@ def main(): samples, classes, ) - naive_rf_acc_vs_n.append(accuracy) + naive_rf_acc_vs_n.append(cohen_kappa) naive_rf_train_time.append(train_time) naive_rf_test_time.append(test_time) @@ -75,16 +41,17 @@ def main(): write_result(prefix + "naive_rf_train_time.txt", naive_rf_train_time) write_result(prefix + "naive_rf_test_time.txt", naive_rf_test_time) + +def run_svm(): svm_acc_vs_n = list() svm_train_time = list() svm_test_time = list() for classes in classes_space: - # accuracy vs num training samples (svm) - samples_space = np.geomspace(100, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (svm) for samples in samples_space: SVM = SVC() - accuracy, train_time, test_time = run_rf_image_set( + cohen_kappa, train_time, test_time = run_rf_image_set( SVM, cifar_train_images, cifar_train_labels, @@ -93,7 +60,7 @@ def main(): samples, classes, ) - svm_acc_vs_n.append(accuracy) + svm_acc_vs_n.append(cohen_kappa) svm_train_time.append(train_time) svm_test_time.append(test_time) @@ -102,17 +69,14 @@ def main(): write_result(prefix + "svm_train_time.txt", svm_train_time) write_result(prefix + "svm_test_time.txt", svm_test_time) - data_transforms = transforms.Compose( - [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] - ) +def run_cnn32(): cnn32_acc_vs_n = list() cnn32_train_time = list() cnn32_test_time = list() for classes in classes_space: - # accuracy vs num training samples (cnn32) - samples_space = np.geomspace(100, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (cnn32) for samples in samples_space: # train data cifar_trainset = datasets.CIFAR100( @@ -135,13 +99,13 @@ def main(): cifar_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_es( + cohen_kappa, train_time, test_time = run_dn_image_es( cnn32, train_loader, valid_loader, test_loader, ) - cnn32_acc_vs_n.append(accuracy) + cnn32_acc_vs_n.append(cohen_kappa) cnn32_train_time.append(train_time) cnn32_test_time.append(test_time) @@ -150,13 +114,14 @@ def main(): write_result(prefix + "cnn32_train_time.txt", cnn32_train_time) write_result(prefix + "cnn32_test_time.txt", cnn32_test_time) + +def run_cnn32_2l(): cnn32_2l_acc_vs_n = list() cnn32_2l_train_time = list() cnn32_2l_test_time = list() for classes in classes_space: - # accuracy vs num training samples (cnn32_2l) - samples_space = np.geomspace(100, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (cnn32_2l) for samples in samples_space: # train data cifar_trainset = datasets.CIFAR100( @@ -179,13 +144,13 @@ def main(): cifar_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_es( + cohen_kappa, train_time, test_time = run_dn_image_es( cnn32_2l, train_loader, valid_loader, test_loader, ) - cnn32_2l_acc_vs_n.append(accuracy) + cnn32_2l_acc_vs_n.append(cohen_kappa) cnn32_2l_train_time.append(train_time) cnn32_2l_test_time.append(test_time) @@ -194,13 +159,14 @@ def main(): write_result(prefix + "cnn32_2l_train_time.txt", cnn32_2l_train_time) write_result(prefix + "cnn32_2l_test_time.txt", cnn32_2l_test_time) + +def run_cnn32_5l(): cnn32_5l_acc_vs_n = list() cnn32_5l_train_time = list() cnn32_5l_test_time = list() for classes in classes_space: - # accuracy vs num training samples (cnn32_5l) - samples_space = np.geomspace(100, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (cnn32_5l) for samples in samples_space: # train data cifar_trainset = datasets.CIFAR100( @@ -223,13 +189,13 @@ def main(): cifar_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_es( + cohen_kappa, train_time, test_time = run_dn_image_es( cnn32_5l, train_loader, valid_loader, test_loader, ) - cnn32_5l_acc_vs_n.append(accuracy) + cnn32_5l_acc_vs_n.append(cohen_kappa) cnn32_5l_train_time.append(train_time) cnn32_5l_test_time.append(test_time) @@ -238,21 +204,14 @@ def main(): write_result(prefix + "cnn32_5l_train_time.txt", cnn32_5l_train_time) write_result(prefix + "cnn32_5l_test_time.txt", cnn32_5l_test_time) - # prepare CIFAR data - data_transforms = transforms.Compose( - [ - transforms.ToTensor(), - transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), - ] - ) +def run_resnet18(): resnet18_acc_vs_n = list() resnet18_train_time = list() resnet18_test_time = list() for classes in classes_space: - # accuracy vs num training samples (resnet18) - samples_space = np.geomspace(100, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (resnet18) for samples in samples_space: # train data cifar_trainset = datasets.CIFAR100( @@ -277,13 +236,13 @@ def main(): cifar_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_es( + cohen_kappa, train_time, test_time = run_dn_image_es( res, train_loader, valid_loader, test_loader, ) - resnet18_acc_vs_n.append(accuracy) + resnet18_acc_vs_n.append(cohen_kappa) resnet18_train_time.append(train_time) resnet18_test_time.append(test_time) @@ -295,4 +254,56 @@ def main(): if __name__ == "__main__": torch.multiprocessing.freeze_support() - main() + + # Example usage: python cifar_100.py -m 90 + parser = argparse.ArgumentParser() + parser.add_argument("-m", help="class number") + args = parser.parse_args() + n_classes = int(args.m) + prefix = args.m + "_class/" + samples_space = np.geomspace(100, 10000, num=8, dtype=int) + + nums = list(range(100)) + random.shuffle(nums) + classes_space = list(combinations_45(nums, n_classes)) + + # normalize + scale = np.mean(np.arange(0, 256)) + normalize = lambda x: (x - scale) / scale + + # train data + cifar_trainset = datasets.CIFAR100( + root="./", train=True, download=True, transform=None + ) + cifar_train_images = normalize(cifar_trainset.data) + cifar_train_labels = np.array(cifar_trainset.targets) + + # test data + cifar_testset = datasets.CIFAR100( + root="./", train=False, download=True, transform=None + ) + cifar_test_images = normalize(cifar_testset.data) + cifar_test_labels = np.array(cifar_testset.targets) + + cifar_train_images = cifar_train_images.reshape(-1, 32 * 32 * 3) + cifar_test_images = cifar_test_images.reshape(-1, 32 * 32 * 3) + + run_naive_rf() + run_svm() + + data_transforms = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] + ) + + run_cnn32() + run_cnn32_2l() + run_cnn32_5l() + + data_transforms = transforms.Compose( + [ + transforms.ToTensor(), + transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), + ] + ) + + run_resnet18() diff --git a/image/cifar_100_stc.py b/image/cifar_100_stc.py index 6495deff..93777d4d 100644 --- a/image/cifar_100_stc.py +++ b/image/cifar_100_stc.py @@ -1,6 +1,6 @@ """ -Coauthors: Yu-Chung Peng - Haoyin Xu +Coauthors: Haoyin Xu + Yu-Chung Peng """ from toolbox import * @@ -12,44 +12,13 @@ import torchvision.transforms as transforms -# prepare CIFAR data -def main(): - # Example usage: python cifar_100.py -m 90 -s l - parser = argparse.ArgumentParser() - parser.add_argument("-m", help="class number") - parser.add_argument("-s", help="computation speed") - args = parser.parse_args() - n_classes = int(args.m) - prefix = args.m + "_class/" - - nums = list(range(100)) - random.shuffle(nums) - classes_space = list(combinations_45(nums, n_classes)) - - if args.s == "h": - # High speed RF - rf_times = produce_mean(load_result(prefix + "naive_rf_train_time.txt")) - suffix = "_st.txt" - ratio = 1.0 - elif args.s == "l": - # Low speed RF - rf_times = produce_mean(load_result(prefix + "naive_rf_train_time_lc.txt")) - suffix = "_sc.txt" - ratio = 0.11 / 0.9 - else: - raise Exception("Wrong configurations for time calibration.") - - data_transforms = transforms.Compose( - [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] - ) - +def run_cnn32(): cnn32_acc_vs_n = list() cnn32_train_time = list() cnn32_test_time = list() for classes in classes_space: - # accuracy vs num training samples (cnn32) - samples_space = np.geomspace(100, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (cnn32) for i, samples in enumerate(samples_space): # train data cifar_trainset = datasets.CIFAR100( @@ -73,14 +42,14 @@ def main(): cifar_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_set( + cohen_kappa, train_time, test_time = run_dn_image_set( cnn32, train_loader, test_loader, time_limit=time_limit, ratio=ratio, ) - cnn32_acc_vs_n.append(accuracy) + cnn32_acc_vs_n.append(cohen_kappa) cnn32_train_time.append(train_time) cnn32_test_time.append(test_time) @@ -89,13 +58,14 @@ def main(): write_result(prefix + "cnn32_train_time" + suffix, cnn32_train_time) write_result(prefix + "cnn32_test_time" + suffix, cnn32_test_time) + +def run_cnn32_2l(): cnn32_2l_acc_vs_n = list() cnn32_2l_train_time = list() cnn32_2l_test_time = list() for classes in classes_space: - # accuracy vs num training samples (cnn32_2l) - samples_space = np.geomspace(100, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (cnn32_2l) for i, samples in enumerate(samples_space): # train data cifar_trainset = datasets.CIFAR100( @@ -119,14 +89,14 @@ def main(): cifar_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_set( + cohen_kappa, train_time, test_time = run_dn_image_set( cnn32_2l, train_loader, test_loader, time_limit=time_limit, ratio=ratio, ) - cnn32_2l_acc_vs_n.append(accuracy) + cnn32_2l_acc_vs_n.append(cohen_kappa) cnn32_2l_train_time.append(train_time) cnn32_2l_test_time.append(test_time) @@ -135,13 +105,14 @@ def main(): write_result(prefix + "cnn32_2l_train_time" + suffix, cnn32_2l_train_time) write_result(prefix + "cnn32_2l_test_time" + suffix, cnn32_2l_test_time) + +def run_cnn32_5l(): cnn32_5l_acc_vs_n = list() cnn32_5l_train_time = list() cnn32_5l_test_time = list() for classes in classes_space: - # accuracy vs num training samples (cnn32_5l) - samples_space = np.geomspace(100, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (cnn32_5l) for i, samples in enumerate(samples_space): # train data cifar_trainset = datasets.CIFAR100( @@ -165,14 +136,14 @@ def main(): cifar_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_set( + cohen_kappa, train_time, test_time = run_dn_image_set( cnn32_5l, train_loader, test_loader, time_limit=time_limit, ratio=ratio, ) - cnn32_5l_acc_vs_n.append(accuracy) + cnn32_5l_acc_vs_n.append(cohen_kappa) cnn32_5l_train_time.append(train_time) cnn32_5l_test_time.append(test_time) @@ -181,21 +152,14 @@ def main(): write_result(prefix + "cnn32_5l_train_time" + suffix, cnn32_5l_train_time) write_result(prefix + "cnn32_5l_test_time" + suffix, cnn32_5l_test_time) - # prepare CIFAR data - data_transforms = transforms.Compose( - [ - transforms.ToTensor(), - transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), - ] - ) +def run_resnet18(): resnet18_acc_vs_n = list() resnet18_train_time = list() resnet18_test_time = list() for classes in classes_space: - # accuracy vs num training samples (resnet18) - samples_space = np.geomspace(100, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (resnet18) for i, samples in enumerate(samples_space): # train data cifar_trainset = datasets.CIFAR100( @@ -221,14 +185,14 @@ def main(): cifar_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_set( + cohen_kappa, train_time, test_time = run_dn_image_set( res, train_loader, test_loader, time_limit=time_limit, ratio=ratio, ) - resnet18_acc_vs_n.append(accuracy) + resnet18_acc_vs_n.append(cohen_kappa) resnet18_train_time.append(train_time) resnet18_test_time.append(test_time) @@ -240,4 +204,46 @@ def main(): if __name__ == "__main__": torch.multiprocessing.freeze_support() - main() + + # Example usage: python cifar_100.py -m 90 -s l + parser = argparse.ArgumentParser() + parser.add_argument("-m", help="class number") + parser.add_argument("-s", help="computation speed") + args = parser.parse_args() + n_classes = int(args.m) + prefix = args.m + "_class/" + samples_space = np.geomspace(100, 10000, num=8, dtype=int) + + nums = list(range(100)) + random.shuffle(nums) + classes_space = list(combinations_45(nums, n_classes)) + + if args.s == "h": + # High speed RF + rf_times = produce_mean(load_result(prefix + "naive_rf_train_time.txt")) + suffix = "_st.txt" + ratio = 1.0 + elif args.s == "l": + # Low speed RF + rf_times = produce_mean(load_result(prefix + "naive_rf_train_time_lc.txt")) + suffix = "_sc.txt" + ratio = 0.11 / 0.9 + else: + raise Exception("Wrong configurations for time calibration.") + + data_transforms = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] + ) + + run_cnn32() + run_cnn32_2l() + run_cnn32_5l() + + data_transforms = transforms.Compose( + [ + transforms.ToTensor(), + transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), + ] + ) + + run_resnet18() diff --git a/image/cifar_10_stc.py b/image/cifar_10_stc.py index b6f5d627..576b6ed9 100644 --- a/image/cifar_10_stc.py +++ b/image/cifar_10_stc.py @@ -1,6 +1,6 @@ """ -Coauthors: Yu-Chung Peng - Haoyin Xu +Coauthors: Haoyin Xu + Yu-Chung Peng """ from toolbox import * @@ -12,44 +12,13 @@ import torchvision.transforms as transforms -# prepare CIFAR data -def main(): - # Example usage: python cifar_10.py -m 3 -s l - parser = argparse.ArgumentParser() - parser.add_argument("-m", help="class number") - parser.add_argument("-s", help="computation speed") - args = parser.parse_args() - n_classes = int(args.m) - prefix = args.m + "_class/" - - nums = list(range(10)) - random.shuffle(nums) - classes_space = list(combinations_45(nums, n_classes)) - - if args.s == "h": - # High speed RF - rf_times = produce_mean(load_result(prefix + "naive_rf_train_time.txt")) - suffix = "_st.txt" - ratio = 1.0 - elif args.s == "l": - # Low speed RF - rf_times = produce_mean(load_result(prefix + "naive_rf_train_time_lc.txt")) - suffix = "_sc.txt" - ratio = 0.11 / 0.9 - else: - raise Exception("Wrong configurations for time calibration.") - - data_transforms = transforms.Compose( - [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] - ) - +def run_cnn32(): cnn32_acc_vs_n = list() cnn32_train_time = list() cnn32_test_time = list() for classes in classes_space: - # accuracy vs num training samples (cnn32) - samples_space = np.geomspace(10, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (cnn32) for i, samples in enumerate(samples_space): # train data cifar_trainset = datasets.CIFAR10( @@ -73,14 +42,14 @@ def main(): cifar_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_set( + cohen_kappa, train_time, test_time = run_dn_image_set( cnn32, train_loader, test_loader, time_limit=time_limit, ratio=ratio, ) - cnn32_acc_vs_n.append(accuracy) + cnn32_acc_vs_n.append(cohen_kappa) cnn32_train_time.append(train_time) cnn32_test_time.append(test_time) @@ -89,13 +58,14 @@ def main(): write_result(prefix + "cnn32_train_time" + suffix, cnn32_train_time) write_result(prefix + "cnn32_test_time" + suffix, cnn32_test_time) + +def run_cnn32_2l(): cnn32_2l_acc_vs_n = list() cnn32_2l_train_time = list() cnn32_2l_test_time = list() for classes in classes_space: - # accuracy vs num training samples (cnn32_2l) - samples_space = np.geomspace(10, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (cnn32_2l) for i, samples in enumerate(samples_space): # train data cifar_trainset = datasets.CIFAR10( @@ -119,14 +89,14 @@ def main(): cifar_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_set( + cohen_kappa, train_time, test_time = run_dn_image_set( cnn32_2l, train_loader, test_loader, time_limit=time_limit, ratio=ratio, ) - cnn32_2l_acc_vs_n.append(accuracy) + cnn32_2l_acc_vs_n.append(cohen_kappa) cnn32_2l_train_time.append(train_time) cnn32_2l_test_time.append(test_time) @@ -135,13 +105,14 @@ def main(): write_result(prefix + "cnn32_2l_train_time" + suffix, cnn32_2l_train_time) write_result(prefix + "cnn32_2l_test_time" + suffix, cnn32_2l_test_time) + +def run_cnn32_5l(): cnn32_5l_acc_vs_n = list() cnn32_5l_train_time = list() cnn32_5l_test_time = list() for classes in classes_space: - # accuracy vs num training samples (cnn32_5l) - samples_space = np.geomspace(10, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (cnn32_5l) for i, samples in enumerate(samples_space): # train data cifar_trainset = datasets.CIFAR10( @@ -165,14 +136,14 @@ def main(): cifar_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_set( + cohen_kappa, train_time, test_time = run_dn_image_set( cnn32_5l, train_loader, test_loader, time_limit=time_limit, ratio=ratio, ) - cnn32_5l_acc_vs_n.append(accuracy) + cnn32_5l_acc_vs_n.append(cohen_kappa) cnn32_5l_train_time.append(train_time) cnn32_5l_test_time.append(test_time) @@ -181,21 +152,14 @@ def main(): write_result(prefix + "cnn32_5l_train_time" + suffix, cnn32_5l_train_time) write_result(prefix + "cnn32_5l_test_time" + suffix, cnn32_5l_test_time) - # prepare CIFAR data - data_transforms = transforms.Compose( - [ - transforms.ToTensor(), - transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), - ] - ) +def run_resnet18(): resnet18_acc_vs_n = list() resnet18_train_time = list() resnet18_test_time = list() for classes in classes_space: - # accuracy vs num training samples (resnet18) - samples_space = np.geomspace(10, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (resnet18) for i, samples in enumerate(samples_space): # train data cifar_trainset = datasets.CIFAR10( @@ -221,14 +185,14 @@ def main(): cifar_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_set( + cohen_kappa, train_time, test_time = run_dn_image_set( res, train_loader, test_loader, time_limit=time_limit, ratio=ratio, ) - resnet18_acc_vs_n.append(accuracy) + resnet18_acc_vs_n.append(cohen_kappa) resnet18_train_time.append(train_time) resnet18_test_time.append(test_time) @@ -240,4 +204,46 @@ def main(): if __name__ == "__main__": torch.multiprocessing.freeze_support() - main() + + # Example usage: python cifar_10.py -m 3 -s l + parser = argparse.ArgumentParser() + parser.add_argument("-m", help="class number") + parser.add_argument("-s", help="computation speed") + args = parser.parse_args() + n_classes = int(args.m) + prefix = args.m + "_class/" + samples_space = np.geomspace(10, 10000, num=8, dtype=int) + + nums = list(range(10)) + random.shuffle(nums) + classes_space = list(combinations_45(nums, n_classes)) + + if args.s == "h": + # High speed RF + rf_times = produce_mean(load_result(prefix + "naive_rf_train_time.txt")) + suffix = "_st.txt" + ratio = 1.0 + elif args.s == "l": + # Low speed RF + rf_times = produce_mean(load_result(prefix + "naive_rf_train_time_lc.txt")) + suffix = "_sc.txt" + ratio = 0.11 / 0.9 + else: + raise Exception("Wrong configurations for time calibration.") + + data_transforms = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] + ) + + run_cnn32() + run_cnn32_2l() + run_cnn32_5l() + + data_transforms = transforms.Compose( + [ + transforms.ToTensor(), + transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), + ] + ) + + run_resnet18() diff --git a/image/svhn.py b/image/svhn.py index 3fce3efa..8ae1bf35 100644 --- a/image/svhn.py +++ b/image/svhn.py @@ -1,6 +1,6 @@ """ -Coauthors: Yu-Chung Peng - Haoyin Xu +Coauthors: Haoyin Xu + Yu-Chung Peng """ from svhn_toolbox import * @@ -14,47 +14,16 @@ import torchvision.transforms as transforms -# prepare CIFAR data -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("-m", help="class number") - args = parser.parse_args() - n_classes = int(args.m) - prefix = args.m + "_class/svhn_" - - nums = list(range(10)) - random.shuffle(nums) - classes_space = list(combinations_45(nums, n_classes)) - - # normalize - scale = np.mean(np.arange(0, 256)) - normalize = lambda x: (x - scale) / scale - - # train data - svhn_trainset = datasets.SVHN( - root="./", split="train", download=True, transform=None - ) - svhn_train_images = normalize(svhn_trainset.data) - svhn_train_labels = np.array(svhn_trainset.labels) - - # test data - svhn_testset = datasets.SVHN(root="./", split="test", download=True, transform=None) - svhn_test_images = normalize(svhn_testset.data) - svhn_test_labels = np.array(svhn_testset.labels) - - svhn_train_images = svhn_train_images.reshape(-1, 32 * 32 * 3) - svhn_test_images = svhn_test_images.reshape(-1, 32 * 32 * 3) - +def run_naive_rf(): naive_rf_acc_vs_n = list() naive_rf_train_time = list() naive_rf_test_time = list() for classes in classes_space: - # accuracy vs num training samples (naive_rf) - samples_space = np.geomspace(10, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (naive_rf) for samples in samples_space: RF = RandomForestClassifier(n_estimators=100, n_jobs=-1) - accuracy, train_time, test_time = run_rf_image_set( + cohen_kappa, train_time, test_time = run_rf_image_set( RF, svhn_train_images, svhn_train_labels, @@ -63,7 +32,7 @@ def main(): samples, classes, ) - naive_rf_acc_vs_n.append(accuracy) + naive_rf_acc_vs_n.append(cohen_kappa) naive_rf_train_time.append(train_time) naive_rf_test_time.append(test_time) @@ -72,16 +41,17 @@ def main(): write_result(prefix + "naive_rf_train_time.txt", naive_rf_train_time) write_result(prefix + "naive_rf_test_time.txt", naive_rf_test_time) + +def run_svm(): svm_acc_vs_n = list() svm_train_time = list() svm_test_time = list() for classes in classes_space: - # accuracy vs num training samples (svm) - samples_space = np.geomspace(10, 10000, num=8, dtype=int) + # cohen_kappa vs num training samples (svm) for samples in samples_space: SVM = SVC() - accuracy, train_time, test_time = run_rf_image_set( + cohen_kappa, train_time, test_time = run_rf_image_set( SVM, svhn_train_images, svhn_train_labels, @@ -90,7 +60,7 @@ def main(): samples, classes, ) - svm_acc_vs_n.append(accuracy) + svm_acc_vs_n.append(cohen_kappa) svm_train_time.append(train_time) svm_test_time.append(test_time) @@ -99,16 +69,14 @@ def main(): write_result(prefix + "svm_train_time.txt", svm_train_time) write_result(prefix + "svm_test_time.txt", svm_test_time) - data_transforms = transforms.Compose( - [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] - ) +def run_cnn32(): cnn32_acc_vs_n = list() cnn32_train_time = list() cnn32_test_time = list() for classes in classes_space: - # accuracy vs num training samples (cnn32) + # cohen_kappa vs num training samples (cnn32) samples_space = np.geomspace(10, 10000, num=8, dtype=int) for samples in samples_space: # train data @@ -132,13 +100,13 @@ def main(): svhn_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_es( + cohen_kappa, train_time, test_time = run_dn_image_es( cnn32, train_loader, valid_loader, test_loader, ) - cnn32_acc_vs_n.append(accuracy) + cnn32_acc_vs_n.append(cohen_kappa) cnn32_train_time.append(train_time) cnn32_test_time.append(test_time) @@ -147,12 +115,14 @@ def main(): write_result(prefix + "cnn32_train_time.txt", cnn32_train_time) write_result(prefix + "cnn32_test_time.txt", cnn32_test_time) + +def run_cnn32_2l(): cnn32_2l_acc_vs_n = list() cnn32_2l_train_time = list() cnn32_2l_test_time = list() for classes in classes_space: - # accuracy vs num training samples (cnn32_2l) + # cohen_kappa vs num training samples (cnn32_2l) samples_space = np.geomspace(10, 10000, num=8, dtype=int) for samples in samples_space: # train data @@ -176,13 +146,13 @@ def main(): svhn_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_es( + cohen_kappa, train_time, test_time = run_dn_image_es( cnn32_2l, train_loader, valid_loader, test_loader, ) - cnn32_2l_acc_vs_n.append(accuracy) + cnn32_2l_acc_vs_n.append(cohen_kappa) cnn32_2l_train_time.append(train_time) cnn32_2l_test_time.append(test_time) @@ -191,12 +161,14 @@ def main(): write_result(prefix + "cnn32_2l_train_time.txt", cnn32_2l_train_time) write_result(prefix + "cnn32_2l_test_time.txt", cnn32_2l_test_time) + +def run_cnn32_5l(): cnn32_5l_acc_vs_n = list() cnn32_5l_train_time = list() cnn32_5l_test_time = list() for classes in classes_space: - # accuracy vs num training samples (cnn32_5l) + # cohen_kappa vs num training samples (cnn32_5l) samples_space = np.geomspace(10, 10000, num=8, dtype=int) for samples in samples_space: # train data @@ -220,13 +192,13 @@ def main(): svhn_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_es( + cohen_kappa, train_time, test_time = run_dn_image_es( cnn32_5l, train_loader, valid_loader, test_loader, ) - cnn32_5l_acc_vs_n.append(accuracy) + cnn32_5l_acc_vs_n.append(cohen_kappa) cnn32_5l_train_time.append(train_time) cnn32_5l_test_time.append(test_time) @@ -235,20 +207,14 @@ def main(): write_result(prefix + "cnn32_5l_train_time.txt", cnn32_5l_train_time) write_result(prefix + "cnn32_5l_test_time.txt", cnn32_5l_test_time) - # prepare CIFAR data - data_transforms = transforms.Compose( - [ - transforms.ToTensor(), - transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), - ] - ) +def run_resnet18(): resnet18_acc_vs_n = list() resnet18_train_time = list() resnet18_test_time = list() for classes in classes_space: - # accuracy vs num training samples (resnet18) + # cohen_kappa vs num training samples (resnet18) samples_space = np.geomspace(10, 10000, num=8, dtype=int) for samples in samples_space: # train data @@ -274,13 +240,13 @@ def main(): svhn_testset, samples, ) - accuracy, train_time, test_time = run_dn_image_es( + cohen_kappa, train_time, test_time = run_dn_image_es( res, train_loader, valid_loader, test_loader, ) - resnet18_acc_vs_n.append(accuracy) + resnet18_acc_vs_n.append(cohen_kappa) resnet18_train_time.append(train_time) resnet18_test_time.append(test_time) @@ -292,4 +258,53 @@ def main(): if __name__ == "__main__": torch.multiprocessing.freeze_support() - main() + + parser = argparse.ArgumentParser() + parser.add_argument("-m", help="class number") + args = parser.parse_args() + n_classes = int(args.m) + prefix = args.m + "_class/svhn_" + samples_space = np.geomspace(10, 10000, num=8, dtype=int) + + nums = list(range(10)) + random.shuffle(nums) + classes_space = list(combinations_45(nums, n_classes)) + + # normalize + scale = np.mean(np.arange(0, 256)) + normalize = lambda x: (x - scale) / scale + + # train data + svhn_trainset = datasets.SVHN( + root="./", split="train", download=True, transform=None + ) + svhn_train_images = normalize(svhn_trainset.data) + svhn_train_labels = np.array(svhn_trainset.labels) + + # test data + svhn_testset = datasets.SVHN(root="./", split="test", download=True, transform=None) + svhn_test_images = normalize(svhn_testset.data) + svhn_test_labels = np.array(svhn_testset.labels) + + svhn_train_images = svhn_train_images.reshape(-1, 32 * 32 * 3) + svhn_test_images = svhn_test_images.reshape(-1, 32 * 32 * 3) + + run_naive_rf() + run_svm() + + data_transforms = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] + ) + + run_cnn32() + run_cnn32_2l() + run_cnn32_5l() + + data_transforms = transforms.Compose( + [ + transforms.ToTensor(), + transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), + ] + ) + + run_resnet18() diff --git a/image/svhn_toolbox.py b/image/svhn_toolbox.py index 1576e357..fbd3a0a1 100644 --- a/image/svhn_toolbox.py +++ b/image/svhn_toolbox.py @@ -1,281 +1,23 @@ """ -Coauthors: Yu-Chung Peng - Haoyin Xu +Coauthors: Haoyin Xu + Yu-Chung Peng """ -import time import numpy as np -from sklearn.metrics import accuracy_score +from sklearn.metrics import cohen_kappa_score import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim - -class SimpleCNN32Filter(nn.Module): - """ - Defines a simple CNN arhcitecture - """ - - def __init__(self, num_classes): - super(SimpleCNN32Filter, self).__init__() - self.conv1 = nn.Conv2d(3, 32, kernel_size=10, stride=2) - self.fc1 = nn.Linear(144 * 32, num_classes) - - def forward(self, x): - x = F.relu(self.conv1(x)) - x = x.view(-1, 144 * 32) - x = self.fc1(x) - return x - - -class SimpleCNN32Filter2Layers(nn.Module): - """ - Define a simple CNN arhcitecture with 2 layers - """ - - def __init__(self, num_classes): - super(SimpleCNN32Filter2Layers, self).__init__() - self.conv1 = nn.Conv2d(3, 32, kernel_size=5, stride=1) - self.conv2 = nn.Conv2d(32, 32, kernel_size=5, stride=2) - self.fc1 = nn.Linear(12 * 12 * 32, 100) - self.fc2 = nn.Linear(100, num_classes) - - def forward(self, x): - b = x.shape[0] - x = F.relu(self.conv1(x)) - x = F.relu(self.conv2(x)) - x = x.view(b, -1) - x = F.relu(self.fc1(x)) - x = self.fc2(x) - return x - - -class SimpleCNN32Filter5Layers(torch.nn.Module): - """ - Define a simple CNN arhcitecture with 5 layers - """ - - def __init__(self, num_classes): - super(SimpleCNN32Filter5Layers, self).__init__() - self.conv1 = torch.nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1) - self.conv2 = torch.nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1) - self.conv3 = torch.nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1) - self.conv4 = torch.nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1) - self.conv5 = torch.nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1) - self.fc1 = torch.nn.Linear(8192, 200) - self.fc2 = torch.nn.Linear(200, num_classes) - self.maxpool = nn.MaxPool2d((2, 2)) - self.bn = nn.BatchNorm2d(32) - self.bn2 = nn.BatchNorm2d(64) - self.bn3 = nn.BatchNorm2d(128) - - def forward(self, x): - b = x.shape[0] - x = F.relu(self.bn(self.conv1(x))) - x = F.relu(self.bn(self.conv2(x))) - x = self.maxpool(x) - x = F.relu(self.bn2(self.conv3(x))) - x = F.relu(self.bn2(self.conv4(x))) - x = F.relu(self.bn3(self.conv5(x))) - x = self.maxpool(x) - x = x.view(b, -1) - x = F.relu(self.fc1(x)) - x = self.fc2(x) - return x - - -def write_result(filename, acc_ls): - """Writes results to specified text file""" - output = open(filename, "w") - for acc in acc_ls: - output.write(str(acc) + "\n") - - -def combinations_45(iterable, r): - """Extracts 45 combinations from given list""" - pool = tuple(iterable) - n = len(pool) - if r > n: - return - indices = list(range(r)) - yield tuple(pool[i] for i in indices) - count = 0 - while count < 44: - count += 1 - for i in reversed(range(r)): - if indices[i] != i + n - r: - break - else: - return - indices[i] += 1 - for j in range(i + 1, r): - indices[j] = indices[j - 1] + 1 - yield tuple(pool[i] for i in indices) - - -def load_result(filename): - """ - Loads results from specified file - """ - inputs = open(filename, "r") - lines = inputs.readlines() - ls = [] - for line in lines: - ls.append(float(line.strip())) - return ls - - -def produce_mean(ls): - """ - Produces means from list of 8 results - """ - ls_space = [] - for i in range(int(len(ls) / 8)): - l = ls[i * 8 : (i + 1) * 8] - ls_space.append(l) - - return np.mean(ls_space, axis=0) - - -def run_rf_image_set( - model, - train_images, - train_labels, - test_images, - test_labels, - samples, - classes, -): - """ - Peforms multiclass predictions for a random forest classifier - with fixed total samples - """ - num_classes = len(classes) - partitions = np.array_split(np.array(range(samples)), num_classes) - - # Obtain only train images and labels for selected classes - image_ls = [] - label_ls = [] - i = 0 - for cls in classes: - class_idx = np.argwhere(train_labels == cls).flatten() - np.random.shuffle(class_idx) - class_img = train_images[class_idx[: len(partitions[i])]] - image_ls.append(class_img) - label_ls.append(np.repeat(cls, len(partitions[i]))) - i += 1 - - train_images = np.concatenate(image_ls) - train_labels = np.concatenate(label_ls) - - # Obtain only test images and labels for selected classes - image_ls = [] - label_ls = [] - for cls in classes: - image_ls.append(test_images[test_labels == cls]) - label_ls.append(np.repeat(cls, np.sum(test_labels == cls))) - - test_images = np.concatenate(image_ls) - test_labels = np.concatenate(label_ls) - - # Train the model - start_time = time.perf_counter() - model.fit(train_images, train_labels) - end_time = time.perf_counter() - train_time = end_time - start_time - - # Test the model - start_time = time.perf_counter() - test_preds = model.predict(test_images) - end_time = time.perf_counter() - test_time = end_time - start_time - - return accuracy_score(test_labels, test_preds), train_time, test_time - - -def run_dn_image_es( - model, - train_loader, - valid_loader, - test_loader, - epochs=30, - lr=0.001, - batch=64, -): - """ - Peforms multiclass predictions for a deep network classifier with set number - of samples and early stopping - """ - # define model - dev = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - model.to(dev) - # loss and optimizer - criterion = nn.CrossEntropyLoss() - optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9) - # early stopping setup - prev_loss = float("inf") - flag = 0 - - start_time = time.perf_counter() - for epoch in range(epochs): # loop over the dataset multiple times - - for i, data in enumerate(train_loader, 0): - # get the inputs - inputs, labels = data - inputs = inputs.clone().detach().to(dev) - labels = labels.clone().detach().to(dev) - # zero the parameter gradients - optimizer.zero_grad() - - # forward + backward + optimize - outputs = model(inputs) - loss = criterion(outputs, labels) - loss.backward() - optimizer.step() - - # test generalization error for early stopping - cur_loss = 0 - with torch.no_grad(): - for i, data in enumerate(valid_loader, 0): - # get the inputs - inputs, labels = data - inputs = inputs.clone().detach().to(dev) - labels = labels.clone().detach().to(dev) - - # forward - outputs = model(inputs) - loss = criterion(outputs, labels) - cur_loss += loss - # early stop if 3 epochs in a row no loss decrease - if cur_loss < prev_loss: - prev_loss = cur_loss - flag = 0 - else: - flag += 1 - if flag >= 3: - print("early stopped at epoch: ", epoch) - break - end_time = time.perf_counter() - train_time = end_time - start_time - - # test the model - start_time = time.perf_counter() - correct = torch.tensor(0).to(dev) - total = torch.tensor(0).to(dev) - with torch.no_grad(): - for i, data in enumerate(test_loader, 0): - inputs, labels = data - inputs = inputs.clone().detach().to(dev) - labels = labels.clone().detach().to(dev) - outputs = model(inputs) - _, predicted = torch.max(outputs.data, 1) - total += labels.size(0) - correct += (predicted == labels.view(-1)).sum().item() - accuracy = float(correct) / float(total) - end_time = time.perf_counter() - test_time = end_time - start_time - return accuracy, train_time, test_time +from toolbox import ( + SimpleCNN32Filter, + SimpleCNN32Filter2Layers, + SimpleCNN32Filter5Layers, + write_result, + combinations_45, + load_result, + produce_mean, + run_rf_image_set, + run_dn_image_es, +) def create_loaders_es( diff --git a/image/toolbox.py b/image/toolbox.py index bd7eea09..1938b493 100644 --- a/image/toolbox.py +++ b/image/toolbox.py @@ -1,10 +1,10 @@ """ -Coauthors: Yu-Chung Peng - Haoyin Xu +Coauthors: Haoyin Xu + Yu-Chung Peng """ import time import numpy as np -from sklearn.metrics import accuracy_score +from sklearn.metrics import cohen_kappa_score import torch import torch.nn as nn @@ -138,54 +138,6 @@ def produce_mean(ls): return np.mean(ls_space, axis=0) -def run_rf_image( - model, - train_images, - train_labels, - test_images, - test_labels, - fraction_of_train_samples, - classes, -): - """ - Peforms multiclass predictions for a random forest classifier - """ - num_train_samples = int(np.sum(train_labels == 0) * fraction_of_train_samples) - - # Obtain only train images and labels for selected classes - image_ls = [] - label_ls = [] - for cls in classes: - cls_images = train_images[train_labels == cls] - np.random.shuffle(cls_images) - image_ls.append(cls_images[:num_train_samples]) - label_ls.append(np.repeat(cls, num_train_samples)) - - train_images = np.concatenate(image_ls) - train_labels = np.concatenate(label_ls) - perm = np.random.permutation(train_images.shape[0]) - train_images = train_images[perm] - train_labels = train_labels[perm] - - # Obtain only test images and labels for selected classes - image_ls = [] - label_ls = [] - for cls in classes: - image_ls.append(test_images[test_labels == cls]) - label_ls.append(np.repeat(cls, np.sum(test_labels == cls))) - - test_images = np.concatenate(image_ls) - test_labels = np.concatenate(label_ls) - - # Train the model - model.fit(train_images, train_labels) - - # Test the model - test_preds = model.predict(test_images) - - return accuracy_score(test_labels, test_preds) - - def run_rf_image_set( model, train_images, @@ -239,57 +191,7 @@ def run_rf_image_set( end_time = time.perf_counter() test_time = end_time - start_time - return accuracy_score(test_labels, test_preds), train_time, test_time - - -def run_dn_image( - model, - train_loader, - test_loader, - epochs=1, - lr=0.001, - batch=64, -): - """ - Peforms multiclass predictions for a deep network classifier - """ - # define model - dev = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - model.to(dev) - # loss and optimizer - criterion = nn.CrossEntropyLoss() - optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9) - - for epoch in range(epochs): # loop over the dataset multiple times - - for i, data in enumerate(train_loader, 0): - # get the inputs - inputs, labels = data - inputs = inputs.clone().detach().to(dev) - labels = labels.clone().detach().to(dev) - # zero the parameter gradients - optimizer.zero_grad() - - # forward + backward + optimize - outputs = model(inputs) - loss = criterion(outputs, labels) - loss.backward() - optimizer.step() - - # test the model - correct = torch.tensor(0).to(dev) - total = torch.tensor(0).to(dev) - with torch.no_grad(): - for data in test_loader: - images, labels = data - labels = labels.clone().detach().to(dev) - images = images.clone().detach().to(dev) - outputs = model(images) - _, predicted = torch.max(outputs.data, 1) - total += labels.size(0) - correct += (predicted == labels.view(-1)).sum().item() - accuracy = float(correct) / float(total) - return accuracy + return cohen_kappa_score(test_labels, test_preds), train_time, test_time def run_dn_image_set( @@ -311,6 +213,7 @@ def run_dn_image_set( criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9) + model.train() start_time = time.perf_counter() while True: # loop over the dataset multiple times @@ -334,22 +237,24 @@ def run_dn_image_set( break # test the model + model.eval() start_time = time.perf_counter() - correct = torch.tensor(0).to(dev) - total = torch.tensor(0).to(dev) + test_preds = [] + test_labels = [] with torch.no_grad(): for data in test_loader: images, labels = data - labels = labels.clone().detach().to(dev) images = images.clone().detach().to(dev) + labels = labels.clone().detach().to(dev) + test_labels = np.concatenate((test_labels, labels.tolist())) + outputs = model(images) _, predicted = torch.max(outputs.data, 1) - total += labels.size(0) - correct += (predicted == labels.view(-1)).sum().item() - accuracy = float(correct) / float(total) + test_preds = np.concatenate((test_preds, predicted.tolist())) + end_time = time.perf_counter() test_time = end_time - start_time - return accuracy, train_time, test_time + return cohen_kappa_score(test_preds, test_labels), train_time, test_time def run_dn_image_es( @@ -377,7 +282,7 @@ def run_dn_image_es( start_time = time.perf_counter() for epoch in range(epochs): # loop over the dataset multiple times - + model.train() for i, data in enumerate(train_loader, 0): # get the inputs inputs, labels = data @@ -393,6 +298,7 @@ def run_dn_image_es( optimizer.step() # test generalization error for early stopping + model.eval() cur_loss = 0 with torch.no_grad(): for i, data in enumerate(valid_loader, 0): @@ -418,81 +324,24 @@ def run_dn_image_es( train_time = end_time - start_time # test the model + model.eval() start_time = time.perf_counter() - correct = torch.tensor(0).to(dev) - total = torch.tensor(0).to(dev) + test_preds = [] + test_labels = [] with torch.no_grad(): for data in test_loader: images, labels = data - labels = labels.clone().detach().to(dev) images = images.clone().detach().to(dev) + labels = labels.clone().detach().to(dev) + test_labels = np.concatenate((test_labels, labels.tolist())) + outputs = model(images) _, predicted = torch.max(outputs.data, 1) - total += labels.size(0) - correct += (predicted == labels.view(-1)).sum().item() - accuracy = float(correct) / float(total) + test_preds = np.concatenate((test_preds, predicted.tolist())) + end_time = time.perf_counter() test_time = end_time - start_time - return accuracy, train_time, test_time - - -def create_loaders( - train_labels, - test_labels, - classes, - fraction_of_train_samples, - trainset, - testset, - batch=64, -): - """ - Creates training and testing loaders with varied total samples - """ - classes = np.array(list(classes)) - - # get indicies of classes we want - class_idxs = [] - for cls in classes: - class_idx = np.argwhere(train_labels == cls).flatten() - np.random.shuffle(class_idx) - class_idx = class_idx[: int(len(class_idx) * fraction_of_train_samples)] - class_idxs.append(class_idx) - - np.random.shuffle(class_idxs) - - train_idxs = np.concatenate(class_idxs) - - # change the labels to be from 0-len(classes) - for i in train_idxs: - trainset.targets[i] = np.where(classes == trainset.targets[i])[0][0] - - train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_idxs) - train_loader = torch.utils.data.DataLoader( - trainset, batch_size=batch, num_workers=4, sampler=train_sampler, drop_last=True - ) - - # get indicies of classes we want - test_idxs = [] - for cls in classes: - test_idx = np.argwhere(test_labels == cls).flatten() - test_idxs.append(test_idx) - - test_idxs = np.concatenate(test_idxs) - - # change the labels to be from 0-len(classes) - for i in test_idxs: - testset.targets[i] = np.where(classes == testset.targets[i])[0][0] - - test_sampler = torch.utils.data.sampler.SubsetRandomSampler(test_idxs) - test_loader = torch.utils.data.DataLoader( - testset, - batch_size=batch, - shuffle=False, - num_workers=4, - sampler=test_sampler, - drop_last=True, - ) - return train_loader, test_loader + return cohen_kappa_score(test_preds, test_labels), train_time, test_time def create_loaders_set( From 0f9c496c7b347ec0bb86c65fba47e6bc46746777 Mon Sep 17 00:00:00 2001 From: Haoyin Xu Date: Thu, 23 Sep 2021 11:41:57 -0400 Subject: [PATCH 2/5] MAINT rename data domains --- {audio => audition}/3_class/cnn32.txt | 0 {audio => audition}/3_class/cnn32_2l.txt | 0 {audio => audition}/3_class/cnn32_2l_test_time.txt | 0 {audio => audition}/3_class/cnn32_2l_train_time.txt | 0 {audio => audition}/3_class/cnn32_5l.txt | 0 {audio => audition}/3_class/cnn32_5l_test_time.txt | 0 {audio => audition}/3_class/cnn32_5l_train_time.txt | 0 {audio => audition}/3_class/cnn32_test_time.txt | 0 {audio => audition}/3_class/cnn32_train_time.txt | 0 {audio => audition}/3_class/naive_rf.txt | 0 {audio => audition}/3_class/naive_rf_test_time.txt | 0 {audio => audition}/3_class/naive_rf_train_time.txt | 0 {audio => audition}/3_class/resnet18.txt | 0 {audio => audition}/3_class/resnet18_test_time.txt | 0 {audio => audition}/3_class/resnet18_train_time.txt | 0 {audio => audition}/3_class/svm.txt | 0 {audio => audition}/3_class/svm_test_time.txt | 0 {audio => audition}/3_class/svm_train_time.txt | 0 {audio => audition}/3_class_mel/cnn32.txt | 0 {audio => audition}/3_class_mel/cnn32_2l.txt | 0 {audio => audition}/3_class_mel/cnn32_2l_test_time.txt | 0 {audio => audition}/3_class_mel/cnn32_2l_train_time.txt | 0 {audio => audition}/3_class_mel/cnn32_5l.txt | 0 {audio => audition}/3_class_mel/cnn32_5l_test_time.txt | 0 {audio => audition}/3_class_mel/cnn32_5l_train_time.txt | 0 {audio => audition}/3_class_mel/cnn32_test_time.txt | 0 {audio => audition}/3_class_mel/cnn32_train_time.txt | 0 {audio => audition}/3_class_mel/naive_rf.txt | 0 {audio => audition}/3_class_mel/naive_rf_test_time.txt | 0 {audio => audition}/3_class_mel/naive_rf_train_time.txt | 0 {audio => audition}/3_class_mel/resnet18.txt | 0 {audio => audition}/3_class_mel/resnet18_test_time.txt | 0 {audio => audition}/3_class_mel/resnet18_train_time.txt | 0 {audio => audition}/3_class_mel/svm.txt | 0 {audio => audition}/3_class_mel/svm_test_time.txt | 0 {audio => audition}/3_class_mel/svm_train_time.txt | 0 {audio => audition}/3_class_mfcc/cnn32.txt | 0 {audio => audition}/3_class_mfcc/cnn32_2l.txt | 0 {audio => audition}/3_class_mfcc/cnn32_2l_test_time.txt | 0 {audio => audition}/3_class_mfcc/cnn32_2l_train_time.txt | 0 {audio => audition}/3_class_mfcc/cnn32_5l.txt | 0 {audio => audition}/3_class_mfcc/cnn32_5l_test_time.txt | 0 {audio => audition}/3_class_mfcc/cnn32_5l_train_time.txt | 0 {audio => audition}/3_class_mfcc/cnn32_test_time.txt | 0 {audio => audition}/3_class_mfcc/cnn32_train_time.txt | 0 {audio => audition}/3_class_mfcc/naive_rf.txt | 0 {audio => audition}/3_class_mfcc/naive_rf_test_time.txt | 0 {audio => audition}/3_class_mfcc/naive_rf_train_time.txt | 0 {audio => audition}/3_class_mfcc/resnet18.txt | 0 {audio => audition}/3_class_mfcc/resnet18_test_time.txt | 0 {audio => audition}/3_class_mfcc/resnet18_train_time.txt | 0 {audio => audition}/3_class_mfcc/svm.txt | 0 {audio => audition}/3_class_mfcc/svm_test_time.txt | 0 {audio => audition}/3_class_mfcc/svm_train_time.txt | 0 {audio => audition}/8_class/cnn32.txt | 0 {audio => audition}/8_class/cnn32_2l.txt | 0 {audio => audition}/8_class/cnn32_2l_test_time.txt | 0 {audio => audition}/8_class/cnn32_2l_train_time.txt | 0 {audio => audition}/8_class/cnn32_5l.txt | 0 {audio => audition}/8_class/cnn32_5l_test_time.txt | 0 {audio => audition}/8_class/cnn32_5l_train_time.txt | 0 {audio => audition}/8_class/cnn32_test_time.txt | 0 {audio => audition}/8_class/cnn32_train_time.txt | 0 {audio => audition}/8_class/naive_rf.txt | 0 {audio => audition}/8_class/naive_rf_test_time.txt | 0 {audio => audition}/8_class/naive_rf_train_time.txt | 0 {audio => audition}/8_class/resnet18.txt | 0 {audio => audition}/8_class/resnet18_test_time.txt | 0 {audio => audition}/8_class/resnet18_train_time.txt | 0 {audio => audition}/8_class/svm.txt | 0 {audio => audition}/8_class/svm_test_time.txt | 0 {audio => audition}/8_class/svm_train_time.txt | 0 {audio => audition}/8_class_mel/cnn32.txt | 0 {audio => audition}/8_class_mel/cnn32_2l.txt | 0 {audio => audition}/8_class_mel/cnn32_2l_test_time.txt | 0 {audio => audition}/8_class_mel/cnn32_2l_train_time.txt | 0 {audio => audition}/8_class_mel/cnn32_5l.txt | 0 {audio => audition}/8_class_mel/cnn32_5l_test_time.txt | 0 {audio => audition}/8_class_mel/cnn32_5l_train_time.txt | 0 {audio => audition}/8_class_mel/cnn32_test_time.txt | 0 {audio => audition}/8_class_mel/cnn32_train_time.txt | 0 {audio => audition}/8_class_mel/naive_rf.txt | 0 {audio => audition}/8_class_mel/naive_rf_test_time.txt | 0 {audio => audition}/8_class_mel/naive_rf_train_time.txt | 0 {audio => audition}/8_class_mel/resnet18.txt | 0 {audio => audition}/8_class_mel/resnet18_test_time.txt | 0 {audio => audition}/8_class_mel/resnet18_train_time.txt | 0 {audio => audition}/8_class_mel/svm.txt | 0 {audio => audition}/8_class_mel/svm_test_time.txt | 0 {audio => audition}/8_class_mel/svm_train_time.txt | 0 {audio => audition}/8_class_mfcc/cnn32.txt | 0 {audio => audition}/8_class_mfcc/cnn32_2l.txt | 0 {audio => audition}/8_class_mfcc/cnn32_2l_test_time.txt | 0 {audio => audition}/8_class_mfcc/cnn32_2l_train_time.txt | 0 {audio => audition}/8_class_mfcc/cnn32_5l.txt | 0 {audio => audition}/8_class_mfcc/cnn32_5l_test_time.txt | 0 {audio => audition}/8_class_mfcc/cnn32_5l_train_time.txt | 0 {audio => audition}/8_class_mfcc/cnn32_test_time.txt | 0 {audio => audition}/8_class_mfcc/cnn32_train_time.txt | 0 {audio => audition}/8_class_mfcc/naive_rf.txt | 0 {audio => audition}/8_class_mfcc/naive_rf_test_time.txt | 0 {audio => audition}/8_class_mfcc/naive_rf_train_time.txt | 0 {audio => audition}/8_class_mfcc/resnet18.txt | 0 {audio => audition}/8_class_mfcc/resnet18_test_time.txt | 0 {audio => audition}/8_class_mfcc/resnet18_train_time.txt | 0 {audio => audition}/8_class_mfcc/svm.txt | 0 {audio => audition}/8_class_mfcc/svm_test_time.txt | 0 {audio => audition}/8_class_mfcc/svm_train_time.txt | 0 {audio => audition}/audio_toolbox.py | 0 {audio => audition}/fsdd.py | 0 {audio => audition}/fsdd_figures.ipynb | 0 {image => vision}/3_class/cnn32.txt | 0 {image => vision}/3_class/cnn32_2l.txt | 0 {image => vision}/3_class/cnn32_2l_sc.txt | 0 {image => vision}/3_class/cnn32_2l_st.txt | 0 {image => vision}/3_class/cnn32_2l_test_time.txt | 0 {image => vision}/3_class/cnn32_2l_test_time_sc.txt | 0 {image => vision}/3_class/cnn32_2l_test_time_st.txt | 0 {image => vision}/3_class/cnn32_2l_train_time.txt | 0 {image => vision}/3_class/cnn32_2l_train_time_sc.txt | 0 {image => vision}/3_class/cnn32_2l_train_time_st.txt | 0 {image => vision}/3_class/cnn32_5l.txt | 0 {image => vision}/3_class/cnn32_5l_sc.txt | 0 {image => vision}/3_class/cnn32_5l_st.txt | 0 {image => vision}/3_class/cnn32_5l_test_time.txt | 0 {image => vision}/3_class/cnn32_5l_test_time_sc.txt | 0 {image => vision}/3_class/cnn32_5l_test_time_st.txt | 0 {image => vision}/3_class/cnn32_5l_train_time.txt | 0 {image => vision}/3_class/cnn32_5l_train_time_sc.txt | 0 {image => vision}/3_class/cnn32_5l_train_time_st.txt | 0 {image => vision}/3_class/cnn32_sc.txt | 0 {image => vision}/3_class/cnn32_st.txt | 0 {image => vision}/3_class/cnn32_test_time.txt | 0 {image => vision}/3_class/cnn32_test_time_sc.txt | 0 {image => vision}/3_class/cnn32_test_time_st.txt | 0 {image => vision}/3_class/cnn32_train_time.txt | 0 {image => vision}/3_class/cnn32_train_time_sc.txt | 0 {image => vision}/3_class/cnn32_train_time_st.txt | 0 {image => vision}/3_class/naive_rf.txt | 0 {image => vision}/3_class/naive_rf_test_time.txt | 0 {image => vision}/3_class/naive_rf_test_time_lc.txt | 0 {image => vision}/3_class/naive_rf_train_time.txt | 0 {image => vision}/3_class/naive_rf_train_time_lc.txt | 0 {image => vision}/3_class/resnet18.txt | 0 {image => vision}/3_class/resnet18_sc.txt | 0 {image => vision}/3_class/resnet18_st.txt | 0 {image => vision}/3_class/resnet18_test_time.txt | 0 {image => vision}/3_class/resnet18_test_time_sc.txt | 0 {image => vision}/3_class/resnet18_test_time_st.txt | 0 {image => vision}/3_class/resnet18_train_time.txt | 0 {image => vision}/3_class/resnet18_train_time_sc.txt | 0 {image => vision}/3_class/resnet18_train_time_st.txt | 0 {image => vision}/3_class/svhn_cnn32.txt | 0 {image => vision}/3_class/svhn_cnn32_2l.txt | 0 {image => vision}/3_class/svhn_cnn32_2l_test_time.txt | 0 {image => vision}/3_class/svhn_cnn32_2l_train_time.txt | 0 {image => vision}/3_class/svhn_cnn32_5l.txt | 0 {image => vision}/3_class/svhn_cnn32_5l_test_time.txt | 0 {image => vision}/3_class/svhn_cnn32_5l_train_time.txt | 0 {image => vision}/3_class/svhn_cnn32_test_time.txt | 0 {image => vision}/3_class/svhn_cnn32_train_time.txt | 0 {image => vision}/3_class/svhn_naive_rf.txt | 0 {image => vision}/3_class/svhn_naive_rf_test_time.txt | 0 {image => vision}/3_class/svhn_naive_rf_train_time.txt | 0 {image => vision}/3_class/svhn_resnet18.txt | 0 {image => vision}/3_class/svhn_resnet18_test_time.txt | 0 {image => vision}/3_class/svhn_resnet18_train_time.txt | 0 {image => vision}/3_class/svhn_svm.txt | 0 {image => vision}/3_class/svhn_svm_test_time.txt | 0 {image => vision}/3_class/svhn_svm_train_time.txt | 0 {image => vision}/3_class/svm.txt | 0 {image => vision}/3_class/svm_test_time.txt | 0 {image => vision}/3_class/svm_test_time_lc.txt | 0 {image => vision}/3_class/svm_train_time.txt | 0 {image => vision}/3_class/svm_train_time_lc.txt | 0 {image => vision}/8_class/cnn32.txt | 0 {image => vision}/8_class/cnn32_2l.txt | 0 {image => vision}/8_class/cnn32_2l_sc.txt | 0 {image => vision}/8_class/cnn32_2l_st.txt | 0 {image => vision}/8_class/cnn32_2l_test_time.txt | 0 {image => vision}/8_class/cnn32_2l_test_time_sc.txt | 0 {image => vision}/8_class/cnn32_2l_test_time_st.txt | 0 {image => vision}/8_class/cnn32_2l_train_time.txt | 0 {image => vision}/8_class/cnn32_2l_train_time_sc.txt | 0 {image => vision}/8_class/cnn32_2l_train_time_st.txt | 0 {image => vision}/8_class/cnn32_5l.txt | 0 {image => vision}/8_class/cnn32_5l_sc.txt | 0 {image => vision}/8_class/cnn32_5l_st.txt | 0 {image => vision}/8_class/cnn32_5l_test_time.txt | 0 {image => vision}/8_class/cnn32_5l_test_time_sc.txt | 0 {image => vision}/8_class/cnn32_5l_test_time_st.txt | 0 {image => vision}/8_class/cnn32_5l_train_time.txt | 0 {image => vision}/8_class/cnn32_5l_train_time_sc.txt | 0 {image => vision}/8_class/cnn32_5l_train_time_st.txt | 0 {image => vision}/8_class/cnn32_sc.txt | 0 {image => vision}/8_class/cnn32_st.txt | 0 {image => vision}/8_class/cnn32_test_time.txt | 0 {image => vision}/8_class/cnn32_test_time_sc.txt | 0 {image => vision}/8_class/cnn32_test_time_st.txt | 0 {image => vision}/8_class/cnn32_train_time.txt | 0 {image => vision}/8_class/cnn32_train_time_sc.txt | 0 {image => vision}/8_class/cnn32_train_time_st.txt | 0 {image => vision}/8_class/naive_rf.txt | 0 {image => vision}/8_class/naive_rf_test_time.txt | 0 {image => vision}/8_class/naive_rf_test_time_lc.txt | 0 {image => vision}/8_class/naive_rf_train_time.txt | 0 {image => vision}/8_class/naive_rf_train_time_lc.txt | 0 {image => vision}/8_class/resnet18.txt | 0 {image => vision}/8_class/resnet18_sc.txt | 0 {image => vision}/8_class/resnet18_st.txt | 0 {image => vision}/8_class/resnet18_test_time.txt | 0 {image => vision}/8_class/resnet18_test_time_sc.txt | 0 {image => vision}/8_class/resnet18_test_time_st.txt | 0 {image => vision}/8_class/resnet18_train_time.txt | 0 {image => vision}/8_class/resnet18_train_time_sc.txt | 0 {image => vision}/8_class/resnet18_train_time_st.txt | 0 {image => vision}/8_class/svhn_cnn32.txt | 0 {image => vision}/8_class/svhn_cnn32_2l.txt | 0 {image => vision}/8_class/svhn_cnn32_2l_test_time.txt | 0 {image => vision}/8_class/svhn_cnn32_2l_train_time.txt | 0 {image => vision}/8_class/svhn_cnn32_5l.txt | 0 {image => vision}/8_class/svhn_cnn32_5l_test_time.txt | 0 {image => vision}/8_class/svhn_cnn32_5l_train_time.txt | 0 {image => vision}/8_class/svhn_cnn32_test_time.txt | 0 {image => vision}/8_class/svhn_cnn32_train_time.txt | 0 {image => vision}/8_class/svhn_naive_rf.txt | 0 {image => vision}/8_class/svhn_naive_rf_test_time.txt | 0 {image => vision}/8_class/svhn_naive_rf_train_time.txt | 0 {image => vision}/8_class/svhn_resnet18.txt | 0 {image => vision}/8_class/svhn_resnet18_test_time.txt | 0 {image => vision}/8_class/svhn_resnet18_train_time.txt | 0 {image => vision}/8_class/svhn_svm.txt | 0 {image => vision}/8_class/svhn_svm_test_time.txt | 0 {image => vision}/8_class/svhn_svm_train_time.txt | 0 {image => vision}/8_class/svm.txt | 0 {image => vision}/8_class/svm_test_time.txt | 0 {image => vision}/8_class/svm_test_time_lc.txt | 0 {image => vision}/8_class/svm_train_time.txt | 0 {image => vision}/8_class/svm_train_time_lc.txt | 0 {image => vision}/90_class/cnn32.txt | 0 {image => vision}/90_class/cnn32_2l.txt | 0 {image => vision}/90_class/cnn32_2l_sc.txt | 0 {image => vision}/90_class/cnn32_2l_st.txt | 0 {image => vision}/90_class/cnn32_2l_test_time.txt | 0 {image => vision}/90_class/cnn32_2l_test_time_sc.txt | 0 {image => vision}/90_class/cnn32_2l_test_time_st.txt | 0 {image => vision}/90_class/cnn32_2l_train_time.txt | 0 {image => vision}/90_class/cnn32_2l_train_time_sc.txt | 0 {image => vision}/90_class/cnn32_2l_train_time_st.txt | 0 {image => vision}/90_class/cnn32_5l.txt | 0 {image => vision}/90_class/cnn32_5l_sc.txt | 0 {image => vision}/90_class/cnn32_5l_st.txt | 0 {image => vision}/90_class/cnn32_5l_test_time.txt | 0 {image => vision}/90_class/cnn32_5l_test_time_sc.txt | 0 {image => vision}/90_class/cnn32_5l_test_time_st.txt | 0 {image => vision}/90_class/cnn32_5l_train_time.txt | 0 {image => vision}/90_class/cnn32_5l_train_time_sc.txt | 0 {image => vision}/90_class/cnn32_5l_train_time_st.txt | 0 {image => vision}/90_class/cnn32_sc.txt | 0 {image => vision}/90_class/cnn32_st.txt | 0 {image => vision}/90_class/cnn32_test_time.txt | 0 {image => vision}/90_class/cnn32_test_time_sc.txt | 0 {image => vision}/90_class/cnn32_test_time_st.txt | 0 {image => vision}/90_class/cnn32_train_time.txt | 0 {image => vision}/90_class/cnn32_train_time_sc.txt | 0 {image => vision}/90_class/cnn32_train_time_st.txt | 0 {image => vision}/90_class/naive_rf.txt | 0 {image => vision}/90_class/naive_rf_test_time.txt | 0 {image => vision}/90_class/naive_rf_test_time_lc.txt | 0 {image => vision}/90_class/naive_rf_train_time.txt | 0 {image => vision}/90_class/naive_rf_train_time_lc.txt | 0 {image => vision}/90_class/resnet18.txt | 0 {image => vision}/90_class/resnet18_sc.txt | 0 {image => vision}/90_class/resnet18_st.txt | 0 {image => vision}/90_class/resnet18_test_time.txt | 0 {image => vision}/90_class/resnet18_test_time_sc.txt | 0 {image => vision}/90_class/resnet18_test_time_st.txt | 0 {image => vision}/90_class/resnet18_train_time.txt | 0 {image => vision}/90_class/resnet18_train_time_sc.txt | 0 {image => vision}/90_class/resnet18_train_time_st.txt | 0 {image => vision}/90_class/svm.txt | 0 {image => vision}/90_class/svm_test_time.txt | 0 {image => vision}/90_class/svm_test_time_lc.txt | 0 {image => vision}/90_class/svm_train_time.txt | 0 {image => vision}/90_class/svm_train_time_lc.txt | 0 {image => vision}/cifar_10.py | 0 {image => vision}/cifar_100.py | 0 {image => vision}/cifar_100_stc.py | 0 {image => vision}/cifar_10_stc.py | 0 {image => vision}/cifar_figures.ipynb | 0 {image => vision}/svhn.py | 0 {image => vision}/svhn_figure.ipynb | 0 {image => vision}/svhn_toolbox.py | 0 {image => vision}/toolbox.py | 0 294 files changed, 0 insertions(+), 0 deletions(-) rename {audio => audition}/3_class/cnn32.txt (100%) rename {audio => audition}/3_class/cnn32_2l.txt (100%) rename {audio => audition}/3_class/cnn32_2l_test_time.txt (100%) rename {audio => audition}/3_class/cnn32_2l_train_time.txt (100%) rename {audio => audition}/3_class/cnn32_5l.txt (100%) rename {audio => audition}/3_class/cnn32_5l_test_time.txt (100%) rename {audio => audition}/3_class/cnn32_5l_train_time.txt (100%) rename {audio => audition}/3_class/cnn32_test_time.txt (100%) rename {audio => audition}/3_class/cnn32_train_time.txt (100%) rename {audio => audition}/3_class/naive_rf.txt (100%) rename {audio => audition}/3_class/naive_rf_test_time.txt (100%) rename {audio => audition}/3_class/naive_rf_train_time.txt (100%) rename {audio => audition}/3_class/resnet18.txt (100%) rename {audio => audition}/3_class/resnet18_test_time.txt (100%) rename {audio => audition}/3_class/resnet18_train_time.txt (100%) rename {audio => audition}/3_class/svm.txt (100%) rename {audio => audition}/3_class/svm_test_time.txt (100%) rename {audio => audition}/3_class/svm_train_time.txt (100%) rename {audio => audition}/3_class_mel/cnn32.txt (100%) rename {audio => audition}/3_class_mel/cnn32_2l.txt (100%) rename {audio => audition}/3_class_mel/cnn32_2l_test_time.txt (100%) rename {audio => audition}/3_class_mel/cnn32_2l_train_time.txt (100%) rename {audio => audition}/3_class_mel/cnn32_5l.txt (100%) rename {audio => audition}/3_class_mel/cnn32_5l_test_time.txt (100%) rename {audio => audition}/3_class_mel/cnn32_5l_train_time.txt (100%) rename {audio => audition}/3_class_mel/cnn32_test_time.txt (100%) rename {audio => audition}/3_class_mel/cnn32_train_time.txt (100%) rename {audio => audition}/3_class_mel/naive_rf.txt (100%) rename {audio => audition}/3_class_mel/naive_rf_test_time.txt (100%) rename {audio => audition}/3_class_mel/naive_rf_train_time.txt (100%) rename {audio => audition}/3_class_mel/resnet18.txt (100%) rename {audio => audition}/3_class_mel/resnet18_test_time.txt (100%) rename {audio => audition}/3_class_mel/resnet18_train_time.txt (100%) rename {audio => audition}/3_class_mel/svm.txt (100%) rename {audio => audition}/3_class_mel/svm_test_time.txt (100%) rename {audio => audition}/3_class_mel/svm_train_time.txt (100%) rename {audio => audition}/3_class_mfcc/cnn32.txt (100%) rename {audio => audition}/3_class_mfcc/cnn32_2l.txt (100%) rename {audio => audition}/3_class_mfcc/cnn32_2l_test_time.txt (100%) rename {audio => audition}/3_class_mfcc/cnn32_2l_train_time.txt (100%) rename {audio => audition}/3_class_mfcc/cnn32_5l.txt (100%) rename {audio => audition}/3_class_mfcc/cnn32_5l_test_time.txt (100%) rename {audio => audition}/3_class_mfcc/cnn32_5l_train_time.txt (100%) rename {audio => audition}/3_class_mfcc/cnn32_test_time.txt (100%) rename {audio => audition}/3_class_mfcc/cnn32_train_time.txt (100%) rename {audio => audition}/3_class_mfcc/naive_rf.txt (100%) rename {audio => audition}/3_class_mfcc/naive_rf_test_time.txt (100%) rename {audio => audition}/3_class_mfcc/naive_rf_train_time.txt (100%) rename {audio => audition}/3_class_mfcc/resnet18.txt (100%) rename {audio => audition}/3_class_mfcc/resnet18_test_time.txt (100%) rename {audio => audition}/3_class_mfcc/resnet18_train_time.txt (100%) rename {audio => audition}/3_class_mfcc/svm.txt (100%) rename {audio => audition}/3_class_mfcc/svm_test_time.txt (100%) rename {audio => audition}/3_class_mfcc/svm_train_time.txt (100%) rename {audio => audition}/8_class/cnn32.txt (100%) rename {audio => audition}/8_class/cnn32_2l.txt (100%) rename {audio => audition}/8_class/cnn32_2l_test_time.txt (100%) rename {audio => audition}/8_class/cnn32_2l_train_time.txt (100%) rename {audio => audition}/8_class/cnn32_5l.txt (100%) rename {audio => audition}/8_class/cnn32_5l_test_time.txt (100%) rename {audio => audition}/8_class/cnn32_5l_train_time.txt (100%) rename {audio => audition}/8_class/cnn32_test_time.txt (100%) rename {audio => audition}/8_class/cnn32_train_time.txt (100%) rename {audio => audition}/8_class/naive_rf.txt (100%) rename {audio => audition}/8_class/naive_rf_test_time.txt (100%) rename {audio => audition}/8_class/naive_rf_train_time.txt (100%) rename {audio => audition}/8_class/resnet18.txt (100%) rename {audio => audition}/8_class/resnet18_test_time.txt (100%) rename {audio => audition}/8_class/resnet18_train_time.txt (100%) rename {audio => audition}/8_class/svm.txt (100%) rename {audio => audition}/8_class/svm_test_time.txt (100%) rename {audio => audition}/8_class/svm_train_time.txt (100%) rename {audio => audition}/8_class_mel/cnn32.txt (100%) rename {audio => audition}/8_class_mel/cnn32_2l.txt (100%) rename {audio => audition}/8_class_mel/cnn32_2l_test_time.txt (100%) rename {audio => audition}/8_class_mel/cnn32_2l_train_time.txt (100%) rename {audio => audition}/8_class_mel/cnn32_5l.txt (100%) rename {audio => audition}/8_class_mel/cnn32_5l_test_time.txt (100%) rename {audio => audition}/8_class_mel/cnn32_5l_train_time.txt (100%) rename {audio => audition}/8_class_mel/cnn32_test_time.txt (100%) rename {audio => audition}/8_class_mel/cnn32_train_time.txt (100%) rename {audio => audition}/8_class_mel/naive_rf.txt (100%) rename {audio => audition}/8_class_mel/naive_rf_test_time.txt (100%) rename {audio => audition}/8_class_mel/naive_rf_train_time.txt (100%) rename {audio => audition}/8_class_mel/resnet18.txt (100%) rename {audio => audition}/8_class_mel/resnet18_test_time.txt (100%) rename {audio => audition}/8_class_mel/resnet18_train_time.txt (100%) rename {audio => audition}/8_class_mel/svm.txt (100%) rename {audio => audition}/8_class_mel/svm_test_time.txt (100%) rename {audio => audition}/8_class_mel/svm_train_time.txt (100%) rename {audio => audition}/8_class_mfcc/cnn32.txt (100%) rename {audio => audition}/8_class_mfcc/cnn32_2l.txt (100%) rename {audio => audition}/8_class_mfcc/cnn32_2l_test_time.txt (100%) rename {audio => audition}/8_class_mfcc/cnn32_2l_train_time.txt (100%) rename {audio => audition}/8_class_mfcc/cnn32_5l.txt (100%) rename {audio => audition}/8_class_mfcc/cnn32_5l_test_time.txt (100%) rename {audio => audition}/8_class_mfcc/cnn32_5l_train_time.txt (100%) rename {audio => audition}/8_class_mfcc/cnn32_test_time.txt (100%) rename {audio => audition}/8_class_mfcc/cnn32_train_time.txt (100%) rename {audio => audition}/8_class_mfcc/naive_rf.txt (100%) rename {audio => audition}/8_class_mfcc/naive_rf_test_time.txt (100%) rename {audio => audition}/8_class_mfcc/naive_rf_train_time.txt (100%) rename {audio => audition}/8_class_mfcc/resnet18.txt (100%) rename {audio => audition}/8_class_mfcc/resnet18_test_time.txt (100%) rename {audio => audition}/8_class_mfcc/resnet18_train_time.txt (100%) rename {audio => audition}/8_class_mfcc/svm.txt (100%) rename {audio => audition}/8_class_mfcc/svm_test_time.txt (100%) rename {audio => audition}/8_class_mfcc/svm_train_time.txt (100%) rename {audio => audition}/audio_toolbox.py (100%) rename {audio => audition}/fsdd.py (100%) rename {audio => audition}/fsdd_figures.ipynb (100%) rename {image => vision}/3_class/cnn32.txt (100%) rename {image => vision}/3_class/cnn32_2l.txt (100%) rename {image => vision}/3_class/cnn32_2l_sc.txt (100%) rename {image => vision}/3_class/cnn32_2l_st.txt (100%) rename {image => vision}/3_class/cnn32_2l_test_time.txt (100%) rename {image => vision}/3_class/cnn32_2l_test_time_sc.txt (100%) rename {image => vision}/3_class/cnn32_2l_test_time_st.txt (100%) rename {image => vision}/3_class/cnn32_2l_train_time.txt (100%) rename {image => vision}/3_class/cnn32_2l_train_time_sc.txt (100%) rename {image => vision}/3_class/cnn32_2l_train_time_st.txt (100%) rename {image => vision}/3_class/cnn32_5l.txt (100%) rename {image => vision}/3_class/cnn32_5l_sc.txt (100%) rename {image => vision}/3_class/cnn32_5l_st.txt (100%) rename {image => vision}/3_class/cnn32_5l_test_time.txt (100%) rename {image => vision}/3_class/cnn32_5l_test_time_sc.txt (100%) rename {image => vision}/3_class/cnn32_5l_test_time_st.txt (100%) rename {image => vision}/3_class/cnn32_5l_train_time.txt (100%) rename {image => vision}/3_class/cnn32_5l_train_time_sc.txt (100%) rename {image => vision}/3_class/cnn32_5l_train_time_st.txt (100%) rename {image => vision}/3_class/cnn32_sc.txt (100%) rename {image => vision}/3_class/cnn32_st.txt (100%) rename {image => vision}/3_class/cnn32_test_time.txt (100%) rename {image => vision}/3_class/cnn32_test_time_sc.txt (100%) rename {image => vision}/3_class/cnn32_test_time_st.txt (100%) rename {image => vision}/3_class/cnn32_train_time.txt (100%) rename {image => vision}/3_class/cnn32_train_time_sc.txt (100%) rename {image => vision}/3_class/cnn32_train_time_st.txt (100%) rename {image => vision}/3_class/naive_rf.txt (100%) rename {image => vision}/3_class/naive_rf_test_time.txt (100%) rename {image => vision}/3_class/naive_rf_test_time_lc.txt (100%) rename {image => vision}/3_class/naive_rf_train_time.txt (100%) rename {image => vision}/3_class/naive_rf_train_time_lc.txt (100%) rename {image => vision}/3_class/resnet18.txt (100%) rename {image => vision}/3_class/resnet18_sc.txt (100%) rename {image => vision}/3_class/resnet18_st.txt (100%) rename {image => vision}/3_class/resnet18_test_time.txt (100%) rename {image => vision}/3_class/resnet18_test_time_sc.txt (100%) rename {image => vision}/3_class/resnet18_test_time_st.txt (100%) rename {image => vision}/3_class/resnet18_train_time.txt (100%) rename {image => vision}/3_class/resnet18_train_time_sc.txt (100%) rename {image => vision}/3_class/resnet18_train_time_st.txt (100%) rename {image => vision}/3_class/svhn_cnn32.txt (100%) rename {image => vision}/3_class/svhn_cnn32_2l.txt (100%) rename {image => vision}/3_class/svhn_cnn32_2l_test_time.txt (100%) rename {image => vision}/3_class/svhn_cnn32_2l_train_time.txt (100%) rename {image => vision}/3_class/svhn_cnn32_5l.txt (100%) rename {image => vision}/3_class/svhn_cnn32_5l_test_time.txt (100%) rename {image => vision}/3_class/svhn_cnn32_5l_train_time.txt (100%) rename {image => vision}/3_class/svhn_cnn32_test_time.txt (100%) rename {image => vision}/3_class/svhn_cnn32_train_time.txt (100%) rename {image => vision}/3_class/svhn_naive_rf.txt (100%) rename {image => vision}/3_class/svhn_naive_rf_test_time.txt (100%) rename {image => vision}/3_class/svhn_naive_rf_train_time.txt (100%) rename {image => vision}/3_class/svhn_resnet18.txt (100%) rename {image => vision}/3_class/svhn_resnet18_test_time.txt (100%) rename {image => vision}/3_class/svhn_resnet18_train_time.txt (100%) rename {image => vision}/3_class/svhn_svm.txt (100%) rename {image => vision}/3_class/svhn_svm_test_time.txt (100%) rename {image => vision}/3_class/svhn_svm_train_time.txt (100%) rename {image => vision}/3_class/svm.txt (100%) rename {image => vision}/3_class/svm_test_time.txt (100%) rename {image => vision}/3_class/svm_test_time_lc.txt (100%) rename {image => vision}/3_class/svm_train_time.txt (100%) rename {image => vision}/3_class/svm_train_time_lc.txt (100%) rename {image => vision}/8_class/cnn32.txt (100%) rename {image => vision}/8_class/cnn32_2l.txt (100%) rename {image => vision}/8_class/cnn32_2l_sc.txt (100%) rename {image => vision}/8_class/cnn32_2l_st.txt (100%) rename {image => vision}/8_class/cnn32_2l_test_time.txt (100%) rename {image => vision}/8_class/cnn32_2l_test_time_sc.txt (100%) rename {image => vision}/8_class/cnn32_2l_test_time_st.txt (100%) rename {image => vision}/8_class/cnn32_2l_train_time.txt (100%) rename {image => vision}/8_class/cnn32_2l_train_time_sc.txt (100%) rename {image => vision}/8_class/cnn32_2l_train_time_st.txt (100%) rename {image => vision}/8_class/cnn32_5l.txt (100%) rename {image => vision}/8_class/cnn32_5l_sc.txt (100%) rename {image => vision}/8_class/cnn32_5l_st.txt (100%) rename {image => vision}/8_class/cnn32_5l_test_time.txt (100%) rename {image => vision}/8_class/cnn32_5l_test_time_sc.txt (100%) rename {image => vision}/8_class/cnn32_5l_test_time_st.txt (100%) rename {image => vision}/8_class/cnn32_5l_train_time.txt (100%) rename {image => vision}/8_class/cnn32_5l_train_time_sc.txt (100%) rename {image => vision}/8_class/cnn32_5l_train_time_st.txt (100%) rename {image => vision}/8_class/cnn32_sc.txt (100%) rename {image => vision}/8_class/cnn32_st.txt (100%) rename {image => vision}/8_class/cnn32_test_time.txt (100%) rename {image => vision}/8_class/cnn32_test_time_sc.txt (100%) rename {image => vision}/8_class/cnn32_test_time_st.txt (100%) rename {image => vision}/8_class/cnn32_train_time.txt (100%) rename {image => vision}/8_class/cnn32_train_time_sc.txt (100%) rename {image => vision}/8_class/cnn32_train_time_st.txt (100%) rename {image => vision}/8_class/naive_rf.txt (100%) rename {image => vision}/8_class/naive_rf_test_time.txt (100%) rename {image => vision}/8_class/naive_rf_test_time_lc.txt (100%) rename {image => vision}/8_class/naive_rf_train_time.txt (100%) rename {image => vision}/8_class/naive_rf_train_time_lc.txt (100%) rename {image => vision}/8_class/resnet18.txt (100%) rename {image => vision}/8_class/resnet18_sc.txt (100%) rename {image => vision}/8_class/resnet18_st.txt (100%) rename {image => vision}/8_class/resnet18_test_time.txt (100%) rename {image => vision}/8_class/resnet18_test_time_sc.txt (100%) rename {image => vision}/8_class/resnet18_test_time_st.txt (100%) rename {image => vision}/8_class/resnet18_train_time.txt (100%) rename {image => vision}/8_class/resnet18_train_time_sc.txt (100%) rename {image => vision}/8_class/resnet18_train_time_st.txt (100%) rename {image => vision}/8_class/svhn_cnn32.txt (100%) rename {image => vision}/8_class/svhn_cnn32_2l.txt (100%) rename {image => vision}/8_class/svhn_cnn32_2l_test_time.txt (100%) rename {image => vision}/8_class/svhn_cnn32_2l_train_time.txt (100%) rename {image => vision}/8_class/svhn_cnn32_5l.txt (100%) rename {image => vision}/8_class/svhn_cnn32_5l_test_time.txt (100%) rename {image => vision}/8_class/svhn_cnn32_5l_train_time.txt (100%) rename {image => vision}/8_class/svhn_cnn32_test_time.txt (100%) rename {image => vision}/8_class/svhn_cnn32_train_time.txt (100%) rename {image => vision}/8_class/svhn_naive_rf.txt (100%) rename {image => vision}/8_class/svhn_naive_rf_test_time.txt (100%) rename {image => vision}/8_class/svhn_naive_rf_train_time.txt (100%) rename {image => vision}/8_class/svhn_resnet18.txt (100%) rename {image => vision}/8_class/svhn_resnet18_test_time.txt (100%) rename {image => vision}/8_class/svhn_resnet18_train_time.txt (100%) rename {image => vision}/8_class/svhn_svm.txt (100%) rename {image => vision}/8_class/svhn_svm_test_time.txt (100%) rename {image => vision}/8_class/svhn_svm_train_time.txt (100%) rename {image => vision}/8_class/svm.txt (100%) rename {image => vision}/8_class/svm_test_time.txt (100%) rename {image => vision}/8_class/svm_test_time_lc.txt (100%) rename {image => vision}/8_class/svm_train_time.txt (100%) rename {image => vision}/8_class/svm_train_time_lc.txt (100%) rename {image => vision}/90_class/cnn32.txt (100%) rename {image => vision}/90_class/cnn32_2l.txt (100%) rename {image => vision}/90_class/cnn32_2l_sc.txt (100%) rename {image => vision}/90_class/cnn32_2l_st.txt (100%) rename {image => vision}/90_class/cnn32_2l_test_time.txt (100%) rename {image => vision}/90_class/cnn32_2l_test_time_sc.txt (100%) rename {image => vision}/90_class/cnn32_2l_test_time_st.txt (100%) rename {image => vision}/90_class/cnn32_2l_train_time.txt (100%) rename {image => vision}/90_class/cnn32_2l_train_time_sc.txt (100%) rename {image => vision}/90_class/cnn32_2l_train_time_st.txt (100%) rename {image => vision}/90_class/cnn32_5l.txt (100%) rename {image => vision}/90_class/cnn32_5l_sc.txt (100%) rename {image => vision}/90_class/cnn32_5l_st.txt (100%) rename {image => vision}/90_class/cnn32_5l_test_time.txt (100%) rename {image => vision}/90_class/cnn32_5l_test_time_sc.txt (100%) rename {image => vision}/90_class/cnn32_5l_test_time_st.txt (100%) rename {image => vision}/90_class/cnn32_5l_train_time.txt (100%) rename {image => vision}/90_class/cnn32_5l_train_time_sc.txt (100%) rename {image => vision}/90_class/cnn32_5l_train_time_st.txt (100%) rename {image => vision}/90_class/cnn32_sc.txt (100%) rename {image => vision}/90_class/cnn32_st.txt (100%) rename {image => vision}/90_class/cnn32_test_time.txt (100%) rename {image => vision}/90_class/cnn32_test_time_sc.txt (100%) rename {image => vision}/90_class/cnn32_test_time_st.txt (100%) rename {image => vision}/90_class/cnn32_train_time.txt (100%) rename {image => vision}/90_class/cnn32_train_time_sc.txt (100%) rename {image => vision}/90_class/cnn32_train_time_st.txt (100%) rename {image => vision}/90_class/naive_rf.txt (100%) rename {image => vision}/90_class/naive_rf_test_time.txt (100%) rename {image => vision}/90_class/naive_rf_test_time_lc.txt (100%) rename {image => vision}/90_class/naive_rf_train_time.txt (100%) rename {image => vision}/90_class/naive_rf_train_time_lc.txt (100%) rename {image => vision}/90_class/resnet18.txt (100%) rename {image => vision}/90_class/resnet18_sc.txt (100%) rename {image => vision}/90_class/resnet18_st.txt (100%) rename {image => vision}/90_class/resnet18_test_time.txt (100%) rename {image => vision}/90_class/resnet18_test_time_sc.txt (100%) rename {image => vision}/90_class/resnet18_test_time_st.txt (100%) rename {image => vision}/90_class/resnet18_train_time.txt (100%) rename {image => vision}/90_class/resnet18_train_time_sc.txt (100%) rename {image => vision}/90_class/resnet18_train_time_st.txt (100%) rename {image => vision}/90_class/svm.txt (100%) rename {image => vision}/90_class/svm_test_time.txt (100%) rename {image => vision}/90_class/svm_test_time_lc.txt (100%) rename {image => vision}/90_class/svm_train_time.txt (100%) rename {image => vision}/90_class/svm_train_time_lc.txt (100%) rename {image => vision}/cifar_10.py (100%) rename {image => vision}/cifar_100.py (100%) rename {image => vision}/cifar_100_stc.py (100%) rename {image => vision}/cifar_10_stc.py (100%) rename {image => vision}/cifar_figures.ipynb (100%) rename {image => vision}/svhn.py (100%) rename {image => vision}/svhn_figure.ipynb (100%) rename {image => vision}/svhn_toolbox.py (100%) rename {image => vision}/toolbox.py (100%) diff --git a/audio/3_class/cnn32.txt b/audition/3_class/cnn32.txt similarity index 100% rename from audio/3_class/cnn32.txt rename to audition/3_class/cnn32.txt diff --git a/audio/3_class/cnn32_2l.txt b/audition/3_class/cnn32_2l.txt similarity index 100% rename from audio/3_class/cnn32_2l.txt rename to audition/3_class/cnn32_2l.txt diff --git a/audio/3_class/cnn32_2l_test_time.txt b/audition/3_class/cnn32_2l_test_time.txt similarity index 100% rename from audio/3_class/cnn32_2l_test_time.txt rename to audition/3_class/cnn32_2l_test_time.txt diff --git a/audio/3_class/cnn32_2l_train_time.txt b/audition/3_class/cnn32_2l_train_time.txt similarity index 100% rename from audio/3_class/cnn32_2l_train_time.txt rename to audition/3_class/cnn32_2l_train_time.txt diff --git a/audio/3_class/cnn32_5l.txt b/audition/3_class/cnn32_5l.txt similarity index 100% rename from audio/3_class/cnn32_5l.txt rename to audition/3_class/cnn32_5l.txt diff --git a/audio/3_class/cnn32_5l_test_time.txt b/audition/3_class/cnn32_5l_test_time.txt similarity index 100% rename from audio/3_class/cnn32_5l_test_time.txt rename to audition/3_class/cnn32_5l_test_time.txt diff --git a/audio/3_class/cnn32_5l_train_time.txt b/audition/3_class/cnn32_5l_train_time.txt similarity index 100% rename from audio/3_class/cnn32_5l_train_time.txt rename to audition/3_class/cnn32_5l_train_time.txt diff --git a/audio/3_class/cnn32_test_time.txt b/audition/3_class/cnn32_test_time.txt similarity index 100% rename from audio/3_class/cnn32_test_time.txt rename to audition/3_class/cnn32_test_time.txt diff --git a/audio/3_class/cnn32_train_time.txt b/audition/3_class/cnn32_train_time.txt similarity index 100% rename from audio/3_class/cnn32_train_time.txt rename to audition/3_class/cnn32_train_time.txt diff --git a/audio/3_class/naive_rf.txt b/audition/3_class/naive_rf.txt similarity index 100% rename from audio/3_class/naive_rf.txt rename to audition/3_class/naive_rf.txt diff --git a/audio/3_class/naive_rf_test_time.txt b/audition/3_class/naive_rf_test_time.txt similarity index 100% rename from audio/3_class/naive_rf_test_time.txt rename to audition/3_class/naive_rf_test_time.txt diff --git a/audio/3_class/naive_rf_train_time.txt b/audition/3_class/naive_rf_train_time.txt similarity index 100% rename from audio/3_class/naive_rf_train_time.txt rename to audition/3_class/naive_rf_train_time.txt diff --git a/audio/3_class/resnet18.txt b/audition/3_class/resnet18.txt similarity index 100% rename from audio/3_class/resnet18.txt rename to audition/3_class/resnet18.txt diff --git a/audio/3_class/resnet18_test_time.txt b/audition/3_class/resnet18_test_time.txt similarity index 100% rename from audio/3_class/resnet18_test_time.txt rename to audition/3_class/resnet18_test_time.txt diff --git a/audio/3_class/resnet18_train_time.txt b/audition/3_class/resnet18_train_time.txt similarity index 100% rename from audio/3_class/resnet18_train_time.txt rename to audition/3_class/resnet18_train_time.txt diff --git a/audio/3_class/svm.txt b/audition/3_class/svm.txt similarity index 100% rename from audio/3_class/svm.txt rename to audition/3_class/svm.txt diff --git a/audio/3_class/svm_test_time.txt b/audition/3_class/svm_test_time.txt similarity index 100% rename from audio/3_class/svm_test_time.txt rename to audition/3_class/svm_test_time.txt diff --git a/audio/3_class/svm_train_time.txt b/audition/3_class/svm_train_time.txt similarity index 100% rename from audio/3_class/svm_train_time.txt rename to audition/3_class/svm_train_time.txt diff --git a/audio/3_class_mel/cnn32.txt b/audition/3_class_mel/cnn32.txt similarity index 100% rename from audio/3_class_mel/cnn32.txt rename to audition/3_class_mel/cnn32.txt diff --git a/audio/3_class_mel/cnn32_2l.txt b/audition/3_class_mel/cnn32_2l.txt similarity index 100% rename from audio/3_class_mel/cnn32_2l.txt rename to audition/3_class_mel/cnn32_2l.txt diff --git a/audio/3_class_mel/cnn32_2l_test_time.txt b/audition/3_class_mel/cnn32_2l_test_time.txt similarity index 100% rename from audio/3_class_mel/cnn32_2l_test_time.txt rename to audition/3_class_mel/cnn32_2l_test_time.txt diff --git a/audio/3_class_mel/cnn32_2l_train_time.txt b/audition/3_class_mel/cnn32_2l_train_time.txt similarity index 100% rename from audio/3_class_mel/cnn32_2l_train_time.txt rename to audition/3_class_mel/cnn32_2l_train_time.txt diff --git a/audio/3_class_mel/cnn32_5l.txt b/audition/3_class_mel/cnn32_5l.txt similarity index 100% rename from audio/3_class_mel/cnn32_5l.txt rename to audition/3_class_mel/cnn32_5l.txt diff --git a/audio/3_class_mel/cnn32_5l_test_time.txt b/audition/3_class_mel/cnn32_5l_test_time.txt similarity index 100% rename from audio/3_class_mel/cnn32_5l_test_time.txt rename to audition/3_class_mel/cnn32_5l_test_time.txt diff --git a/audio/3_class_mel/cnn32_5l_train_time.txt b/audition/3_class_mel/cnn32_5l_train_time.txt similarity index 100% rename from audio/3_class_mel/cnn32_5l_train_time.txt rename to audition/3_class_mel/cnn32_5l_train_time.txt diff --git a/audio/3_class_mel/cnn32_test_time.txt b/audition/3_class_mel/cnn32_test_time.txt similarity index 100% rename from audio/3_class_mel/cnn32_test_time.txt rename to audition/3_class_mel/cnn32_test_time.txt diff --git a/audio/3_class_mel/cnn32_train_time.txt b/audition/3_class_mel/cnn32_train_time.txt similarity index 100% rename from audio/3_class_mel/cnn32_train_time.txt rename to audition/3_class_mel/cnn32_train_time.txt diff --git a/audio/3_class_mel/naive_rf.txt b/audition/3_class_mel/naive_rf.txt similarity index 100% rename from audio/3_class_mel/naive_rf.txt rename to audition/3_class_mel/naive_rf.txt diff --git a/audio/3_class_mel/naive_rf_test_time.txt b/audition/3_class_mel/naive_rf_test_time.txt similarity index 100% rename from audio/3_class_mel/naive_rf_test_time.txt rename to audition/3_class_mel/naive_rf_test_time.txt diff --git a/audio/3_class_mel/naive_rf_train_time.txt b/audition/3_class_mel/naive_rf_train_time.txt similarity index 100% rename from audio/3_class_mel/naive_rf_train_time.txt rename to audition/3_class_mel/naive_rf_train_time.txt diff --git a/audio/3_class_mel/resnet18.txt b/audition/3_class_mel/resnet18.txt similarity index 100% rename from audio/3_class_mel/resnet18.txt rename to audition/3_class_mel/resnet18.txt diff --git a/audio/3_class_mel/resnet18_test_time.txt b/audition/3_class_mel/resnet18_test_time.txt similarity index 100% rename from audio/3_class_mel/resnet18_test_time.txt rename to audition/3_class_mel/resnet18_test_time.txt diff --git a/audio/3_class_mel/resnet18_train_time.txt b/audition/3_class_mel/resnet18_train_time.txt similarity index 100% rename from audio/3_class_mel/resnet18_train_time.txt rename to audition/3_class_mel/resnet18_train_time.txt diff --git a/audio/3_class_mel/svm.txt b/audition/3_class_mel/svm.txt similarity index 100% rename from audio/3_class_mel/svm.txt rename to audition/3_class_mel/svm.txt diff --git a/audio/3_class_mel/svm_test_time.txt b/audition/3_class_mel/svm_test_time.txt similarity index 100% rename from audio/3_class_mel/svm_test_time.txt rename to audition/3_class_mel/svm_test_time.txt diff --git a/audio/3_class_mel/svm_train_time.txt b/audition/3_class_mel/svm_train_time.txt similarity index 100% rename from audio/3_class_mel/svm_train_time.txt rename to audition/3_class_mel/svm_train_time.txt diff --git a/audio/3_class_mfcc/cnn32.txt b/audition/3_class_mfcc/cnn32.txt similarity index 100% rename from audio/3_class_mfcc/cnn32.txt rename to audition/3_class_mfcc/cnn32.txt diff --git a/audio/3_class_mfcc/cnn32_2l.txt b/audition/3_class_mfcc/cnn32_2l.txt similarity index 100% rename from audio/3_class_mfcc/cnn32_2l.txt rename to audition/3_class_mfcc/cnn32_2l.txt diff --git a/audio/3_class_mfcc/cnn32_2l_test_time.txt b/audition/3_class_mfcc/cnn32_2l_test_time.txt similarity index 100% rename from audio/3_class_mfcc/cnn32_2l_test_time.txt rename to audition/3_class_mfcc/cnn32_2l_test_time.txt diff --git a/audio/3_class_mfcc/cnn32_2l_train_time.txt b/audition/3_class_mfcc/cnn32_2l_train_time.txt similarity index 100% rename from audio/3_class_mfcc/cnn32_2l_train_time.txt rename to audition/3_class_mfcc/cnn32_2l_train_time.txt diff --git a/audio/3_class_mfcc/cnn32_5l.txt b/audition/3_class_mfcc/cnn32_5l.txt similarity index 100% rename from audio/3_class_mfcc/cnn32_5l.txt rename to audition/3_class_mfcc/cnn32_5l.txt diff --git a/audio/3_class_mfcc/cnn32_5l_test_time.txt b/audition/3_class_mfcc/cnn32_5l_test_time.txt similarity index 100% rename from audio/3_class_mfcc/cnn32_5l_test_time.txt rename to audition/3_class_mfcc/cnn32_5l_test_time.txt diff --git a/audio/3_class_mfcc/cnn32_5l_train_time.txt b/audition/3_class_mfcc/cnn32_5l_train_time.txt similarity index 100% rename from audio/3_class_mfcc/cnn32_5l_train_time.txt rename to audition/3_class_mfcc/cnn32_5l_train_time.txt diff --git a/audio/3_class_mfcc/cnn32_test_time.txt b/audition/3_class_mfcc/cnn32_test_time.txt similarity index 100% rename from audio/3_class_mfcc/cnn32_test_time.txt rename to audition/3_class_mfcc/cnn32_test_time.txt diff --git a/audio/3_class_mfcc/cnn32_train_time.txt b/audition/3_class_mfcc/cnn32_train_time.txt similarity index 100% rename from audio/3_class_mfcc/cnn32_train_time.txt rename to audition/3_class_mfcc/cnn32_train_time.txt diff --git a/audio/3_class_mfcc/naive_rf.txt b/audition/3_class_mfcc/naive_rf.txt similarity index 100% rename from audio/3_class_mfcc/naive_rf.txt rename to audition/3_class_mfcc/naive_rf.txt diff --git a/audio/3_class_mfcc/naive_rf_test_time.txt b/audition/3_class_mfcc/naive_rf_test_time.txt similarity index 100% rename from audio/3_class_mfcc/naive_rf_test_time.txt rename to audition/3_class_mfcc/naive_rf_test_time.txt diff --git a/audio/3_class_mfcc/naive_rf_train_time.txt b/audition/3_class_mfcc/naive_rf_train_time.txt similarity index 100% rename from audio/3_class_mfcc/naive_rf_train_time.txt rename to audition/3_class_mfcc/naive_rf_train_time.txt diff --git a/audio/3_class_mfcc/resnet18.txt b/audition/3_class_mfcc/resnet18.txt similarity index 100% rename from audio/3_class_mfcc/resnet18.txt rename to audition/3_class_mfcc/resnet18.txt diff --git a/audio/3_class_mfcc/resnet18_test_time.txt b/audition/3_class_mfcc/resnet18_test_time.txt similarity index 100% rename from audio/3_class_mfcc/resnet18_test_time.txt rename to audition/3_class_mfcc/resnet18_test_time.txt diff --git a/audio/3_class_mfcc/resnet18_train_time.txt b/audition/3_class_mfcc/resnet18_train_time.txt similarity index 100% rename from audio/3_class_mfcc/resnet18_train_time.txt rename to audition/3_class_mfcc/resnet18_train_time.txt diff --git a/audio/3_class_mfcc/svm.txt b/audition/3_class_mfcc/svm.txt similarity index 100% rename from audio/3_class_mfcc/svm.txt rename to audition/3_class_mfcc/svm.txt diff --git a/audio/3_class_mfcc/svm_test_time.txt b/audition/3_class_mfcc/svm_test_time.txt similarity index 100% rename from audio/3_class_mfcc/svm_test_time.txt rename to audition/3_class_mfcc/svm_test_time.txt diff --git a/audio/3_class_mfcc/svm_train_time.txt b/audition/3_class_mfcc/svm_train_time.txt similarity index 100% rename from audio/3_class_mfcc/svm_train_time.txt rename to audition/3_class_mfcc/svm_train_time.txt diff --git a/audio/8_class/cnn32.txt b/audition/8_class/cnn32.txt similarity index 100% rename from audio/8_class/cnn32.txt rename to audition/8_class/cnn32.txt diff --git a/audio/8_class/cnn32_2l.txt b/audition/8_class/cnn32_2l.txt similarity index 100% rename from audio/8_class/cnn32_2l.txt rename to audition/8_class/cnn32_2l.txt diff --git a/audio/8_class/cnn32_2l_test_time.txt b/audition/8_class/cnn32_2l_test_time.txt similarity index 100% rename from audio/8_class/cnn32_2l_test_time.txt rename to audition/8_class/cnn32_2l_test_time.txt diff --git a/audio/8_class/cnn32_2l_train_time.txt b/audition/8_class/cnn32_2l_train_time.txt similarity index 100% rename from audio/8_class/cnn32_2l_train_time.txt rename to audition/8_class/cnn32_2l_train_time.txt diff --git a/audio/8_class/cnn32_5l.txt b/audition/8_class/cnn32_5l.txt similarity index 100% rename from audio/8_class/cnn32_5l.txt rename to audition/8_class/cnn32_5l.txt diff --git a/audio/8_class/cnn32_5l_test_time.txt b/audition/8_class/cnn32_5l_test_time.txt similarity index 100% rename from audio/8_class/cnn32_5l_test_time.txt rename to audition/8_class/cnn32_5l_test_time.txt diff --git a/audio/8_class/cnn32_5l_train_time.txt b/audition/8_class/cnn32_5l_train_time.txt similarity index 100% rename from audio/8_class/cnn32_5l_train_time.txt rename to audition/8_class/cnn32_5l_train_time.txt diff --git a/audio/8_class/cnn32_test_time.txt b/audition/8_class/cnn32_test_time.txt similarity index 100% rename from audio/8_class/cnn32_test_time.txt rename to audition/8_class/cnn32_test_time.txt diff --git a/audio/8_class/cnn32_train_time.txt b/audition/8_class/cnn32_train_time.txt similarity index 100% rename from audio/8_class/cnn32_train_time.txt rename to audition/8_class/cnn32_train_time.txt diff --git a/audio/8_class/naive_rf.txt b/audition/8_class/naive_rf.txt similarity index 100% rename from audio/8_class/naive_rf.txt rename to audition/8_class/naive_rf.txt diff --git a/audio/8_class/naive_rf_test_time.txt b/audition/8_class/naive_rf_test_time.txt similarity index 100% rename from audio/8_class/naive_rf_test_time.txt rename to audition/8_class/naive_rf_test_time.txt diff --git a/audio/8_class/naive_rf_train_time.txt b/audition/8_class/naive_rf_train_time.txt similarity index 100% rename from audio/8_class/naive_rf_train_time.txt rename to audition/8_class/naive_rf_train_time.txt diff --git a/audio/8_class/resnet18.txt b/audition/8_class/resnet18.txt similarity index 100% rename from audio/8_class/resnet18.txt rename to audition/8_class/resnet18.txt diff --git a/audio/8_class/resnet18_test_time.txt b/audition/8_class/resnet18_test_time.txt similarity index 100% rename from audio/8_class/resnet18_test_time.txt rename to audition/8_class/resnet18_test_time.txt diff --git a/audio/8_class/resnet18_train_time.txt b/audition/8_class/resnet18_train_time.txt similarity index 100% rename from audio/8_class/resnet18_train_time.txt rename to audition/8_class/resnet18_train_time.txt diff --git a/audio/8_class/svm.txt b/audition/8_class/svm.txt similarity index 100% rename from audio/8_class/svm.txt rename to audition/8_class/svm.txt diff --git a/audio/8_class/svm_test_time.txt b/audition/8_class/svm_test_time.txt similarity index 100% rename from audio/8_class/svm_test_time.txt rename to audition/8_class/svm_test_time.txt diff --git a/audio/8_class/svm_train_time.txt b/audition/8_class/svm_train_time.txt similarity index 100% rename from audio/8_class/svm_train_time.txt rename to audition/8_class/svm_train_time.txt diff --git a/audio/8_class_mel/cnn32.txt b/audition/8_class_mel/cnn32.txt similarity index 100% rename from audio/8_class_mel/cnn32.txt rename to audition/8_class_mel/cnn32.txt diff --git a/audio/8_class_mel/cnn32_2l.txt b/audition/8_class_mel/cnn32_2l.txt similarity index 100% rename from audio/8_class_mel/cnn32_2l.txt rename to audition/8_class_mel/cnn32_2l.txt diff --git a/audio/8_class_mel/cnn32_2l_test_time.txt b/audition/8_class_mel/cnn32_2l_test_time.txt similarity index 100% rename from audio/8_class_mel/cnn32_2l_test_time.txt rename to audition/8_class_mel/cnn32_2l_test_time.txt diff --git a/audio/8_class_mel/cnn32_2l_train_time.txt b/audition/8_class_mel/cnn32_2l_train_time.txt similarity index 100% rename from audio/8_class_mel/cnn32_2l_train_time.txt rename to audition/8_class_mel/cnn32_2l_train_time.txt diff --git a/audio/8_class_mel/cnn32_5l.txt b/audition/8_class_mel/cnn32_5l.txt similarity index 100% rename from audio/8_class_mel/cnn32_5l.txt rename to audition/8_class_mel/cnn32_5l.txt diff --git a/audio/8_class_mel/cnn32_5l_test_time.txt b/audition/8_class_mel/cnn32_5l_test_time.txt similarity index 100% rename from audio/8_class_mel/cnn32_5l_test_time.txt rename to audition/8_class_mel/cnn32_5l_test_time.txt diff --git a/audio/8_class_mel/cnn32_5l_train_time.txt b/audition/8_class_mel/cnn32_5l_train_time.txt similarity index 100% rename from audio/8_class_mel/cnn32_5l_train_time.txt rename to audition/8_class_mel/cnn32_5l_train_time.txt diff --git a/audio/8_class_mel/cnn32_test_time.txt b/audition/8_class_mel/cnn32_test_time.txt similarity index 100% rename from audio/8_class_mel/cnn32_test_time.txt rename to audition/8_class_mel/cnn32_test_time.txt diff --git a/audio/8_class_mel/cnn32_train_time.txt b/audition/8_class_mel/cnn32_train_time.txt similarity index 100% rename from audio/8_class_mel/cnn32_train_time.txt rename to audition/8_class_mel/cnn32_train_time.txt diff --git a/audio/8_class_mel/naive_rf.txt b/audition/8_class_mel/naive_rf.txt similarity index 100% rename from audio/8_class_mel/naive_rf.txt rename to audition/8_class_mel/naive_rf.txt diff --git a/audio/8_class_mel/naive_rf_test_time.txt b/audition/8_class_mel/naive_rf_test_time.txt similarity index 100% rename from audio/8_class_mel/naive_rf_test_time.txt rename to audition/8_class_mel/naive_rf_test_time.txt diff --git a/audio/8_class_mel/naive_rf_train_time.txt b/audition/8_class_mel/naive_rf_train_time.txt similarity index 100% rename from audio/8_class_mel/naive_rf_train_time.txt rename to audition/8_class_mel/naive_rf_train_time.txt diff --git a/audio/8_class_mel/resnet18.txt b/audition/8_class_mel/resnet18.txt similarity index 100% rename from audio/8_class_mel/resnet18.txt rename to audition/8_class_mel/resnet18.txt diff --git a/audio/8_class_mel/resnet18_test_time.txt b/audition/8_class_mel/resnet18_test_time.txt similarity index 100% rename from audio/8_class_mel/resnet18_test_time.txt rename to audition/8_class_mel/resnet18_test_time.txt diff --git a/audio/8_class_mel/resnet18_train_time.txt b/audition/8_class_mel/resnet18_train_time.txt similarity index 100% rename from audio/8_class_mel/resnet18_train_time.txt rename to audition/8_class_mel/resnet18_train_time.txt diff --git a/audio/8_class_mel/svm.txt b/audition/8_class_mel/svm.txt similarity index 100% rename from audio/8_class_mel/svm.txt rename to audition/8_class_mel/svm.txt diff --git a/audio/8_class_mel/svm_test_time.txt b/audition/8_class_mel/svm_test_time.txt similarity index 100% rename from audio/8_class_mel/svm_test_time.txt rename to audition/8_class_mel/svm_test_time.txt diff --git a/audio/8_class_mel/svm_train_time.txt b/audition/8_class_mel/svm_train_time.txt similarity index 100% rename from audio/8_class_mel/svm_train_time.txt rename to audition/8_class_mel/svm_train_time.txt diff --git a/audio/8_class_mfcc/cnn32.txt b/audition/8_class_mfcc/cnn32.txt similarity index 100% rename from audio/8_class_mfcc/cnn32.txt rename to audition/8_class_mfcc/cnn32.txt diff --git a/audio/8_class_mfcc/cnn32_2l.txt b/audition/8_class_mfcc/cnn32_2l.txt similarity index 100% rename from audio/8_class_mfcc/cnn32_2l.txt rename to audition/8_class_mfcc/cnn32_2l.txt diff --git a/audio/8_class_mfcc/cnn32_2l_test_time.txt b/audition/8_class_mfcc/cnn32_2l_test_time.txt similarity index 100% rename from audio/8_class_mfcc/cnn32_2l_test_time.txt rename to audition/8_class_mfcc/cnn32_2l_test_time.txt diff --git a/audio/8_class_mfcc/cnn32_2l_train_time.txt b/audition/8_class_mfcc/cnn32_2l_train_time.txt similarity index 100% rename from audio/8_class_mfcc/cnn32_2l_train_time.txt rename to audition/8_class_mfcc/cnn32_2l_train_time.txt diff --git a/audio/8_class_mfcc/cnn32_5l.txt b/audition/8_class_mfcc/cnn32_5l.txt similarity index 100% rename from audio/8_class_mfcc/cnn32_5l.txt rename to audition/8_class_mfcc/cnn32_5l.txt diff --git a/audio/8_class_mfcc/cnn32_5l_test_time.txt b/audition/8_class_mfcc/cnn32_5l_test_time.txt similarity index 100% rename from audio/8_class_mfcc/cnn32_5l_test_time.txt rename to audition/8_class_mfcc/cnn32_5l_test_time.txt diff --git a/audio/8_class_mfcc/cnn32_5l_train_time.txt b/audition/8_class_mfcc/cnn32_5l_train_time.txt similarity index 100% rename from audio/8_class_mfcc/cnn32_5l_train_time.txt rename to audition/8_class_mfcc/cnn32_5l_train_time.txt diff --git a/audio/8_class_mfcc/cnn32_test_time.txt b/audition/8_class_mfcc/cnn32_test_time.txt similarity index 100% rename from audio/8_class_mfcc/cnn32_test_time.txt rename to audition/8_class_mfcc/cnn32_test_time.txt diff --git a/audio/8_class_mfcc/cnn32_train_time.txt b/audition/8_class_mfcc/cnn32_train_time.txt similarity index 100% rename from audio/8_class_mfcc/cnn32_train_time.txt rename to audition/8_class_mfcc/cnn32_train_time.txt diff --git a/audio/8_class_mfcc/naive_rf.txt b/audition/8_class_mfcc/naive_rf.txt similarity index 100% rename from audio/8_class_mfcc/naive_rf.txt rename to audition/8_class_mfcc/naive_rf.txt diff --git a/audio/8_class_mfcc/naive_rf_test_time.txt b/audition/8_class_mfcc/naive_rf_test_time.txt similarity index 100% rename from audio/8_class_mfcc/naive_rf_test_time.txt rename to audition/8_class_mfcc/naive_rf_test_time.txt diff --git a/audio/8_class_mfcc/naive_rf_train_time.txt b/audition/8_class_mfcc/naive_rf_train_time.txt similarity index 100% rename from audio/8_class_mfcc/naive_rf_train_time.txt rename to audition/8_class_mfcc/naive_rf_train_time.txt diff --git a/audio/8_class_mfcc/resnet18.txt b/audition/8_class_mfcc/resnet18.txt similarity index 100% rename from audio/8_class_mfcc/resnet18.txt rename to audition/8_class_mfcc/resnet18.txt diff --git a/audio/8_class_mfcc/resnet18_test_time.txt b/audition/8_class_mfcc/resnet18_test_time.txt similarity index 100% rename from audio/8_class_mfcc/resnet18_test_time.txt rename to audition/8_class_mfcc/resnet18_test_time.txt diff --git a/audio/8_class_mfcc/resnet18_train_time.txt b/audition/8_class_mfcc/resnet18_train_time.txt similarity index 100% rename from audio/8_class_mfcc/resnet18_train_time.txt rename to audition/8_class_mfcc/resnet18_train_time.txt diff --git a/audio/8_class_mfcc/svm.txt b/audition/8_class_mfcc/svm.txt similarity index 100% rename from audio/8_class_mfcc/svm.txt rename to audition/8_class_mfcc/svm.txt diff --git a/audio/8_class_mfcc/svm_test_time.txt b/audition/8_class_mfcc/svm_test_time.txt similarity index 100% rename from audio/8_class_mfcc/svm_test_time.txt rename to audition/8_class_mfcc/svm_test_time.txt diff --git a/audio/8_class_mfcc/svm_train_time.txt b/audition/8_class_mfcc/svm_train_time.txt similarity index 100% rename from audio/8_class_mfcc/svm_train_time.txt rename to audition/8_class_mfcc/svm_train_time.txt diff --git a/audio/audio_toolbox.py b/audition/audio_toolbox.py similarity index 100% rename from audio/audio_toolbox.py rename to audition/audio_toolbox.py diff --git a/audio/fsdd.py b/audition/fsdd.py similarity index 100% rename from audio/fsdd.py rename to audition/fsdd.py diff --git a/audio/fsdd_figures.ipynb b/audition/fsdd_figures.ipynb similarity index 100% rename from audio/fsdd_figures.ipynb rename to audition/fsdd_figures.ipynb diff --git a/image/3_class/cnn32.txt b/vision/3_class/cnn32.txt similarity index 100% rename from image/3_class/cnn32.txt rename to vision/3_class/cnn32.txt diff --git a/image/3_class/cnn32_2l.txt b/vision/3_class/cnn32_2l.txt similarity index 100% rename from image/3_class/cnn32_2l.txt rename to vision/3_class/cnn32_2l.txt diff --git a/image/3_class/cnn32_2l_sc.txt b/vision/3_class/cnn32_2l_sc.txt similarity index 100% rename from image/3_class/cnn32_2l_sc.txt rename to vision/3_class/cnn32_2l_sc.txt diff --git a/image/3_class/cnn32_2l_st.txt b/vision/3_class/cnn32_2l_st.txt similarity index 100% rename from image/3_class/cnn32_2l_st.txt rename to vision/3_class/cnn32_2l_st.txt diff --git a/image/3_class/cnn32_2l_test_time.txt b/vision/3_class/cnn32_2l_test_time.txt similarity index 100% rename from image/3_class/cnn32_2l_test_time.txt rename to vision/3_class/cnn32_2l_test_time.txt diff --git a/image/3_class/cnn32_2l_test_time_sc.txt b/vision/3_class/cnn32_2l_test_time_sc.txt similarity index 100% rename from image/3_class/cnn32_2l_test_time_sc.txt rename to vision/3_class/cnn32_2l_test_time_sc.txt diff --git a/image/3_class/cnn32_2l_test_time_st.txt b/vision/3_class/cnn32_2l_test_time_st.txt similarity index 100% rename from image/3_class/cnn32_2l_test_time_st.txt rename to vision/3_class/cnn32_2l_test_time_st.txt diff --git a/image/3_class/cnn32_2l_train_time.txt b/vision/3_class/cnn32_2l_train_time.txt similarity index 100% rename from image/3_class/cnn32_2l_train_time.txt rename to vision/3_class/cnn32_2l_train_time.txt diff --git a/image/3_class/cnn32_2l_train_time_sc.txt b/vision/3_class/cnn32_2l_train_time_sc.txt similarity index 100% rename from image/3_class/cnn32_2l_train_time_sc.txt rename to vision/3_class/cnn32_2l_train_time_sc.txt diff --git a/image/3_class/cnn32_2l_train_time_st.txt b/vision/3_class/cnn32_2l_train_time_st.txt similarity index 100% rename from image/3_class/cnn32_2l_train_time_st.txt rename to vision/3_class/cnn32_2l_train_time_st.txt diff --git a/image/3_class/cnn32_5l.txt b/vision/3_class/cnn32_5l.txt similarity index 100% rename from image/3_class/cnn32_5l.txt rename to vision/3_class/cnn32_5l.txt diff --git a/image/3_class/cnn32_5l_sc.txt b/vision/3_class/cnn32_5l_sc.txt similarity index 100% rename from image/3_class/cnn32_5l_sc.txt rename to vision/3_class/cnn32_5l_sc.txt diff --git a/image/3_class/cnn32_5l_st.txt b/vision/3_class/cnn32_5l_st.txt similarity index 100% rename from image/3_class/cnn32_5l_st.txt rename to vision/3_class/cnn32_5l_st.txt diff --git a/image/3_class/cnn32_5l_test_time.txt b/vision/3_class/cnn32_5l_test_time.txt similarity index 100% rename from image/3_class/cnn32_5l_test_time.txt rename to vision/3_class/cnn32_5l_test_time.txt diff --git a/image/3_class/cnn32_5l_test_time_sc.txt b/vision/3_class/cnn32_5l_test_time_sc.txt similarity index 100% rename from image/3_class/cnn32_5l_test_time_sc.txt rename to vision/3_class/cnn32_5l_test_time_sc.txt diff --git a/image/3_class/cnn32_5l_test_time_st.txt b/vision/3_class/cnn32_5l_test_time_st.txt similarity index 100% rename from image/3_class/cnn32_5l_test_time_st.txt rename to vision/3_class/cnn32_5l_test_time_st.txt diff --git a/image/3_class/cnn32_5l_train_time.txt b/vision/3_class/cnn32_5l_train_time.txt similarity index 100% rename from image/3_class/cnn32_5l_train_time.txt rename to vision/3_class/cnn32_5l_train_time.txt diff --git a/image/3_class/cnn32_5l_train_time_sc.txt b/vision/3_class/cnn32_5l_train_time_sc.txt similarity index 100% rename from image/3_class/cnn32_5l_train_time_sc.txt rename to vision/3_class/cnn32_5l_train_time_sc.txt diff --git a/image/3_class/cnn32_5l_train_time_st.txt b/vision/3_class/cnn32_5l_train_time_st.txt similarity index 100% rename from image/3_class/cnn32_5l_train_time_st.txt rename to vision/3_class/cnn32_5l_train_time_st.txt diff --git a/image/3_class/cnn32_sc.txt b/vision/3_class/cnn32_sc.txt similarity index 100% rename from image/3_class/cnn32_sc.txt rename to vision/3_class/cnn32_sc.txt diff --git a/image/3_class/cnn32_st.txt b/vision/3_class/cnn32_st.txt similarity index 100% rename from image/3_class/cnn32_st.txt rename to vision/3_class/cnn32_st.txt diff --git a/image/3_class/cnn32_test_time.txt b/vision/3_class/cnn32_test_time.txt similarity index 100% rename from image/3_class/cnn32_test_time.txt rename to vision/3_class/cnn32_test_time.txt diff --git a/image/3_class/cnn32_test_time_sc.txt b/vision/3_class/cnn32_test_time_sc.txt similarity index 100% rename from image/3_class/cnn32_test_time_sc.txt rename to vision/3_class/cnn32_test_time_sc.txt diff --git a/image/3_class/cnn32_test_time_st.txt b/vision/3_class/cnn32_test_time_st.txt similarity index 100% rename from image/3_class/cnn32_test_time_st.txt rename to vision/3_class/cnn32_test_time_st.txt diff --git a/image/3_class/cnn32_train_time.txt b/vision/3_class/cnn32_train_time.txt similarity index 100% rename from image/3_class/cnn32_train_time.txt rename to vision/3_class/cnn32_train_time.txt diff --git a/image/3_class/cnn32_train_time_sc.txt b/vision/3_class/cnn32_train_time_sc.txt similarity index 100% rename from image/3_class/cnn32_train_time_sc.txt rename to vision/3_class/cnn32_train_time_sc.txt diff --git a/image/3_class/cnn32_train_time_st.txt b/vision/3_class/cnn32_train_time_st.txt similarity index 100% rename from image/3_class/cnn32_train_time_st.txt rename to vision/3_class/cnn32_train_time_st.txt diff --git a/image/3_class/naive_rf.txt b/vision/3_class/naive_rf.txt similarity index 100% rename from image/3_class/naive_rf.txt rename to vision/3_class/naive_rf.txt diff --git a/image/3_class/naive_rf_test_time.txt b/vision/3_class/naive_rf_test_time.txt similarity index 100% rename from image/3_class/naive_rf_test_time.txt rename to vision/3_class/naive_rf_test_time.txt diff --git a/image/3_class/naive_rf_test_time_lc.txt b/vision/3_class/naive_rf_test_time_lc.txt similarity index 100% rename from image/3_class/naive_rf_test_time_lc.txt rename to vision/3_class/naive_rf_test_time_lc.txt diff --git a/image/3_class/naive_rf_train_time.txt b/vision/3_class/naive_rf_train_time.txt similarity index 100% rename from image/3_class/naive_rf_train_time.txt rename to vision/3_class/naive_rf_train_time.txt diff --git a/image/3_class/naive_rf_train_time_lc.txt b/vision/3_class/naive_rf_train_time_lc.txt similarity index 100% rename from image/3_class/naive_rf_train_time_lc.txt rename to vision/3_class/naive_rf_train_time_lc.txt diff --git a/image/3_class/resnet18.txt b/vision/3_class/resnet18.txt similarity index 100% rename from image/3_class/resnet18.txt rename to vision/3_class/resnet18.txt diff --git a/image/3_class/resnet18_sc.txt b/vision/3_class/resnet18_sc.txt similarity index 100% rename from image/3_class/resnet18_sc.txt rename to vision/3_class/resnet18_sc.txt diff --git a/image/3_class/resnet18_st.txt b/vision/3_class/resnet18_st.txt similarity index 100% rename from image/3_class/resnet18_st.txt rename to vision/3_class/resnet18_st.txt diff --git a/image/3_class/resnet18_test_time.txt b/vision/3_class/resnet18_test_time.txt similarity index 100% rename from image/3_class/resnet18_test_time.txt rename to vision/3_class/resnet18_test_time.txt diff --git a/image/3_class/resnet18_test_time_sc.txt b/vision/3_class/resnet18_test_time_sc.txt similarity index 100% rename from image/3_class/resnet18_test_time_sc.txt rename to vision/3_class/resnet18_test_time_sc.txt diff --git a/image/3_class/resnet18_test_time_st.txt b/vision/3_class/resnet18_test_time_st.txt similarity index 100% rename from image/3_class/resnet18_test_time_st.txt rename to vision/3_class/resnet18_test_time_st.txt diff --git a/image/3_class/resnet18_train_time.txt b/vision/3_class/resnet18_train_time.txt similarity index 100% rename from image/3_class/resnet18_train_time.txt rename to vision/3_class/resnet18_train_time.txt diff --git a/image/3_class/resnet18_train_time_sc.txt b/vision/3_class/resnet18_train_time_sc.txt similarity index 100% rename from image/3_class/resnet18_train_time_sc.txt rename to vision/3_class/resnet18_train_time_sc.txt diff --git a/image/3_class/resnet18_train_time_st.txt b/vision/3_class/resnet18_train_time_st.txt similarity index 100% rename from image/3_class/resnet18_train_time_st.txt rename to vision/3_class/resnet18_train_time_st.txt diff --git a/image/3_class/svhn_cnn32.txt b/vision/3_class/svhn_cnn32.txt similarity index 100% rename from image/3_class/svhn_cnn32.txt rename to vision/3_class/svhn_cnn32.txt diff --git a/image/3_class/svhn_cnn32_2l.txt b/vision/3_class/svhn_cnn32_2l.txt similarity index 100% rename from image/3_class/svhn_cnn32_2l.txt rename to vision/3_class/svhn_cnn32_2l.txt diff --git a/image/3_class/svhn_cnn32_2l_test_time.txt b/vision/3_class/svhn_cnn32_2l_test_time.txt similarity index 100% rename from image/3_class/svhn_cnn32_2l_test_time.txt rename to vision/3_class/svhn_cnn32_2l_test_time.txt diff --git a/image/3_class/svhn_cnn32_2l_train_time.txt b/vision/3_class/svhn_cnn32_2l_train_time.txt similarity index 100% rename from image/3_class/svhn_cnn32_2l_train_time.txt rename to vision/3_class/svhn_cnn32_2l_train_time.txt diff --git a/image/3_class/svhn_cnn32_5l.txt b/vision/3_class/svhn_cnn32_5l.txt similarity index 100% rename from image/3_class/svhn_cnn32_5l.txt rename to vision/3_class/svhn_cnn32_5l.txt diff --git a/image/3_class/svhn_cnn32_5l_test_time.txt b/vision/3_class/svhn_cnn32_5l_test_time.txt similarity index 100% rename from image/3_class/svhn_cnn32_5l_test_time.txt rename to vision/3_class/svhn_cnn32_5l_test_time.txt diff --git a/image/3_class/svhn_cnn32_5l_train_time.txt b/vision/3_class/svhn_cnn32_5l_train_time.txt similarity index 100% rename from image/3_class/svhn_cnn32_5l_train_time.txt rename to vision/3_class/svhn_cnn32_5l_train_time.txt diff --git a/image/3_class/svhn_cnn32_test_time.txt b/vision/3_class/svhn_cnn32_test_time.txt similarity index 100% rename from image/3_class/svhn_cnn32_test_time.txt rename to vision/3_class/svhn_cnn32_test_time.txt diff --git a/image/3_class/svhn_cnn32_train_time.txt b/vision/3_class/svhn_cnn32_train_time.txt similarity index 100% rename from image/3_class/svhn_cnn32_train_time.txt rename to vision/3_class/svhn_cnn32_train_time.txt diff --git a/image/3_class/svhn_naive_rf.txt b/vision/3_class/svhn_naive_rf.txt similarity index 100% rename from image/3_class/svhn_naive_rf.txt rename to vision/3_class/svhn_naive_rf.txt diff --git a/image/3_class/svhn_naive_rf_test_time.txt b/vision/3_class/svhn_naive_rf_test_time.txt similarity index 100% rename from image/3_class/svhn_naive_rf_test_time.txt rename to vision/3_class/svhn_naive_rf_test_time.txt diff --git a/image/3_class/svhn_naive_rf_train_time.txt b/vision/3_class/svhn_naive_rf_train_time.txt similarity index 100% rename from image/3_class/svhn_naive_rf_train_time.txt rename to vision/3_class/svhn_naive_rf_train_time.txt diff --git a/image/3_class/svhn_resnet18.txt b/vision/3_class/svhn_resnet18.txt similarity index 100% rename from image/3_class/svhn_resnet18.txt rename to vision/3_class/svhn_resnet18.txt diff --git a/image/3_class/svhn_resnet18_test_time.txt b/vision/3_class/svhn_resnet18_test_time.txt similarity index 100% rename from image/3_class/svhn_resnet18_test_time.txt rename to vision/3_class/svhn_resnet18_test_time.txt diff --git a/image/3_class/svhn_resnet18_train_time.txt b/vision/3_class/svhn_resnet18_train_time.txt similarity index 100% rename from image/3_class/svhn_resnet18_train_time.txt rename to vision/3_class/svhn_resnet18_train_time.txt diff --git a/image/3_class/svhn_svm.txt b/vision/3_class/svhn_svm.txt similarity index 100% rename from image/3_class/svhn_svm.txt rename to vision/3_class/svhn_svm.txt diff --git a/image/3_class/svhn_svm_test_time.txt b/vision/3_class/svhn_svm_test_time.txt similarity index 100% rename from image/3_class/svhn_svm_test_time.txt rename to vision/3_class/svhn_svm_test_time.txt diff --git a/image/3_class/svhn_svm_train_time.txt b/vision/3_class/svhn_svm_train_time.txt similarity index 100% rename from image/3_class/svhn_svm_train_time.txt rename to vision/3_class/svhn_svm_train_time.txt diff --git a/image/3_class/svm.txt b/vision/3_class/svm.txt similarity index 100% rename from image/3_class/svm.txt rename to vision/3_class/svm.txt diff --git a/image/3_class/svm_test_time.txt b/vision/3_class/svm_test_time.txt similarity index 100% rename from image/3_class/svm_test_time.txt rename to vision/3_class/svm_test_time.txt diff --git a/image/3_class/svm_test_time_lc.txt b/vision/3_class/svm_test_time_lc.txt similarity index 100% rename from image/3_class/svm_test_time_lc.txt rename to vision/3_class/svm_test_time_lc.txt diff --git a/image/3_class/svm_train_time.txt b/vision/3_class/svm_train_time.txt similarity index 100% rename from image/3_class/svm_train_time.txt rename to vision/3_class/svm_train_time.txt diff --git a/image/3_class/svm_train_time_lc.txt b/vision/3_class/svm_train_time_lc.txt similarity index 100% rename from image/3_class/svm_train_time_lc.txt rename to vision/3_class/svm_train_time_lc.txt diff --git a/image/8_class/cnn32.txt b/vision/8_class/cnn32.txt similarity index 100% rename from image/8_class/cnn32.txt rename to vision/8_class/cnn32.txt diff --git a/image/8_class/cnn32_2l.txt b/vision/8_class/cnn32_2l.txt similarity index 100% rename from image/8_class/cnn32_2l.txt rename to vision/8_class/cnn32_2l.txt diff --git a/image/8_class/cnn32_2l_sc.txt b/vision/8_class/cnn32_2l_sc.txt similarity index 100% rename from image/8_class/cnn32_2l_sc.txt rename to vision/8_class/cnn32_2l_sc.txt diff --git a/image/8_class/cnn32_2l_st.txt b/vision/8_class/cnn32_2l_st.txt similarity index 100% rename from image/8_class/cnn32_2l_st.txt rename to vision/8_class/cnn32_2l_st.txt diff --git a/image/8_class/cnn32_2l_test_time.txt b/vision/8_class/cnn32_2l_test_time.txt similarity index 100% rename from image/8_class/cnn32_2l_test_time.txt rename to vision/8_class/cnn32_2l_test_time.txt diff --git a/image/8_class/cnn32_2l_test_time_sc.txt b/vision/8_class/cnn32_2l_test_time_sc.txt similarity index 100% rename from image/8_class/cnn32_2l_test_time_sc.txt rename to vision/8_class/cnn32_2l_test_time_sc.txt diff --git a/image/8_class/cnn32_2l_test_time_st.txt b/vision/8_class/cnn32_2l_test_time_st.txt similarity index 100% rename from image/8_class/cnn32_2l_test_time_st.txt rename to vision/8_class/cnn32_2l_test_time_st.txt diff --git a/image/8_class/cnn32_2l_train_time.txt b/vision/8_class/cnn32_2l_train_time.txt similarity index 100% rename from image/8_class/cnn32_2l_train_time.txt rename to vision/8_class/cnn32_2l_train_time.txt diff --git a/image/8_class/cnn32_2l_train_time_sc.txt b/vision/8_class/cnn32_2l_train_time_sc.txt similarity index 100% rename from image/8_class/cnn32_2l_train_time_sc.txt rename to vision/8_class/cnn32_2l_train_time_sc.txt diff --git a/image/8_class/cnn32_2l_train_time_st.txt b/vision/8_class/cnn32_2l_train_time_st.txt similarity index 100% rename from image/8_class/cnn32_2l_train_time_st.txt rename to vision/8_class/cnn32_2l_train_time_st.txt diff --git a/image/8_class/cnn32_5l.txt b/vision/8_class/cnn32_5l.txt similarity index 100% rename from image/8_class/cnn32_5l.txt rename to vision/8_class/cnn32_5l.txt diff --git a/image/8_class/cnn32_5l_sc.txt b/vision/8_class/cnn32_5l_sc.txt similarity index 100% rename from image/8_class/cnn32_5l_sc.txt rename to vision/8_class/cnn32_5l_sc.txt diff --git a/image/8_class/cnn32_5l_st.txt b/vision/8_class/cnn32_5l_st.txt similarity index 100% rename from image/8_class/cnn32_5l_st.txt rename to vision/8_class/cnn32_5l_st.txt diff --git a/image/8_class/cnn32_5l_test_time.txt b/vision/8_class/cnn32_5l_test_time.txt similarity index 100% rename from image/8_class/cnn32_5l_test_time.txt rename to vision/8_class/cnn32_5l_test_time.txt diff --git a/image/8_class/cnn32_5l_test_time_sc.txt b/vision/8_class/cnn32_5l_test_time_sc.txt similarity index 100% rename from image/8_class/cnn32_5l_test_time_sc.txt rename to vision/8_class/cnn32_5l_test_time_sc.txt diff --git a/image/8_class/cnn32_5l_test_time_st.txt b/vision/8_class/cnn32_5l_test_time_st.txt similarity index 100% rename from image/8_class/cnn32_5l_test_time_st.txt rename to vision/8_class/cnn32_5l_test_time_st.txt diff --git a/image/8_class/cnn32_5l_train_time.txt b/vision/8_class/cnn32_5l_train_time.txt similarity index 100% rename from image/8_class/cnn32_5l_train_time.txt rename to vision/8_class/cnn32_5l_train_time.txt diff --git a/image/8_class/cnn32_5l_train_time_sc.txt b/vision/8_class/cnn32_5l_train_time_sc.txt similarity index 100% rename from image/8_class/cnn32_5l_train_time_sc.txt rename to vision/8_class/cnn32_5l_train_time_sc.txt diff --git a/image/8_class/cnn32_5l_train_time_st.txt b/vision/8_class/cnn32_5l_train_time_st.txt similarity index 100% rename from image/8_class/cnn32_5l_train_time_st.txt rename to vision/8_class/cnn32_5l_train_time_st.txt diff --git a/image/8_class/cnn32_sc.txt b/vision/8_class/cnn32_sc.txt similarity index 100% rename from image/8_class/cnn32_sc.txt rename to vision/8_class/cnn32_sc.txt diff --git a/image/8_class/cnn32_st.txt b/vision/8_class/cnn32_st.txt similarity index 100% rename from image/8_class/cnn32_st.txt rename to vision/8_class/cnn32_st.txt diff --git a/image/8_class/cnn32_test_time.txt b/vision/8_class/cnn32_test_time.txt similarity index 100% rename from image/8_class/cnn32_test_time.txt rename to vision/8_class/cnn32_test_time.txt diff --git a/image/8_class/cnn32_test_time_sc.txt b/vision/8_class/cnn32_test_time_sc.txt similarity index 100% rename from image/8_class/cnn32_test_time_sc.txt rename to vision/8_class/cnn32_test_time_sc.txt diff --git a/image/8_class/cnn32_test_time_st.txt b/vision/8_class/cnn32_test_time_st.txt similarity index 100% rename from image/8_class/cnn32_test_time_st.txt rename to vision/8_class/cnn32_test_time_st.txt diff --git a/image/8_class/cnn32_train_time.txt b/vision/8_class/cnn32_train_time.txt similarity index 100% rename from image/8_class/cnn32_train_time.txt rename to vision/8_class/cnn32_train_time.txt diff --git a/image/8_class/cnn32_train_time_sc.txt b/vision/8_class/cnn32_train_time_sc.txt similarity index 100% rename from image/8_class/cnn32_train_time_sc.txt rename to vision/8_class/cnn32_train_time_sc.txt diff --git a/image/8_class/cnn32_train_time_st.txt b/vision/8_class/cnn32_train_time_st.txt similarity index 100% rename from image/8_class/cnn32_train_time_st.txt rename to vision/8_class/cnn32_train_time_st.txt diff --git a/image/8_class/naive_rf.txt b/vision/8_class/naive_rf.txt similarity index 100% rename from image/8_class/naive_rf.txt rename to vision/8_class/naive_rf.txt diff --git a/image/8_class/naive_rf_test_time.txt b/vision/8_class/naive_rf_test_time.txt similarity index 100% rename from image/8_class/naive_rf_test_time.txt rename to vision/8_class/naive_rf_test_time.txt diff --git a/image/8_class/naive_rf_test_time_lc.txt b/vision/8_class/naive_rf_test_time_lc.txt similarity index 100% rename from image/8_class/naive_rf_test_time_lc.txt rename to vision/8_class/naive_rf_test_time_lc.txt diff --git a/image/8_class/naive_rf_train_time.txt b/vision/8_class/naive_rf_train_time.txt similarity index 100% rename from image/8_class/naive_rf_train_time.txt rename to vision/8_class/naive_rf_train_time.txt diff --git a/image/8_class/naive_rf_train_time_lc.txt b/vision/8_class/naive_rf_train_time_lc.txt similarity index 100% rename from image/8_class/naive_rf_train_time_lc.txt rename to vision/8_class/naive_rf_train_time_lc.txt diff --git a/image/8_class/resnet18.txt b/vision/8_class/resnet18.txt similarity index 100% rename from image/8_class/resnet18.txt rename to vision/8_class/resnet18.txt diff --git a/image/8_class/resnet18_sc.txt b/vision/8_class/resnet18_sc.txt similarity index 100% rename from image/8_class/resnet18_sc.txt rename to vision/8_class/resnet18_sc.txt diff --git a/image/8_class/resnet18_st.txt b/vision/8_class/resnet18_st.txt similarity index 100% rename from image/8_class/resnet18_st.txt rename to vision/8_class/resnet18_st.txt diff --git a/image/8_class/resnet18_test_time.txt b/vision/8_class/resnet18_test_time.txt similarity index 100% rename from image/8_class/resnet18_test_time.txt rename to vision/8_class/resnet18_test_time.txt diff --git a/image/8_class/resnet18_test_time_sc.txt b/vision/8_class/resnet18_test_time_sc.txt similarity index 100% rename from image/8_class/resnet18_test_time_sc.txt rename to vision/8_class/resnet18_test_time_sc.txt diff --git a/image/8_class/resnet18_test_time_st.txt b/vision/8_class/resnet18_test_time_st.txt similarity index 100% rename from image/8_class/resnet18_test_time_st.txt rename to vision/8_class/resnet18_test_time_st.txt diff --git a/image/8_class/resnet18_train_time.txt b/vision/8_class/resnet18_train_time.txt similarity index 100% rename from image/8_class/resnet18_train_time.txt rename to vision/8_class/resnet18_train_time.txt diff --git a/image/8_class/resnet18_train_time_sc.txt b/vision/8_class/resnet18_train_time_sc.txt similarity index 100% rename from image/8_class/resnet18_train_time_sc.txt rename to vision/8_class/resnet18_train_time_sc.txt diff --git a/image/8_class/resnet18_train_time_st.txt b/vision/8_class/resnet18_train_time_st.txt similarity index 100% rename from image/8_class/resnet18_train_time_st.txt rename to vision/8_class/resnet18_train_time_st.txt diff --git a/image/8_class/svhn_cnn32.txt b/vision/8_class/svhn_cnn32.txt similarity index 100% rename from image/8_class/svhn_cnn32.txt rename to vision/8_class/svhn_cnn32.txt diff --git a/image/8_class/svhn_cnn32_2l.txt b/vision/8_class/svhn_cnn32_2l.txt similarity index 100% rename from image/8_class/svhn_cnn32_2l.txt rename to vision/8_class/svhn_cnn32_2l.txt diff --git a/image/8_class/svhn_cnn32_2l_test_time.txt b/vision/8_class/svhn_cnn32_2l_test_time.txt similarity index 100% rename from image/8_class/svhn_cnn32_2l_test_time.txt rename to vision/8_class/svhn_cnn32_2l_test_time.txt diff --git a/image/8_class/svhn_cnn32_2l_train_time.txt b/vision/8_class/svhn_cnn32_2l_train_time.txt similarity index 100% rename from image/8_class/svhn_cnn32_2l_train_time.txt rename to vision/8_class/svhn_cnn32_2l_train_time.txt diff --git a/image/8_class/svhn_cnn32_5l.txt b/vision/8_class/svhn_cnn32_5l.txt similarity index 100% rename from image/8_class/svhn_cnn32_5l.txt rename to vision/8_class/svhn_cnn32_5l.txt diff --git a/image/8_class/svhn_cnn32_5l_test_time.txt b/vision/8_class/svhn_cnn32_5l_test_time.txt similarity index 100% rename from image/8_class/svhn_cnn32_5l_test_time.txt rename to vision/8_class/svhn_cnn32_5l_test_time.txt diff --git a/image/8_class/svhn_cnn32_5l_train_time.txt b/vision/8_class/svhn_cnn32_5l_train_time.txt similarity index 100% rename from image/8_class/svhn_cnn32_5l_train_time.txt rename to vision/8_class/svhn_cnn32_5l_train_time.txt diff --git a/image/8_class/svhn_cnn32_test_time.txt b/vision/8_class/svhn_cnn32_test_time.txt similarity index 100% rename from image/8_class/svhn_cnn32_test_time.txt rename to vision/8_class/svhn_cnn32_test_time.txt diff --git a/image/8_class/svhn_cnn32_train_time.txt b/vision/8_class/svhn_cnn32_train_time.txt similarity index 100% rename from image/8_class/svhn_cnn32_train_time.txt rename to vision/8_class/svhn_cnn32_train_time.txt diff --git a/image/8_class/svhn_naive_rf.txt b/vision/8_class/svhn_naive_rf.txt similarity index 100% rename from image/8_class/svhn_naive_rf.txt rename to vision/8_class/svhn_naive_rf.txt diff --git a/image/8_class/svhn_naive_rf_test_time.txt b/vision/8_class/svhn_naive_rf_test_time.txt similarity index 100% rename from image/8_class/svhn_naive_rf_test_time.txt rename to vision/8_class/svhn_naive_rf_test_time.txt diff --git a/image/8_class/svhn_naive_rf_train_time.txt b/vision/8_class/svhn_naive_rf_train_time.txt similarity index 100% rename from image/8_class/svhn_naive_rf_train_time.txt rename to vision/8_class/svhn_naive_rf_train_time.txt diff --git a/image/8_class/svhn_resnet18.txt b/vision/8_class/svhn_resnet18.txt similarity index 100% rename from image/8_class/svhn_resnet18.txt rename to vision/8_class/svhn_resnet18.txt diff --git a/image/8_class/svhn_resnet18_test_time.txt b/vision/8_class/svhn_resnet18_test_time.txt similarity index 100% rename from image/8_class/svhn_resnet18_test_time.txt rename to vision/8_class/svhn_resnet18_test_time.txt diff --git a/image/8_class/svhn_resnet18_train_time.txt b/vision/8_class/svhn_resnet18_train_time.txt similarity index 100% rename from image/8_class/svhn_resnet18_train_time.txt rename to vision/8_class/svhn_resnet18_train_time.txt diff --git a/image/8_class/svhn_svm.txt b/vision/8_class/svhn_svm.txt similarity index 100% rename from image/8_class/svhn_svm.txt rename to vision/8_class/svhn_svm.txt diff --git a/image/8_class/svhn_svm_test_time.txt b/vision/8_class/svhn_svm_test_time.txt similarity index 100% rename from image/8_class/svhn_svm_test_time.txt rename to vision/8_class/svhn_svm_test_time.txt diff --git a/image/8_class/svhn_svm_train_time.txt b/vision/8_class/svhn_svm_train_time.txt similarity index 100% rename from image/8_class/svhn_svm_train_time.txt rename to vision/8_class/svhn_svm_train_time.txt diff --git a/image/8_class/svm.txt b/vision/8_class/svm.txt similarity index 100% rename from image/8_class/svm.txt rename to vision/8_class/svm.txt diff --git a/image/8_class/svm_test_time.txt b/vision/8_class/svm_test_time.txt similarity index 100% rename from image/8_class/svm_test_time.txt rename to vision/8_class/svm_test_time.txt diff --git a/image/8_class/svm_test_time_lc.txt b/vision/8_class/svm_test_time_lc.txt similarity index 100% rename from image/8_class/svm_test_time_lc.txt rename to vision/8_class/svm_test_time_lc.txt diff --git a/image/8_class/svm_train_time.txt b/vision/8_class/svm_train_time.txt similarity index 100% rename from image/8_class/svm_train_time.txt rename to vision/8_class/svm_train_time.txt diff --git a/image/8_class/svm_train_time_lc.txt b/vision/8_class/svm_train_time_lc.txt similarity index 100% rename from image/8_class/svm_train_time_lc.txt rename to vision/8_class/svm_train_time_lc.txt diff --git a/image/90_class/cnn32.txt b/vision/90_class/cnn32.txt similarity index 100% rename from image/90_class/cnn32.txt rename to vision/90_class/cnn32.txt diff --git a/image/90_class/cnn32_2l.txt b/vision/90_class/cnn32_2l.txt similarity index 100% rename from image/90_class/cnn32_2l.txt rename to vision/90_class/cnn32_2l.txt diff --git a/image/90_class/cnn32_2l_sc.txt b/vision/90_class/cnn32_2l_sc.txt similarity index 100% rename from image/90_class/cnn32_2l_sc.txt rename to vision/90_class/cnn32_2l_sc.txt diff --git a/image/90_class/cnn32_2l_st.txt b/vision/90_class/cnn32_2l_st.txt similarity index 100% rename from image/90_class/cnn32_2l_st.txt rename to vision/90_class/cnn32_2l_st.txt diff --git a/image/90_class/cnn32_2l_test_time.txt b/vision/90_class/cnn32_2l_test_time.txt similarity index 100% rename from image/90_class/cnn32_2l_test_time.txt rename to vision/90_class/cnn32_2l_test_time.txt diff --git a/image/90_class/cnn32_2l_test_time_sc.txt b/vision/90_class/cnn32_2l_test_time_sc.txt similarity index 100% rename from image/90_class/cnn32_2l_test_time_sc.txt rename to vision/90_class/cnn32_2l_test_time_sc.txt diff --git a/image/90_class/cnn32_2l_test_time_st.txt b/vision/90_class/cnn32_2l_test_time_st.txt similarity index 100% rename from image/90_class/cnn32_2l_test_time_st.txt rename to vision/90_class/cnn32_2l_test_time_st.txt diff --git a/image/90_class/cnn32_2l_train_time.txt b/vision/90_class/cnn32_2l_train_time.txt similarity index 100% rename from image/90_class/cnn32_2l_train_time.txt rename to vision/90_class/cnn32_2l_train_time.txt diff --git a/image/90_class/cnn32_2l_train_time_sc.txt b/vision/90_class/cnn32_2l_train_time_sc.txt similarity index 100% rename from image/90_class/cnn32_2l_train_time_sc.txt rename to vision/90_class/cnn32_2l_train_time_sc.txt diff --git a/image/90_class/cnn32_2l_train_time_st.txt b/vision/90_class/cnn32_2l_train_time_st.txt similarity index 100% rename from image/90_class/cnn32_2l_train_time_st.txt rename to vision/90_class/cnn32_2l_train_time_st.txt diff --git a/image/90_class/cnn32_5l.txt b/vision/90_class/cnn32_5l.txt similarity index 100% rename from image/90_class/cnn32_5l.txt rename to vision/90_class/cnn32_5l.txt diff --git a/image/90_class/cnn32_5l_sc.txt b/vision/90_class/cnn32_5l_sc.txt similarity index 100% rename from image/90_class/cnn32_5l_sc.txt rename to vision/90_class/cnn32_5l_sc.txt diff --git a/image/90_class/cnn32_5l_st.txt b/vision/90_class/cnn32_5l_st.txt similarity index 100% rename from image/90_class/cnn32_5l_st.txt rename to vision/90_class/cnn32_5l_st.txt diff --git a/image/90_class/cnn32_5l_test_time.txt b/vision/90_class/cnn32_5l_test_time.txt similarity index 100% rename from image/90_class/cnn32_5l_test_time.txt rename to vision/90_class/cnn32_5l_test_time.txt diff --git a/image/90_class/cnn32_5l_test_time_sc.txt b/vision/90_class/cnn32_5l_test_time_sc.txt similarity index 100% rename from image/90_class/cnn32_5l_test_time_sc.txt rename to vision/90_class/cnn32_5l_test_time_sc.txt diff --git a/image/90_class/cnn32_5l_test_time_st.txt b/vision/90_class/cnn32_5l_test_time_st.txt similarity index 100% rename from image/90_class/cnn32_5l_test_time_st.txt rename to vision/90_class/cnn32_5l_test_time_st.txt diff --git a/image/90_class/cnn32_5l_train_time.txt b/vision/90_class/cnn32_5l_train_time.txt similarity index 100% rename from image/90_class/cnn32_5l_train_time.txt rename to vision/90_class/cnn32_5l_train_time.txt diff --git a/image/90_class/cnn32_5l_train_time_sc.txt b/vision/90_class/cnn32_5l_train_time_sc.txt similarity index 100% rename from image/90_class/cnn32_5l_train_time_sc.txt rename to vision/90_class/cnn32_5l_train_time_sc.txt diff --git a/image/90_class/cnn32_5l_train_time_st.txt b/vision/90_class/cnn32_5l_train_time_st.txt similarity index 100% rename from image/90_class/cnn32_5l_train_time_st.txt rename to vision/90_class/cnn32_5l_train_time_st.txt diff --git a/image/90_class/cnn32_sc.txt b/vision/90_class/cnn32_sc.txt similarity index 100% rename from image/90_class/cnn32_sc.txt rename to vision/90_class/cnn32_sc.txt diff --git a/image/90_class/cnn32_st.txt b/vision/90_class/cnn32_st.txt similarity index 100% rename from image/90_class/cnn32_st.txt rename to vision/90_class/cnn32_st.txt diff --git a/image/90_class/cnn32_test_time.txt b/vision/90_class/cnn32_test_time.txt similarity index 100% rename from image/90_class/cnn32_test_time.txt rename to vision/90_class/cnn32_test_time.txt diff --git a/image/90_class/cnn32_test_time_sc.txt b/vision/90_class/cnn32_test_time_sc.txt similarity index 100% rename from image/90_class/cnn32_test_time_sc.txt rename to vision/90_class/cnn32_test_time_sc.txt diff --git a/image/90_class/cnn32_test_time_st.txt b/vision/90_class/cnn32_test_time_st.txt similarity index 100% rename from image/90_class/cnn32_test_time_st.txt rename to vision/90_class/cnn32_test_time_st.txt diff --git a/image/90_class/cnn32_train_time.txt b/vision/90_class/cnn32_train_time.txt similarity index 100% rename from image/90_class/cnn32_train_time.txt rename to vision/90_class/cnn32_train_time.txt diff --git a/image/90_class/cnn32_train_time_sc.txt b/vision/90_class/cnn32_train_time_sc.txt similarity index 100% rename from image/90_class/cnn32_train_time_sc.txt rename to vision/90_class/cnn32_train_time_sc.txt diff --git a/image/90_class/cnn32_train_time_st.txt b/vision/90_class/cnn32_train_time_st.txt similarity index 100% rename from image/90_class/cnn32_train_time_st.txt rename to vision/90_class/cnn32_train_time_st.txt diff --git a/image/90_class/naive_rf.txt b/vision/90_class/naive_rf.txt similarity index 100% rename from image/90_class/naive_rf.txt rename to vision/90_class/naive_rf.txt diff --git a/image/90_class/naive_rf_test_time.txt b/vision/90_class/naive_rf_test_time.txt similarity index 100% rename from image/90_class/naive_rf_test_time.txt rename to vision/90_class/naive_rf_test_time.txt diff --git a/image/90_class/naive_rf_test_time_lc.txt b/vision/90_class/naive_rf_test_time_lc.txt similarity index 100% rename from image/90_class/naive_rf_test_time_lc.txt rename to vision/90_class/naive_rf_test_time_lc.txt diff --git a/image/90_class/naive_rf_train_time.txt b/vision/90_class/naive_rf_train_time.txt similarity index 100% rename from image/90_class/naive_rf_train_time.txt rename to vision/90_class/naive_rf_train_time.txt diff --git a/image/90_class/naive_rf_train_time_lc.txt b/vision/90_class/naive_rf_train_time_lc.txt similarity index 100% rename from image/90_class/naive_rf_train_time_lc.txt rename to vision/90_class/naive_rf_train_time_lc.txt diff --git a/image/90_class/resnet18.txt b/vision/90_class/resnet18.txt similarity index 100% rename from image/90_class/resnet18.txt rename to vision/90_class/resnet18.txt diff --git a/image/90_class/resnet18_sc.txt b/vision/90_class/resnet18_sc.txt similarity index 100% rename from image/90_class/resnet18_sc.txt rename to vision/90_class/resnet18_sc.txt diff --git a/image/90_class/resnet18_st.txt b/vision/90_class/resnet18_st.txt similarity index 100% rename from image/90_class/resnet18_st.txt rename to vision/90_class/resnet18_st.txt diff --git a/image/90_class/resnet18_test_time.txt b/vision/90_class/resnet18_test_time.txt similarity index 100% rename from image/90_class/resnet18_test_time.txt rename to vision/90_class/resnet18_test_time.txt diff --git a/image/90_class/resnet18_test_time_sc.txt b/vision/90_class/resnet18_test_time_sc.txt similarity index 100% rename from image/90_class/resnet18_test_time_sc.txt rename to vision/90_class/resnet18_test_time_sc.txt diff --git a/image/90_class/resnet18_test_time_st.txt b/vision/90_class/resnet18_test_time_st.txt similarity index 100% rename from image/90_class/resnet18_test_time_st.txt rename to vision/90_class/resnet18_test_time_st.txt diff --git a/image/90_class/resnet18_train_time.txt b/vision/90_class/resnet18_train_time.txt similarity index 100% rename from image/90_class/resnet18_train_time.txt rename to vision/90_class/resnet18_train_time.txt diff --git a/image/90_class/resnet18_train_time_sc.txt b/vision/90_class/resnet18_train_time_sc.txt similarity index 100% rename from image/90_class/resnet18_train_time_sc.txt rename to vision/90_class/resnet18_train_time_sc.txt diff --git a/image/90_class/resnet18_train_time_st.txt b/vision/90_class/resnet18_train_time_st.txt similarity index 100% rename from image/90_class/resnet18_train_time_st.txt rename to vision/90_class/resnet18_train_time_st.txt diff --git a/image/90_class/svm.txt b/vision/90_class/svm.txt similarity index 100% rename from image/90_class/svm.txt rename to vision/90_class/svm.txt diff --git a/image/90_class/svm_test_time.txt b/vision/90_class/svm_test_time.txt similarity index 100% rename from image/90_class/svm_test_time.txt rename to vision/90_class/svm_test_time.txt diff --git a/image/90_class/svm_test_time_lc.txt b/vision/90_class/svm_test_time_lc.txt similarity index 100% rename from image/90_class/svm_test_time_lc.txt rename to vision/90_class/svm_test_time_lc.txt diff --git a/image/90_class/svm_train_time.txt b/vision/90_class/svm_train_time.txt similarity index 100% rename from image/90_class/svm_train_time.txt rename to vision/90_class/svm_train_time.txt diff --git a/image/90_class/svm_train_time_lc.txt b/vision/90_class/svm_train_time_lc.txt similarity index 100% rename from image/90_class/svm_train_time_lc.txt rename to vision/90_class/svm_train_time_lc.txt diff --git a/image/cifar_10.py b/vision/cifar_10.py similarity index 100% rename from image/cifar_10.py rename to vision/cifar_10.py diff --git a/image/cifar_100.py b/vision/cifar_100.py similarity index 100% rename from image/cifar_100.py rename to vision/cifar_100.py diff --git a/image/cifar_100_stc.py b/vision/cifar_100_stc.py similarity index 100% rename from image/cifar_100_stc.py rename to vision/cifar_100_stc.py diff --git a/image/cifar_10_stc.py b/vision/cifar_10_stc.py similarity index 100% rename from image/cifar_10_stc.py rename to vision/cifar_10_stc.py diff --git a/image/cifar_figures.ipynb b/vision/cifar_figures.ipynb similarity index 100% rename from image/cifar_figures.ipynb rename to vision/cifar_figures.ipynb diff --git a/image/svhn.py b/vision/svhn.py similarity index 100% rename from image/svhn.py rename to vision/svhn.py diff --git a/image/svhn_figure.ipynb b/vision/svhn_figure.ipynb similarity index 100% rename from image/svhn_figure.ipynb rename to vision/svhn_figure.ipynb diff --git a/image/svhn_toolbox.py b/vision/svhn_toolbox.py similarity index 100% rename from image/svhn_toolbox.py rename to vision/svhn_toolbox.py diff --git a/image/toolbox.py b/vision/toolbox.py similarity index 100% rename from image/toolbox.py rename to vision/toolbox.py From 1c67c1d0bef692d2f33559123f4b4c01e73ea389 Mon Sep 17 00:00:00 2001 From: Haoyin Xu Date: Thu, 23 Sep 2021 11:49:23 -0400 Subject: [PATCH 3/5] EHN switch from accuracy to cohen's kappa --- audition/audio_toolbox.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/audition/audio_toolbox.py b/audition/audio_toolbox.py index e374d629..ae658cfb 100644 --- a/audition/audio_toolbox.py +++ b/audition/audio_toolbox.py @@ -4,7 +4,7 @@ Madi Kusmanov """ import numpy as np -from sklearn.metrics import accuracy_score +from sklearn.metrics import cohen_kappa_score import time import torch import os @@ -206,7 +206,7 @@ def run_rf_image_set( end_time = time.perf_counter() test_time = end_time - start_time - return accuracy_score(test_labels, test_preds), train_time, test_time + return cohen_kappa_score(test_labels, test_preds), train_time, test_time def run_dn_image_es( @@ -236,7 +236,7 @@ def run_dn_image_es( flag = 0 start_time = time.perf_counter() for epoch in range(epochs): # loop over the dataset multiple times - + model.train() for i in range(0, len(train_data), batch): # get the inputs inputs = train_data[i : i + batch].to(dev) @@ -251,6 +251,7 @@ def run_dn_image_es( optimizer.step() # test generalization error for early stopping + model.eval() cur_loss = 0 with torch.no_grad(): for i in range(0, len(valid_data), batch): @@ -273,22 +274,25 @@ def run_dn_image_es( break end_time = time.perf_counter() train_time = end_time - start_time + # test the model - correct = torch.tensor(0).to(dev) - total = torch.tensor(0).to(dev) + model.eval() start_time = time.perf_counter() + test_preds = [] + test_labels = [] with torch.no_grad(): for i in range(0, len(test_data), batch): inputs = test_data[i : i + batch].to(dev) labels = test_labels[i : i + batch].to(dev) + test_labels = np.concatenate((test_labels, labels.tolist())) + outputs = model(inputs) _, predicted = torch.max(outputs.data, 1) - total += labels.size(0) - correct += (predicted == labels.view(-1)).sum().item() + test_preds = np.concatenate((test_preds, predicted.tolist())) + end_time = time.perf_counter() test_time = end_time - start_time - accuracy = float(correct) / float(total) - return accuracy, train_time, test_time + return cohen_kappa_score(test_preds, test_labels), train_time, test_time def prepare_data( From ae808167f87fcec57dfdbb054656f0df28df4c3e Mon Sep 17 00:00:00 2001 From: Haoyin Xu Date: Thu, 23 Sep 2021 11:52:39 -0400 Subject: [PATCH 4/5] MAINT rename benchmark folders --- {audition => benchmark_audition}/3_class/cnn32.txt | 0 {audition => benchmark_audition}/3_class/cnn32_2l.txt | 0 {audition => benchmark_audition}/3_class/cnn32_2l_test_time.txt | 0 {audition => benchmark_audition}/3_class/cnn32_2l_train_time.txt | 0 {audition => benchmark_audition}/3_class/cnn32_5l.txt | 0 {audition => benchmark_audition}/3_class/cnn32_5l_test_time.txt | 0 {audition => benchmark_audition}/3_class/cnn32_5l_train_time.txt | 0 {audition => benchmark_audition}/3_class/cnn32_test_time.txt | 0 {audition => benchmark_audition}/3_class/cnn32_train_time.txt | 0 {audition => benchmark_audition}/3_class/naive_rf.txt | 0 {audition => benchmark_audition}/3_class/naive_rf_test_time.txt | 0 {audition => benchmark_audition}/3_class/naive_rf_train_time.txt | 0 {audition => benchmark_audition}/3_class/resnet18.txt | 0 {audition => benchmark_audition}/3_class/resnet18_test_time.txt | 0 {audition => benchmark_audition}/3_class/resnet18_train_time.txt | 0 {audition => benchmark_audition}/3_class/svm.txt | 0 {audition => benchmark_audition}/3_class/svm_test_time.txt | 0 {audition => benchmark_audition}/3_class/svm_train_time.txt | 0 {audition => benchmark_audition}/3_class_mel/cnn32.txt | 0 {audition => benchmark_audition}/3_class_mel/cnn32_2l.txt | 0 .../3_class_mel/cnn32_2l_test_time.txt | 0 .../3_class_mel/cnn32_2l_train_time.txt | 0 {audition => benchmark_audition}/3_class_mel/cnn32_5l.txt | 0 .../3_class_mel/cnn32_5l_test_time.txt | 0 .../3_class_mel/cnn32_5l_train_time.txt | 0 {audition => benchmark_audition}/3_class_mel/cnn32_test_time.txt | 0 {audition => benchmark_audition}/3_class_mel/cnn32_train_time.txt | 0 {audition => benchmark_audition}/3_class_mel/naive_rf.txt | 0 .../3_class_mel/naive_rf_test_time.txt | 0 .../3_class_mel/naive_rf_train_time.txt | 0 {audition => benchmark_audition}/3_class_mel/resnet18.txt | 0 .../3_class_mel/resnet18_test_time.txt | 0 .../3_class_mel/resnet18_train_time.txt | 0 {audition => benchmark_audition}/3_class_mel/svm.txt | 0 {audition => benchmark_audition}/3_class_mel/svm_test_time.txt | 0 {audition => benchmark_audition}/3_class_mel/svm_train_time.txt | 0 {audition => benchmark_audition}/3_class_mfcc/cnn32.txt | 0 {audition => benchmark_audition}/3_class_mfcc/cnn32_2l.txt | 0 .../3_class_mfcc/cnn32_2l_test_time.txt | 0 .../3_class_mfcc/cnn32_2l_train_time.txt | 0 {audition => benchmark_audition}/3_class_mfcc/cnn32_5l.txt | 0 .../3_class_mfcc/cnn32_5l_test_time.txt | 0 .../3_class_mfcc/cnn32_5l_train_time.txt | 0 {audition => benchmark_audition}/3_class_mfcc/cnn32_test_time.txt | 0 .../3_class_mfcc/cnn32_train_time.txt | 0 {audition => benchmark_audition}/3_class_mfcc/naive_rf.txt | 0 .../3_class_mfcc/naive_rf_test_time.txt | 0 .../3_class_mfcc/naive_rf_train_time.txt | 0 {audition => benchmark_audition}/3_class_mfcc/resnet18.txt | 0 .../3_class_mfcc/resnet18_test_time.txt | 0 .../3_class_mfcc/resnet18_train_time.txt | 0 {audition => benchmark_audition}/3_class_mfcc/svm.txt | 0 {audition => benchmark_audition}/3_class_mfcc/svm_test_time.txt | 0 {audition => benchmark_audition}/3_class_mfcc/svm_train_time.txt | 0 {audition => benchmark_audition}/8_class/cnn32.txt | 0 {audition => benchmark_audition}/8_class/cnn32_2l.txt | 0 {audition => benchmark_audition}/8_class/cnn32_2l_test_time.txt | 0 {audition => benchmark_audition}/8_class/cnn32_2l_train_time.txt | 0 {audition => benchmark_audition}/8_class/cnn32_5l.txt | 0 {audition => benchmark_audition}/8_class/cnn32_5l_test_time.txt | 0 {audition => benchmark_audition}/8_class/cnn32_5l_train_time.txt | 0 {audition => benchmark_audition}/8_class/cnn32_test_time.txt | 0 {audition => benchmark_audition}/8_class/cnn32_train_time.txt | 0 {audition => benchmark_audition}/8_class/naive_rf.txt | 0 {audition => benchmark_audition}/8_class/naive_rf_test_time.txt | 0 {audition => benchmark_audition}/8_class/naive_rf_train_time.txt | 0 {audition => benchmark_audition}/8_class/resnet18.txt | 0 {audition => benchmark_audition}/8_class/resnet18_test_time.txt | 0 {audition => benchmark_audition}/8_class/resnet18_train_time.txt | 0 {audition => benchmark_audition}/8_class/svm.txt | 0 {audition => benchmark_audition}/8_class/svm_test_time.txt | 0 {audition => benchmark_audition}/8_class/svm_train_time.txt | 0 {audition => benchmark_audition}/8_class_mel/cnn32.txt | 0 {audition => benchmark_audition}/8_class_mel/cnn32_2l.txt | 0 .../8_class_mel/cnn32_2l_test_time.txt | 0 .../8_class_mel/cnn32_2l_train_time.txt | 0 {audition => benchmark_audition}/8_class_mel/cnn32_5l.txt | 0 .../8_class_mel/cnn32_5l_test_time.txt | 0 .../8_class_mel/cnn32_5l_train_time.txt | 0 {audition => benchmark_audition}/8_class_mel/cnn32_test_time.txt | 0 {audition => benchmark_audition}/8_class_mel/cnn32_train_time.txt | 0 {audition => benchmark_audition}/8_class_mel/naive_rf.txt | 0 .../8_class_mel/naive_rf_test_time.txt | 0 .../8_class_mel/naive_rf_train_time.txt | 0 {audition => benchmark_audition}/8_class_mel/resnet18.txt | 0 .../8_class_mel/resnet18_test_time.txt | 0 .../8_class_mel/resnet18_train_time.txt | 0 {audition => benchmark_audition}/8_class_mel/svm.txt | 0 {audition => benchmark_audition}/8_class_mel/svm_test_time.txt | 0 {audition => benchmark_audition}/8_class_mel/svm_train_time.txt | 0 {audition => benchmark_audition}/8_class_mfcc/cnn32.txt | 0 {audition => benchmark_audition}/8_class_mfcc/cnn32_2l.txt | 0 .../8_class_mfcc/cnn32_2l_test_time.txt | 0 .../8_class_mfcc/cnn32_2l_train_time.txt | 0 {audition => benchmark_audition}/8_class_mfcc/cnn32_5l.txt | 0 .../8_class_mfcc/cnn32_5l_test_time.txt | 0 .../8_class_mfcc/cnn32_5l_train_time.txt | 0 {audition => benchmark_audition}/8_class_mfcc/cnn32_test_time.txt | 0 .../8_class_mfcc/cnn32_train_time.txt | 0 {audition => benchmark_audition}/8_class_mfcc/naive_rf.txt | 0 .../8_class_mfcc/naive_rf_test_time.txt | 0 .../8_class_mfcc/naive_rf_train_time.txt | 0 {audition => benchmark_audition}/8_class_mfcc/resnet18.txt | 0 .../8_class_mfcc/resnet18_test_time.txt | 0 .../8_class_mfcc/resnet18_train_time.txt | 0 {audition => benchmark_audition}/8_class_mfcc/svm.txt | 0 {audition => benchmark_audition}/8_class_mfcc/svm_test_time.txt | 0 {audition => benchmark_audition}/8_class_mfcc/svm_train_time.txt | 0 {audition => benchmark_audition}/audio_toolbox.py | 0 {audition => benchmark_audition}/fsdd.py | 0 {audition => benchmark_audition}/fsdd_figures.ipynb | 0 {tabular => benchmark_tabular}/cc18_figures.ipynb | 0 {tabular => benchmark_tabular}/cc18_hyperparameter.py | 0 {tabular => benchmark_tabular}/cc18_kappa_ece.py | 0 {tabular => benchmark_tabular}/cc18_times.py | 0 {tabular => benchmark_tabular}/metrics/cc18_all_parameters.txt | 0 {tabular => benchmark_tabular}/metrics/cc18_sample_sizes.txt | 0 {tabular => benchmark_tabular}/results/cc18_dn_ece.txt | 0 {tabular => benchmark_tabular}/results/cc18_dn_kappa.txt | 0 {tabular => benchmark_tabular}/results/cc18_dn_times_test.txt | 0 {tabular => benchmark_tabular}/results/cc18_dn_times_train.txt | 0 {tabular => benchmark_tabular}/results/cc18_rf_ece.txt | 0 {tabular => benchmark_tabular}/results/cc18_rf_kappa.txt | 0 {tabular => benchmark_tabular}/results/cc18_rf_times_test.txt | 0 {tabular => benchmark_tabular}/results/cc18_rf_times_train.txt | 0 {vision => benchmark_vision}/3_class/cnn32.txt | 0 {vision => benchmark_vision}/3_class/cnn32_2l.txt | 0 {vision => benchmark_vision}/3_class/cnn32_2l_sc.txt | 0 {vision => benchmark_vision}/3_class/cnn32_2l_st.txt | 0 {vision => benchmark_vision}/3_class/cnn32_2l_test_time.txt | 0 {vision => benchmark_vision}/3_class/cnn32_2l_test_time_sc.txt | 0 {vision => benchmark_vision}/3_class/cnn32_2l_test_time_st.txt | 0 {vision => benchmark_vision}/3_class/cnn32_2l_train_time.txt | 0 {vision => benchmark_vision}/3_class/cnn32_2l_train_time_sc.txt | 0 {vision => benchmark_vision}/3_class/cnn32_2l_train_time_st.txt | 0 {vision => benchmark_vision}/3_class/cnn32_5l.txt | 0 {vision => benchmark_vision}/3_class/cnn32_5l_sc.txt | 0 {vision => benchmark_vision}/3_class/cnn32_5l_st.txt | 0 {vision => benchmark_vision}/3_class/cnn32_5l_test_time.txt | 0 {vision => benchmark_vision}/3_class/cnn32_5l_test_time_sc.txt | 0 {vision => benchmark_vision}/3_class/cnn32_5l_test_time_st.txt | 0 {vision => benchmark_vision}/3_class/cnn32_5l_train_time.txt | 0 {vision => benchmark_vision}/3_class/cnn32_5l_train_time_sc.txt | 0 {vision => benchmark_vision}/3_class/cnn32_5l_train_time_st.txt | 0 {vision => benchmark_vision}/3_class/cnn32_sc.txt | 0 {vision => benchmark_vision}/3_class/cnn32_st.txt | 0 {vision => benchmark_vision}/3_class/cnn32_test_time.txt | 0 {vision => benchmark_vision}/3_class/cnn32_test_time_sc.txt | 0 {vision => benchmark_vision}/3_class/cnn32_test_time_st.txt | 0 {vision => benchmark_vision}/3_class/cnn32_train_time.txt | 0 {vision => benchmark_vision}/3_class/cnn32_train_time_sc.txt | 0 {vision => benchmark_vision}/3_class/cnn32_train_time_st.txt | 0 {vision => benchmark_vision}/3_class/naive_rf.txt | 0 {vision => benchmark_vision}/3_class/naive_rf_test_time.txt | 0 {vision => benchmark_vision}/3_class/naive_rf_test_time_lc.txt | 0 {vision => benchmark_vision}/3_class/naive_rf_train_time.txt | 0 {vision => benchmark_vision}/3_class/naive_rf_train_time_lc.txt | 0 {vision => benchmark_vision}/3_class/resnet18.txt | 0 {vision => benchmark_vision}/3_class/resnet18_sc.txt | 0 {vision => benchmark_vision}/3_class/resnet18_st.txt | 0 {vision => benchmark_vision}/3_class/resnet18_test_time.txt | 0 {vision => benchmark_vision}/3_class/resnet18_test_time_sc.txt | 0 {vision => benchmark_vision}/3_class/resnet18_test_time_st.txt | 0 {vision => benchmark_vision}/3_class/resnet18_train_time.txt | 0 {vision => benchmark_vision}/3_class/resnet18_train_time_sc.txt | 0 {vision => benchmark_vision}/3_class/resnet18_train_time_st.txt | 0 {vision => benchmark_vision}/3_class/svhn_cnn32.txt | 0 {vision => benchmark_vision}/3_class/svhn_cnn32_2l.txt | 0 {vision => benchmark_vision}/3_class/svhn_cnn32_2l_test_time.txt | 0 {vision => benchmark_vision}/3_class/svhn_cnn32_2l_train_time.txt | 0 {vision => benchmark_vision}/3_class/svhn_cnn32_5l.txt | 0 {vision => benchmark_vision}/3_class/svhn_cnn32_5l_test_time.txt | 0 {vision => benchmark_vision}/3_class/svhn_cnn32_5l_train_time.txt | 0 {vision => benchmark_vision}/3_class/svhn_cnn32_test_time.txt | 0 {vision => benchmark_vision}/3_class/svhn_cnn32_train_time.txt | 0 {vision => benchmark_vision}/3_class/svhn_naive_rf.txt | 0 {vision => benchmark_vision}/3_class/svhn_naive_rf_test_time.txt | 0 {vision => benchmark_vision}/3_class/svhn_naive_rf_train_time.txt | 0 {vision => benchmark_vision}/3_class/svhn_resnet18.txt | 0 {vision => benchmark_vision}/3_class/svhn_resnet18_test_time.txt | 0 {vision => benchmark_vision}/3_class/svhn_resnet18_train_time.txt | 0 {vision => benchmark_vision}/3_class/svhn_svm.txt | 0 {vision => benchmark_vision}/3_class/svhn_svm_test_time.txt | 0 {vision => benchmark_vision}/3_class/svhn_svm_train_time.txt | 0 {vision => benchmark_vision}/3_class/svm.txt | 0 {vision => benchmark_vision}/3_class/svm_test_time.txt | 0 {vision => benchmark_vision}/3_class/svm_test_time_lc.txt | 0 {vision => benchmark_vision}/3_class/svm_train_time.txt | 0 {vision => benchmark_vision}/3_class/svm_train_time_lc.txt | 0 {vision => benchmark_vision}/8_class/cnn32.txt | 0 {vision => benchmark_vision}/8_class/cnn32_2l.txt | 0 {vision => benchmark_vision}/8_class/cnn32_2l_sc.txt | 0 {vision => benchmark_vision}/8_class/cnn32_2l_st.txt | 0 {vision => benchmark_vision}/8_class/cnn32_2l_test_time.txt | 0 {vision => benchmark_vision}/8_class/cnn32_2l_test_time_sc.txt | 0 {vision => benchmark_vision}/8_class/cnn32_2l_test_time_st.txt | 0 {vision => benchmark_vision}/8_class/cnn32_2l_train_time.txt | 0 {vision => benchmark_vision}/8_class/cnn32_2l_train_time_sc.txt | 0 {vision => benchmark_vision}/8_class/cnn32_2l_train_time_st.txt | 0 {vision => benchmark_vision}/8_class/cnn32_5l.txt | 0 {vision => benchmark_vision}/8_class/cnn32_5l_sc.txt | 0 {vision => benchmark_vision}/8_class/cnn32_5l_st.txt | 0 {vision => benchmark_vision}/8_class/cnn32_5l_test_time.txt | 0 {vision => benchmark_vision}/8_class/cnn32_5l_test_time_sc.txt | 0 {vision => benchmark_vision}/8_class/cnn32_5l_test_time_st.txt | 0 {vision => benchmark_vision}/8_class/cnn32_5l_train_time.txt | 0 {vision => benchmark_vision}/8_class/cnn32_5l_train_time_sc.txt | 0 {vision => benchmark_vision}/8_class/cnn32_5l_train_time_st.txt | 0 {vision => benchmark_vision}/8_class/cnn32_sc.txt | 0 {vision => benchmark_vision}/8_class/cnn32_st.txt | 0 {vision => benchmark_vision}/8_class/cnn32_test_time.txt | 0 {vision => benchmark_vision}/8_class/cnn32_test_time_sc.txt | 0 {vision => benchmark_vision}/8_class/cnn32_test_time_st.txt | 0 {vision => benchmark_vision}/8_class/cnn32_train_time.txt | 0 {vision => benchmark_vision}/8_class/cnn32_train_time_sc.txt | 0 {vision => benchmark_vision}/8_class/cnn32_train_time_st.txt | 0 {vision => benchmark_vision}/8_class/naive_rf.txt | 0 {vision => benchmark_vision}/8_class/naive_rf_test_time.txt | 0 {vision => benchmark_vision}/8_class/naive_rf_test_time_lc.txt | 0 {vision => benchmark_vision}/8_class/naive_rf_train_time.txt | 0 {vision => benchmark_vision}/8_class/naive_rf_train_time_lc.txt | 0 {vision => benchmark_vision}/8_class/resnet18.txt | 0 {vision => benchmark_vision}/8_class/resnet18_sc.txt | 0 {vision => benchmark_vision}/8_class/resnet18_st.txt | 0 {vision => benchmark_vision}/8_class/resnet18_test_time.txt | 0 {vision => benchmark_vision}/8_class/resnet18_test_time_sc.txt | 0 {vision => benchmark_vision}/8_class/resnet18_test_time_st.txt | 0 {vision => benchmark_vision}/8_class/resnet18_train_time.txt | 0 {vision => benchmark_vision}/8_class/resnet18_train_time_sc.txt | 0 {vision => benchmark_vision}/8_class/resnet18_train_time_st.txt | 0 {vision => benchmark_vision}/8_class/svhn_cnn32.txt | 0 {vision => benchmark_vision}/8_class/svhn_cnn32_2l.txt | 0 {vision => benchmark_vision}/8_class/svhn_cnn32_2l_test_time.txt | 0 {vision => benchmark_vision}/8_class/svhn_cnn32_2l_train_time.txt | 0 {vision => benchmark_vision}/8_class/svhn_cnn32_5l.txt | 0 {vision => benchmark_vision}/8_class/svhn_cnn32_5l_test_time.txt | 0 {vision => benchmark_vision}/8_class/svhn_cnn32_5l_train_time.txt | 0 {vision => benchmark_vision}/8_class/svhn_cnn32_test_time.txt | 0 {vision => benchmark_vision}/8_class/svhn_cnn32_train_time.txt | 0 {vision => benchmark_vision}/8_class/svhn_naive_rf.txt | 0 {vision => benchmark_vision}/8_class/svhn_naive_rf_test_time.txt | 0 {vision => benchmark_vision}/8_class/svhn_naive_rf_train_time.txt | 0 {vision => benchmark_vision}/8_class/svhn_resnet18.txt | 0 {vision => benchmark_vision}/8_class/svhn_resnet18_test_time.txt | 0 {vision => benchmark_vision}/8_class/svhn_resnet18_train_time.txt | 0 {vision => benchmark_vision}/8_class/svhn_svm.txt | 0 {vision => benchmark_vision}/8_class/svhn_svm_test_time.txt | 0 {vision => benchmark_vision}/8_class/svhn_svm_train_time.txt | 0 {vision => benchmark_vision}/8_class/svm.txt | 0 {vision => benchmark_vision}/8_class/svm_test_time.txt | 0 {vision => benchmark_vision}/8_class/svm_test_time_lc.txt | 0 {vision => benchmark_vision}/8_class/svm_train_time.txt | 0 {vision => benchmark_vision}/8_class/svm_train_time_lc.txt | 0 {vision => benchmark_vision}/90_class/cnn32.txt | 0 {vision => benchmark_vision}/90_class/cnn32_2l.txt | 0 {vision => benchmark_vision}/90_class/cnn32_2l_sc.txt | 0 {vision => benchmark_vision}/90_class/cnn32_2l_st.txt | 0 {vision => benchmark_vision}/90_class/cnn32_2l_test_time.txt | 0 {vision => benchmark_vision}/90_class/cnn32_2l_test_time_sc.txt | 0 {vision => benchmark_vision}/90_class/cnn32_2l_test_time_st.txt | 0 {vision => benchmark_vision}/90_class/cnn32_2l_train_time.txt | 0 {vision => benchmark_vision}/90_class/cnn32_2l_train_time_sc.txt | 0 {vision => benchmark_vision}/90_class/cnn32_2l_train_time_st.txt | 0 {vision => benchmark_vision}/90_class/cnn32_5l.txt | 0 {vision => benchmark_vision}/90_class/cnn32_5l_sc.txt | 0 {vision => benchmark_vision}/90_class/cnn32_5l_st.txt | 0 {vision => benchmark_vision}/90_class/cnn32_5l_test_time.txt | 0 {vision => benchmark_vision}/90_class/cnn32_5l_test_time_sc.txt | 0 {vision => benchmark_vision}/90_class/cnn32_5l_test_time_st.txt | 0 {vision => benchmark_vision}/90_class/cnn32_5l_train_time.txt | 0 {vision => benchmark_vision}/90_class/cnn32_5l_train_time_sc.txt | 0 {vision => benchmark_vision}/90_class/cnn32_5l_train_time_st.txt | 0 {vision => benchmark_vision}/90_class/cnn32_sc.txt | 0 {vision => benchmark_vision}/90_class/cnn32_st.txt | 0 {vision => benchmark_vision}/90_class/cnn32_test_time.txt | 0 {vision => benchmark_vision}/90_class/cnn32_test_time_sc.txt | 0 {vision => benchmark_vision}/90_class/cnn32_test_time_st.txt | 0 {vision => benchmark_vision}/90_class/cnn32_train_time.txt | 0 {vision => benchmark_vision}/90_class/cnn32_train_time_sc.txt | 0 {vision => benchmark_vision}/90_class/cnn32_train_time_st.txt | 0 {vision => benchmark_vision}/90_class/naive_rf.txt | 0 {vision => benchmark_vision}/90_class/naive_rf_test_time.txt | 0 {vision => benchmark_vision}/90_class/naive_rf_test_time_lc.txt | 0 {vision => benchmark_vision}/90_class/naive_rf_train_time.txt | 0 {vision => benchmark_vision}/90_class/naive_rf_train_time_lc.txt | 0 {vision => benchmark_vision}/90_class/resnet18.txt | 0 {vision => benchmark_vision}/90_class/resnet18_sc.txt | 0 {vision => benchmark_vision}/90_class/resnet18_st.txt | 0 {vision => benchmark_vision}/90_class/resnet18_test_time.txt | 0 {vision => benchmark_vision}/90_class/resnet18_test_time_sc.txt | 0 {vision => benchmark_vision}/90_class/resnet18_test_time_st.txt | 0 {vision => benchmark_vision}/90_class/resnet18_train_time.txt | 0 {vision => benchmark_vision}/90_class/resnet18_train_time_sc.txt | 0 {vision => benchmark_vision}/90_class/resnet18_train_time_st.txt | 0 {vision => benchmark_vision}/90_class/svm.txt | 0 {vision => benchmark_vision}/90_class/svm_test_time.txt | 0 {vision => benchmark_vision}/90_class/svm_test_time_lc.txt | 0 {vision => benchmark_vision}/90_class/svm_train_time.txt | 0 {vision => benchmark_vision}/90_class/svm_train_time_lc.txt | 0 {vision => benchmark_vision}/cifar_10.py | 0 {vision => benchmark_vision}/cifar_100.py | 0 {vision => benchmark_vision}/cifar_100_stc.py | 0 {vision => benchmark_vision}/cifar_10_stc.py | 0 {vision => benchmark_vision}/cifar_figures.ipynb | 0 {vision => benchmark_vision}/svhn.py | 0 {vision => benchmark_vision}/svhn_figure.ipynb | 0 {vision => benchmark_vision}/svhn_toolbox.py | 0 {vision => benchmark_vision}/toolbox.py | 0 308 files changed, 0 insertions(+), 0 deletions(-) rename {audition => benchmark_audition}/3_class/cnn32.txt (100%) rename {audition => benchmark_audition}/3_class/cnn32_2l.txt (100%) rename {audition => benchmark_audition}/3_class/cnn32_2l_test_time.txt (100%) rename {audition => benchmark_audition}/3_class/cnn32_2l_train_time.txt (100%) rename {audition => benchmark_audition}/3_class/cnn32_5l.txt (100%) rename {audition => benchmark_audition}/3_class/cnn32_5l_test_time.txt (100%) rename {audition => benchmark_audition}/3_class/cnn32_5l_train_time.txt (100%) rename {audition => benchmark_audition}/3_class/cnn32_test_time.txt (100%) rename {audition => benchmark_audition}/3_class/cnn32_train_time.txt (100%) rename {audition => benchmark_audition}/3_class/naive_rf.txt (100%) rename {audition => benchmark_audition}/3_class/naive_rf_test_time.txt (100%) rename {audition => benchmark_audition}/3_class/naive_rf_train_time.txt (100%) rename {audition => benchmark_audition}/3_class/resnet18.txt (100%) rename {audition => benchmark_audition}/3_class/resnet18_test_time.txt (100%) rename {audition => benchmark_audition}/3_class/resnet18_train_time.txt (100%) rename {audition => benchmark_audition}/3_class/svm.txt (100%) rename {audition => benchmark_audition}/3_class/svm_test_time.txt (100%) rename {audition => benchmark_audition}/3_class/svm_train_time.txt (100%) rename {audition => benchmark_audition}/3_class_mel/cnn32.txt (100%) rename {audition => benchmark_audition}/3_class_mel/cnn32_2l.txt (100%) rename {audition => benchmark_audition}/3_class_mel/cnn32_2l_test_time.txt (100%) rename {audition => benchmark_audition}/3_class_mel/cnn32_2l_train_time.txt (100%) rename {audition => benchmark_audition}/3_class_mel/cnn32_5l.txt (100%) rename {audition => benchmark_audition}/3_class_mel/cnn32_5l_test_time.txt (100%) rename {audition => benchmark_audition}/3_class_mel/cnn32_5l_train_time.txt (100%) rename {audition => benchmark_audition}/3_class_mel/cnn32_test_time.txt (100%) rename {audition => benchmark_audition}/3_class_mel/cnn32_train_time.txt (100%) rename {audition => benchmark_audition}/3_class_mel/naive_rf.txt (100%) rename {audition => benchmark_audition}/3_class_mel/naive_rf_test_time.txt (100%) rename {audition => benchmark_audition}/3_class_mel/naive_rf_train_time.txt (100%) rename {audition => benchmark_audition}/3_class_mel/resnet18.txt (100%) rename {audition => benchmark_audition}/3_class_mel/resnet18_test_time.txt (100%) rename {audition => benchmark_audition}/3_class_mel/resnet18_train_time.txt (100%) rename {audition => benchmark_audition}/3_class_mel/svm.txt (100%) rename {audition => benchmark_audition}/3_class_mel/svm_test_time.txt (100%) rename {audition => benchmark_audition}/3_class_mel/svm_train_time.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/cnn32.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/cnn32_2l.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/cnn32_2l_test_time.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/cnn32_2l_train_time.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/cnn32_5l.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/cnn32_5l_test_time.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/cnn32_5l_train_time.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/cnn32_test_time.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/cnn32_train_time.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/naive_rf.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/naive_rf_test_time.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/naive_rf_train_time.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/resnet18.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/resnet18_test_time.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/resnet18_train_time.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/svm.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/svm_test_time.txt (100%) rename {audition => benchmark_audition}/3_class_mfcc/svm_train_time.txt (100%) rename {audition => benchmark_audition}/8_class/cnn32.txt (100%) rename {audition => benchmark_audition}/8_class/cnn32_2l.txt (100%) rename {audition => benchmark_audition}/8_class/cnn32_2l_test_time.txt (100%) rename {audition => benchmark_audition}/8_class/cnn32_2l_train_time.txt (100%) rename {audition => benchmark_audition}/8_class/cnn32_5l.txt (100%) rename {audition => benchmark_audition}/8_class/cnn32_5l_test_time.txt (100%) rename {audition => benchmark_audition}/8_class/cnn32_5l_train_time.txt (100%) rename {audition => benchmark_audition}/8_class/cnn32_test_time.txt (100%) rename {audition => benchmark_audition}/8_class/cnn32_train_time.txt (100%) rename {audition => benchmark_audition}/8_class/naive_rf.txt (100%) rename {audition => benchmark_audition}/8_class/naive_rf_test_time.txt (100%) rename {audition => benchmark_audition}/8_class/naive_rf_train_time.txt (100%) rename {audition => benchmark_audition}/8_class/resnet18.txt (100%) rename {audition => benchmark_audition}/8_class/resnet18_test_time.txt (100%) rename {audition => benchmark_audition}/8_class/resnet18_train_time.txt (100%) rename {audition => benchmark_audition}/8_class/svm.txt (100%) rename {audition => benchmark_audition}/8_class/svm_test_time.txt (100%) rename {audition => benchmark_audition}/8_class/svm_train_time.txt (100%) rename {audition => benchmark_audition}/8_class_mel/cnn32.txt (100%) rename {audition => benchmark_audition}/8_class_mel/cnn32_2l.txt (100%) rename {audition => benchmark_audition}/8_class_mel/cnn32_2l_test_time.txt (100%) rename {audition => benchmark_audition}/8_class_mel/cnn32_2l_train_time.txt (100%) rename {audition => benchmark_audition}/8_class_mel/cnn32_5l.txt (100%) rename {audition => benchmark_audition}/8_class_mel/cnn32_5l_test_time.txt (100%) rename {audition => benchmark_audition}/8_class_mel/cnn32_5l_train_time.txt (100%) rename {audition => benchmark_audition}/8_class_mel/cnn32_test_time.txt (100%) rename {audition => benchmark_audition}/8_class_mel/cnn32_train_time.txt (100%) rename {audition => benchmark_audition}/8_class_mel/naive_rf.txt (100%) rename {audition => benchmark_audition}/8_class_mel/naive_rf_test_time.txt (100%) rename {audition => benchmark_audition}/8_class_mel/naive_rf_train_time.txt (100%) rename {audition => benchmark_audition}/8_class_mel/resnet18.txt (100%) rename {audition => benchmark_audition}/8_class_mel/resnet18_test_time.txt (100%) rename {audition => benchmark_audition}/8_class_mel/resnet18_train_time.txt (100%) rename {audition => benchmark_audition}/8_class_mel/svm.txt (100%) rename {audition => benchmark_audition}/8_class_mel/svm_test_time.txt (100%) rename {audition => benchmark_audition}/8_class_mel/svm_train_time.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/cnn32.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/cnn32_2l.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/cnn32_2l_test_time.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/cnn32_2l_train_time.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/cnn32_5l.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/cnn32_5l_test_time.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/cnn32_5l_train_time.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/cnn32_test_time.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/cnn32_train_time.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/naive_rf.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/naive_rf_test_time.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/naive_rf_train_time.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/resnet18.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/resnet18_test_time.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/resnet18_train_time.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/svm.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/svm_test_time.txt (100%) rename {audition => benchmark_audition}/8_class_mfcc/svm_train_time.txt (100%) rename {audition => benchmark_audition}/audio_toolbox.py (100%) rename {audition => benchmark_audition}/fsdd.py (100%) rename {audition => benchmark_audition}/fsdd_figures.ipynb (100%) rename {tabular => benchmark_tabular}/cc18_figures.ipynb (100%) rename {tabular => benchmark_tabular}/cc18_hyperparameter.py (100%) rename {tabular => benchmark_tabular}/cc18_kappa_ece.py (100%) rename {tabular => benchmark_tabular}/cc18_times.py (100%) rename {tabular => benchmark_tabular}/metrics/cc18_all_parameters.txt (100%) rename {tabular => benchmark_tabular}/metrics/cc18_sample_sizes.txt (100%) rename {tabular => benchmark_tabular}/results/cc18_dn_ece.txt (100%) rename {tabular => benchmark_tabular}/results/cc18_dn_kappa.txt (100%) rename {tabular => benchmark_tabular}/results/cc18_dn_times_test.txt (100%) rename {tabular => benchmark_tabular}/results/cc18_dn_times_train.txt (100%) rename {tabular => benchmark_tabular}/results/cc18_rf_ece.txt (100%) rename {tabular => benchmark_tabular}/results/cc18_rf_kappa.txt (100%) rename {tabular => benchmark_tabular}/results/cc18_rf_times_test.txt (100%) rename {tabular => benchmark_tabular}/results/cc18_rf_times_train.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_2l.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_2l_sc.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_2l_st.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_2l_test_time.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_2l_test_time_sc.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_2l_test_time_st.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_2l_train_time.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_2l_train_time_sc.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_2l_train_time_st.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_5l.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_5l_sc.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_5l_st.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_5l_test_time.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_5l_test_time_sc.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_5l_test_time_st.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_5l_train_time.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_5l_train_time_sc.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_5l_train_time_st.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_sc.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_st.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_test_time.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_test_time_sc.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_test_time_st.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_train_time.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_train_time_sc.txt (100%) rename {vision => benchmark_vision}/3_class/cnn32_train_time_st.txt (100%) rename {vision => benchmark_vision}/3_class/naive_rf.txt (100%) rename {vision => benchmark_vision}/3_class/naive_rf_test_time.txt (100%) rename {vision => benchmark_vision}/3_class/naive_rf_test_time_lc.txt (100%) rename {vision => benchmark_vision}/3_class/naive_rf_train_time.txt (100%) rename {vision => benchmark_vision}/3_class/naive_rf_train_time_lc.txt (100%) rename {vision => benchmark_vision}/3_class/resnet18.txt (100%) rename {vision => benchmark_vision}/3_class/resnet18_sc.txt (100%) rename {vision => benchmark_vision}/3_class/resnet18_st.txt (100%) rename {vision => benchmark_vision}/3_class/resnet18_test_time.txt (100%) rename {vision => benchmark_vision}/3_class/resnet18_test_time_sc.txt (100%) rename {vision => benchmark_vision}/3_class/resnet18_test_time_st.txt (100%) rename {vision => benchmark_vision}/3_class/resnet18_train_time.txt (100%) rename {vision => benchmark_vision}/3_class/resnet18_train_time_sc.txt (100%) rename {vision => benchmark_vision}/3_class/resnet18_train_time_st.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_cnn32.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_cnn32_2l.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_cnn32_2l_test_time.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_cnn32_2l_train_time.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_cnn32_5l.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_cnn32_5l_test_time.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_cnn32_5l_train_time.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_cnn32_test_time.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_cnn32_train_time.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_naive_rf.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_naive_rf_test_time.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_naive_rf_train_time.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_resnet18.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_resnet18_test_time.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_resnet18_train_time.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_svm.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_svm_test_time.txt (100%) rename {vision => benchmark_vision}/3_class/svhn_svm_train_time.txt (100%) rename {vision => benchmark_vision}/3_class/svm.txt (100%) rename {vision => benchmark_vision}/3_class/svm_test_time.txt (100%) rename {vision => benchmark_vision}/3_class/svm_test_time_lc.txt (100%) rename {vision => benchmark_vision}/3_class/svm_train_time.txt (100%) rename {vision => benchmark_vision}/3_class/svm_train_time_lc.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_2l.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_2l_sc.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_2l_st.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_2l_test_time.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_2l_test_time_sc.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_2l_test_time_st.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_2l_train_time.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_2l_train_time_sc.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_2l_train_time_st.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_5l.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_5l_sc.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_5l_st.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_5l_test_time.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_5l_test_time_sc.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_5l_test_time_st.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_5l_train_time.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_5l_train_time_sc.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_5l_train_time_st.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_sc.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_st.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_test_time.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_test_time_sc.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_test_time_st.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_train_time.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_train_time_sc.txt (100%) rename {vision => benchmark_vision}/8_class/cnn32_train_time_st.txt (100%) rename {vision => benchmark_vision}/8_class/naive_rf.txt (100%) rename {vision => benchmark_vision}/8_class/naive_rf_test_time.txt (100%) rename {vision => benchmark_vision}/8_class/naive_rf_test_time_lc.txt (100%) rename {vision => benchmark_vision}/8_class/naive_rf_train_time.txt (100%) rename {vision => benchmark_vision}/8_class/naive_rf_train_time_lc.txt (100%) rename {vision => benchmark_vision}/8_class/resnet18.txt (100%) rename {vision => benchmark_vision}/8_class/resnet18_sc.txt (100%) rename {vision => benchmark_vision}/8_class/resnet18_st.txt (100%) rename {vision => benchmark_vision}/8_class/resnet18_test_time.txt (100%) rename {vision => benchmark_vision}/8_class/resnet18_test_time_sc.txt (100%) rename {vision => benchmark_vision}/8_class/resnet18_test_time_st.txt (100%) rename {vision => benchmark_vision}/8_class/resnet18_train_time.txt (100%) rename {vision => benchmark_vision}/8_class/resnet18_train_time_sc.txt (100%) rename {vision => benchmark_vision}/8_class/resnet18_train_time_st.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_cnn32.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_cnn32_2l.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_cnn32_2l_test_time.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_cnn32_2l_train_time.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_cnn32_5l.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_cnn32_5l_test_time.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_cnn32_5l_train_time.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_cnn32_test_time.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_cnn32_train_time.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_naive_rf.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_naive_rf_test_time.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_naive_rf_train_time.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_resnet18.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_resnet18_test_time.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_resnet18_train_time.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_svm.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_svm_test_time.txt (100%) rename {vision => benchmark_vision}/8_class/svhn_svm_train_time.txt (100%) rename {vision => benchmark_vision}/8_class/svm.txt (100%) rename {vision => benchmark_vision}/8_class/svm_test_time.txt (100%) rename {vision => benchmark_vision}/8_class/svm_test_time_lc.txt (100%) rename {vision => benchmark_vision}/8_class/svm_train_time.txt (100%) rename {vision => benchmark_vision}/8_class/svm_train_time_lc.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_2l.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_2l_sc.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_2l_st.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_2l_test_time.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_2l_test_time_sc.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_2l_test_time_st.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_2l_train_time.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_2l_train_time_sc.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_2l_train_time_st.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_5l.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_5l_sc.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_5l_st.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_5l_test_time.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_5l_test_time_sc.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_5l_test_time_st.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_5l_train_time.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_5l_train_time_sc.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_5l_train_time_st.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_sc.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_st.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_test_time.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_test_time_sc.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_test_time_st.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_train_time.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_train_time_sc.txt (100%) rename {vision => benchmark_vision}/90_class/cnn32_train_time_st.txt (100%) rename {vision => benchmark_vision}/90_class/naive_rf.txt (100%) rename {vision => benchmark_vision}/90_class/naive_rf_test_time.txt (100%) rename {vision => benchmark_vision}/90_class/naive_rf_test_time_lc.txt (100%) rename {vision => benchmark_vision}/90_class/naive_rf_train_time.txt (100%) rename {vision => benchmark_vision}/90_class/naive_rf_train_time_lc.txt (100%) rename {vision => benchmark_vision}/90_class/resnet18.txt (100%) rename {vision => benchmark_vision}/90_class/resnet18_sc.txt (100%) rename {vision => benchmark_vision}/90_class/resnet18_st.txt (100%) rename {vision => benchmark_vision}/90_class/resnet18_test_time.txt (100%) rename {vision => benchmark_vision}/90_class/resnet18_test_time_sc.txt (100%) rename {vision => benchmark_vision}/90_class/resnet18_test_time_st.txt (100%) rename {vision => benchmark_vision}/90_class/resnet18_train_time.txt (100%) rename {vision => benchmark_vision}/90_class/resnet18_train_time_sc.txt (100%) rename {vision => benchmark_vision}/90_class/resnet18_train_time_st.txt (100%) rename {vision => benchmark_vision}/90_class/svm.txt (100%) rename {vision => benchmark_vision}/90_class/svm_test_time.txt (100%) rename {vision => benchmark_vision}/90_class/svm_test_time_lc.txt (100%) rename {vision => benchmark_vision}/90_class/svm_train_time.txt (100%) rename {vision => benchmark_vision}/90_class/svm_train_time_lc.txt (100%) rename {vision => benchmark_vision}/cifar_10.py (100%) rename {vision => benchmark_vision}/cifar_100.py (100%) rename {vision => benchmark_vision}/cifar_100_stc.py (100%) rename {vision => benchmark_vision}/cifar_10_stc.py (100%) rename {vision => benchmark_vision}/cifar_figures.ipynb (100%) rename {vision => benchmark_vision}/svhn.py (100%) rename {vision => benchmark_vision}/svhn_figure.ipynb (100%) rename {vision => benchmark_vision}/svhn_toolbox.py (100%) rename {vision => benchmark_vision}/toolbox.py (100%) diff --git a/audition/3_class/cnn32.txt b/benchmark_audition/3_class/cnn32.txt similarity index 100% rename from audition/3_class/cnn32.txt rename to benchmark_audition/3_class/cnn32.txt diff --git a/audition/3_class/cnn32_2l.txt b/benchmark_audition/3_class/cnn32_2l.txt similarity index 100% rename from audition/3_class/cnn32_2l.txt rename to benchmark_audition/3_class/cnn32_2l.txt diff --git a/audition/3_class/cnn32_2l_test_time.txt b/benchmark_audition/3_class/cnn32_2l_test_time.txt similarity index 100% rename from audition/3_class/cnn32_2l_test_time.txt rename to benchmark_audition/3_class/cnn32_2l_test_time.txt diff --git a/audition/3_class/cnn32_2l_train_time.txt b/benchmark_audition/3_class/cnn32_2l_train_time.txt similarity index 100% rename from audition/3_class/cnn32_2l_train_time.txt rename to benchmark_audition/3_class/cnn32_2l_train_time.txt diff --git a/audition/3_class/cnn32_5l.txt b/benchmark_audition/3_class/cnn32_5l.txt similarity index 100% rename from audition/3_class/cnn32_5l.txt rename to benchmark_audition/3_class/cnn32_5l.txt diff --git a/audition/3_class/cnn32_5l_test_time.txt b/benchmark_audition/3_class/cnn32_5l_test_time.txt similarity index 100% rename from audition/3_class/cnn32_5l_test_time.txt rename to benchmark_audition/3_class/cnn32_5l_test_time.txt diff --git a/audition/3_class/cnn32_5l_train_time.txt b/benchmark_audition/3_class/cnn32_5l_train_time.txt similarity index 100% rename from audition/3_class/cnn32_5l_train_time.txt rename to benchmark_audition/3_class/cnn32_5l_train_time.txt diff --git a/audition/3_class/cnn32_test_time.txt b/benchmark_audition/3_class/cnn32_test_time.txt similarity index 100% rename from audition/3_class/cnn32_test_time.txt rename to benchmark_audition/3_class/cnn32_test_time.txt diff --git a/audition/3_class/cnn32_train_time.txt b/benchmark_audition/3_class/cnn32_train_time.txt similarity index 100% rename from audition/3_class/cnn32_train_time.txt rename to benchmark_audition/3_class/cnn32_train_time.txt diff --git a/audition/3_class/naive_rf.txt b/benchmark_audition/3_class/naive_rf.txt similarity index 100% rename from audition/3_class/naive_rf.txt rename to benchmark_audition/3_class/naive_rf.txt diff --git a/audition/3_class/naive_rf_test_time.txt b/benchmark_audition/3_class/naive_rf_test_time.txt similarity index 100% rename from audition/3_class/naive_rf_test_time.txt rename to benchmark_audition/3_class/naive_rf_test_time.txt diff --git a/audition/3_class/naive_rf_train_time.txt b/benchmark_audition/3_class/naive_rf_train_time.txt similarity index 100% rename from audition/3_class/naive_rf_train_time.txt rename to benchmark_audition/3_class/naive_rf_train_time.txt diff --git a/audition/3_class/resnet18.txt b/benchmark_audition/3_class/resnet18.txt similarity index 100% rename from audition/3_class/resnet18.txt rename to benchmark_audition/3_class/resnet18.txt diff --git a/audition/3_class/resnet18_test_time.txt b/benchmark_audition/3_class/resnet18_test_time.txt similarity index 100% rename from audition/3_class/resnet18_test_time.txt rename to benchmark_audition/3_class/resnet18_test_time.txt diff --git a/audition/3_class/resnet18_train_time.txt b/benchmark_audition/3_class/resnet18_train_time.txt similarity index 100% rename from audition/3_class/resnet18_train_time.txt rename to benchmark_audition/3_class/resnet18_train_time.txt diff --git a/audition/3_class/svm.txt b/benchmark_audition/3_class/svm.txt similarity index 100% rename from audition/3_class/svm.txt rename to benchmark_audition/3_class/svm.txt diff --git a/audition/3_class/svm_test_time.txt b/benchmark_audition/3_class/svm_test_time.txt similarity index 100% rename from audition/3_class/svm_test_time.txt rename to benchmark_audition/3_class/svm_test_time.txt diff --git a/audition/3_class/svm_train_time.txt b/benchmark_audition/3_class/svm_train_time.txt similarity index 100% rename from audition/3_class/svm_train_time.txt rename to benchmark_audition/3_class/svm_train_time.txt diff --git a/audition/3_class_mel/cnn32.txt b/benchmark_audition/3_class_mel/cnn32.txt similarity index 100% rename from audition/3_class_mel/cnn32.txt rename to benchmark_audition/3_class_mel/cnn32.txt diff --git a/audition/3_class_mel/cnn32_2l.txt b/benchmark_audition/3_class_mel/cnn32_2l.txt similarity index 100% rename from audition/3_class_mel/cnn32_2l.txt rename to benchmark_audition/3_class_mel/cnn32_2l.txt diff --git a/audition/3_class_mel/cnn32_2l_test_time.txt b/benchmark_audition/3_class_mel/cnn32_2l_test_time.txt similarity index 100% rename from audition/3_class_mel/cnn32_2l_test_time.txt rename to benchmark_audition/3_class_mel/cnn32_2l_test_time.txt diff --git a/audition/3_class_mel/cnn32_2l_train_time.txt b/benchmark_audition/3_class_mel/cnn32_2l_train_time.txt similarity index 100% rename from audition/3_class_mel/cnn32_2l_train_time.txt rename to benchmark_audition/3_class_mel/cnn32_2l_train_time.txt diff --git a/audition/3_class_mel/cnn32_5l.txt b/benchmark_audition/3_class_mel/cnn32_5l.txt similarity index 100% rename from audition/3_class_mel/cnn32_5l.txt rename to benchmark_audition/3_class_mel/cnn32_5l.txt diff --git a/audition/3_class_mel/cnn32_5l_test_time.txt b/benchmark_audition/3_class_mel/cnn32_5l_test_time.txt similarity index 100% rename from audition/3_class_mel/cnn32_5l_test_time.txt rename to benchmark_audition/3_class_mel/cnn32_5l_test_time.txt diff --git a/audition/3_class_mel/cnn32_5l_train_time.txt b/benchmark_audition/3_class_mel/cnn32_5l_train_time.txt similarity index 100% rename from audition/3_class_mel/cnn32_5l_train_time.txt rename to benchmark_audition/3_class_mel/cnn32_5l_train_time.txt diff --git a/audition/3_class_mel/cnn32_test_time.txt b/benchmark_audition/3_class_mel/cnn32_test_time.txt similarity index 100% rename from audition/3_class_mel/cnn32_test_time.txt rename to benchmark_audition/3_class_mel/cnn32_test_time.txt diff --git a/audition/3_class_mel/cnn32_train_time.txt b/benchmark_audition/3_class_mel/cnn32_train_time.txt similarity index 100% rename from audition/3_class_mel/cnn32_train_time.txt rename to benchmark_audition/3_class_mel/cnn32_train_time.txt diff --git a/audition/3_class_mel/naive_rf.txt b/benchmark_audition/3_class_mel/naive_rf.txt similarity index 100% rename from audition/3_class_mel/naive_rf.txt rename to benchmark_audition/3_class_mel/naive_rf.txt diff --git a/audition/3_class_mel/naive_rf_test_time.txt b/benchmark_audition/3_class_mel/naive_rf_test_time.txt similarity index 100% rename from audition/3_class_mel/naive_rf_test_time.txt rename to benchmark_audition/3_class_mel/naive_rf_test_time.txt diff --git a/audition/3_class_mel/naive_rf_train_time.txt b/benchmark_audition/3_class_mel/naive_rf_train_time.txt similarity index 100% rename from audition/3_class_mel/naive_rf_train_time.txt rename to benchmark_audition/3_class_mel/naive_rf_train_time.txt diff --git a/audition/3_class_mel/resnet18.txt b/benchmark_audition/3_class_mel/resnet18.txt similarity index 100% rename from audition/3_class_mel/resnet18.txt rename to benchmark_audition/3_class_mel/resnet18.txt diff --git a/audition/3_class_mel/resnet18_test_time.txt b/benchmark_audition/3_class_mel/resnet18_test_time.txt similarity index 100% rename from audition/3_class_mel/resnet18_test_time.txt rename to benchmark_audition/3_class_mel/resnet18_test_time.txt diff --git a/audition/3_class_mel/resnet18_train_time.txt b/benchmark_audition/3_class_mel/resnet18_train_time.txt similarity index 100% rename from audition/3_class_mel/resnet18_train_time.txt rename to benchmark_audition/3_class_mel/resnet18_train_time.txt diff --git a/audition/3_class_mel/svm.txt b/benchmark_audition/3_class_mel/svm.txt similarity index 100% rename from audition/3_class_mel/svm.txt rename to benchmark_audition/3_class_mel/svm.txt diff --git a/audition/3_class_mel/svm_test_time.txt b/benchmark_audition/3_class_mel/svm_test_time.txt similarity index 100% rename from audition/3_class_mel/svm_test_time.txt rename to benchmark_audition/3_class_mel/svm_test_time.txt diff --git a/audition/3_class_mel/svm_train_time.txt b/benchmark_audition/3_class_mel/svm_train_time.txt similarity index 100% rename from audition/3_class_mel/svm_train_time.txt rename to benchmark_audition/3_class_mel/svm_train_time.txt diff --git a/audition/3_class_mfcc/cnn32.txt b/benchmark_audition/3_class_mfcc/cnn32.txt similarity index 100% rename from audition/3_class_mfcc/cnn32.txt rename to benchmark_audition/3_class_mfcc/cnn32.txt diff --git a/audition/3_class_mfcc/cnn32_2l.txt b/benchmark_audition/3_class_mfcc/cnn32_2l.txt similarity index 100% rename from audition/3_class_mfcc/cnn32_2l.txt rename to benchmark_audition/3_class_mfcc/cnn32_2l.txt diff --git a/audition/3_class_mfcc/cnn32_2l_test_time.txt b/benchmark_audition/3_class_mfcc/cnn32_2l_test_time.txt similarity index 100% rename from audition/3_class_mfcc/cnn32_2l_test_time.txt rename to benchmark_audition/3_class_mfcc/cnn32_2l_test_time.txt diff --git a/audition/3_class_mfcc/cnn32_2l_train_time.txt b/benchmark_audition/3_class_mfcc/cnn32_2l_train_time.txt similarity index 100% rename from audition/3_class_mfcc/cnn32_2l_train_time.txt rename to benchmark_audition/3_class_mfcc/cnn32_2l_train_time.txt diff --git a/audition/3_class_mfcc/cnn32_5l.txt b/benchmark_audition/3_class_mfcc/cnn32_5l.txt similarity index 100% rename from audition/3_class_mfcc/cnn32_5l.txt rename to benchmark_audition/3_class_mfcc/cnn32_5l.txt diff --git a/audition/3_class_mfcc/cnn32_5l_test_time.txt b/benchmark_audition/3_class_mfcc/cnn32_5l_test_time.txt similarity index 100% rename from audition/3_class_mfcc/cnn32_5l_test_time.txt rename to benchmark_audition/3_class_mfcc/cnn32_5l_test_time.txt diff --git a/audition/3_class_mfcc/cnn32_5l_train_time.txt b/benchmark_audition/3_class_mfcc/cnn32_5l_train_time.txt similarity index 100% rename from audition/3_class_mfcc/cnn32_5l_train_time.txt rename to benchmark_audition/3_class_mfcc/cnn32_5l_train_time.txt diff --git a/audition/3_class_mfcc/cnn32_test_time.txt b/benchmark_audition/3_class_mfcc/cnn32_test_time.txt similarity index 100% rename from audition/3_class_mfcc/cnn32_test_time.txt rename to benchmark_audition/3_class_mfcc/cnn32_test_time.txt diff --git a/audition/3_class_mfcc/cnn32_train_time.txt b/benchmark_audition/3_class_mfcc/cnn32_train_time.txt similarity index 100% rename from audition/3_class_mfcc/cnn32_train_time.txt rename to benchmark_audition/3_class_mfcc/cnn32_train_time.txt diff --git a/audition/3_class_mfcc/naive_rf.txt b/benchmark_audition/3_class_mfcc/naive_rf.txt similarity index 100% rename from audition/3_class_mfcc/naive_rf.txt rename to benchmark_audition/3_class_mfcc/naive_rf.txt diff --git a/audition/3_class_mfcc/naive_rf_test_time.txt b/benchmark_audition/3_class_mfcc/naive_rf_test_time.txt similarity index 100% rename from audition/3_class_mfcc/naive_rf_test_time.txt rename to benchmark_audition/3_class_mfcc/naive_rf_test_time.txt diff --git a/audition/3_class_mfcc/naive_rf_train_time.txt b/benchmark_audition/3_class_mfcc/naive_rf_train_time.txt similarity index 100% rename from audition/3_class_mfcc/naive_rf_train_time.txt rename to benchmark_audition/3_class_mfcc/naive_rf_train_time.txt diff --git a/audition/3_class_mfcc/resnet18.txt b/benchmark_audition/3_class_mfcc/resnet18.txt similarity index 100% rename from audition/3_class_mfcc/resnet18.txt rename to benchmark_audition/3_class_mfcc/resnet18.txt diff --git a/audition/3_class_mfcc/resnet18_test_time.txt b/benchmark_audition/3_class_mfcc/resnet18_test_time.txt similarity index 100% rename from audition/3_class_mfcc/resnet18_test_time.txt rename to benchmark_audition/3_class_mfcc/resnet18_test_time.txt diff --git a/audition/3_class_mfcc/resnet18_train_time.txt b/benchmark_audition/3_class_mfcc/resnet18_train_time.txt similarity index 100% rename from audition/3_class_mfcc/resnet18_train_time.txt rename to benchmark_audition/3_class_mfcc/resnet18_train_time.txt diff --git a/audition/3_class_mfcc/svm.txt b/benchmark_audition/3_class_mfcc/svm.txt similarity index 100% rename from audition/3_class_mfcc/svm.txt rename to benchmark_audition/3_class_mfcc/svm.txt diff --git a/audition/3_class_mfcc/svm_test_time.txt b/benchmark_audition/3_class_mfcc/svm_test_time.txt similarity index 100% rename from audition/3_class_mfcc/svm_test_time.txt rename to benchmark_audition/3_class_mfcc/svm_test_time.txt diff --git a/audition/3_class_mfcc/svm_train_time.txt b/benchmark_audition/3_class_mfcc/svm_train_time.txt similarity index 100% rename from audition/3_class_mfcc/svm_train_time.txt rename to benchmark_audition/3_class_mfcc/svm_train_time.txt diff --git a/audition/8_class/cnn32.txt b/benchmark_audition/8_class/cnn32.txt similarity index 100% rename from audition/8_class/cnn32.txt rename to benchmark_audition/8_class/cnn32.txt diff --git a/audition/8_class/cnn32_2l.txt b/benchmark_audition/8_class/cnn32_2l.txt similarity index 100% rename from audition/8_class/cnn32_2l.txt rename to benchmark_audition/8_class/cnn32_2l.txt diff --git a/audition/8_class/cnn32_2l_test_time.txt b/benchmark_audition/8_class/cnn32_2l_test_time.txt similarity index 100% rename from audition/8_class/cnn32_2l_test_time.txt rename to benchmark_audition/8_class/cnn32_2l_test_time.txt diff --git a/audition/8_class/cnn32_2l_train_time.txt b/benchmark_audition/8_class/cnn32_2l_train_time.txt similarity index 100% rename from audition/8_class/cnn32_2l_train_time.txt rename to benchmark_audition/8_class/cnn32_2l_train_time.txt diff --git a/audition/8_class/cnn32_5l.txt b/benchmark_audition/8_class/cnn32_5l.txt similarity index 100% rename from audition/8_class/cnn32_5l.txt rename to benchmark_audition/8_class/cnn32_5l.txt diff --git a/audition/8_class/cnn32_5l_test_time.txt b/benchmark_audition/8_class/cnn32_5l_test_time.txt similarity index 100% rename from audition/8_class/cnn32_5l_test_time.txt rename to benchmark_audition/8_class/cnn32_5l_test_time.txt diff --git a/audition/8_class/cnn32_5l_train_time.txt b/benchmark_audition/8_class/cnn32_5l_train_time.txt similarity index 100% rename from audition/8_class/cnn32_5l_train_time.txt rename to benchmark_audition/8_class/cnn32_5l_train_time.txt diff --git a/audition/8_class/cnn32_test_time.txt b/benchmark_audition/8_class/cnn32_test_time.txt similarity index 100% rename from audition/8_class/cnn32_test_time.txt rename to benchmark_audition/8_class/cnn32_test_time.txt diff --git a/audition/8_class/cnn32_train_time.txt b/benchmark_audition/8_class/cnn32_train_time.txt similarity index 100% rename from audition/8_class/cnn32_train_time.txt rename to benchmark_audition/8_class/cnn32_train_time.txt diff --git a/audition/8_class/naive_rf.txt b/benchmark_audition/8_class/naive_rf.txt similarity index 100% rename from audition/8_class/naive_rf.txt rename to benchmark_audition/8_class/naive_rf.txt diff --git a/audition/8_class/naive_rf_test_time.txt b/benchmark_audition/8_class/naive_rf_test_time.txt similarity index 100% rename from audition/8_class/naive_rf_test_time.txt rename to benchmark_audition/8_class/naive_rf_test_time.txt diff --git a/audition/8_class/naive_rf_train_time.txt b/benchmark_audition/8_class/naive_rf_train_time.txt similarity index 100% rename from audition/8_class/naive_rf_train_time.txt rename to benchmark_audition/8_class/naive_rf_train_time.txt diff --git a/audition/8_class/resnet18.txt b/benchmark_audition/8_class/resnet18.txt similarity index 100% rename from audition/8_class/resnet18.txt rename to benchmark_audition/8_class/resnet18.txt diff --git a/audition/8_class/resnet18_test_time.txt b/benchmark_audition/8_class/resnet18_test_time.txt similarity index 100% rename from audition/8_class/resnet18_test_time.txt rename to benchmark_audition/8_class/resnet18_test_time.txt diff --git a/audition/8_class/resnet18_train_time.txt b/benchmark_audition/8_class/resnet18_train_time.txt similarity index 100% rename from audition/8_class/resnet18_train_time.txt rename to benchmark_audition/8_class/resnet18_train_time.txt diff --git a/audition/8_class/svm.txt b/benchmark_audition/8_class/svm.txt similarity index 100% rename from audition/8_class/svm.txt rename to benchmark_audition/8_class/svm.txt diff --git a/audition/8_class/svm_test_time.txt b/benchmark_audition/8_class/svm_test_time.txt similarity index 100% rename from audition/8_class/svm_test_time.txt rename to benchmark_audition/8_class/svm_test_time.txt diff --git a/audition/8_class/svm_train_time.txt b/benchmark_audition/8_class/svm_train_time.txt similarity index 100% rename from audition/8_class/svm_train_time.txt rename to benchmark_audition/8_class/svm_train_time.txt diff --git a/audition/8_class_mel/cnn32.txt b/benchmark_audition/8_class_mel/cnn32.txt similarity index 100% rename from audition/8_class_mel/cnn32.txt rename to benchmark_audition/8_class_mel/cnn32.txt diff --git a/audition/8_class_mel/cnn32_2l.txt b/benchmark_audition/8_class_mel/cnn32_2l.txt similarity index 100% rename from audition/8_class_mel/cnn32_2l.txt rename to benchmark_audition/8_class_mel/cnn32_2l.txt diff --git a/audition/8_class_mel/cnn32_2l_test_time.txt b/benchmark_audition/8_class_mel/cnn32_2l_test_time.txt similarity index 100% rename from audition/8_class_mel/cnn32_2l_test_time.txt rename to benchmark_audition/8_class_mel/cnn32_2l_test_time.txt diff --git a/audition/8_class_mel/cnn32_2l_train_time.txt b/benchmark_audition/8_class_mel/cnn32_2l_train_time.txt similarity index 100% rename from audition/8_class_mel/cnn32_2l_train_time.txt rename to benchmark_audition/8_class_mel/cnn32_2l_train_time.txt diff --git a/audition/8_class_mel/cnn32_5l.txt b/benchmark_audition/8_class_mel/cnn32_5l.txt similarity index 100% rename from audition/8_class_mel/cnn32_5l.txt rename to benchmark_audition/8_class_mel/cnn32_5l.txt diff --git a/audition/8_class_mel/cnn32_5l_test_time.txt b/benchmark_audition/8_class_mel/cnn32_5l_test_time.txt similarity index 100% rename from audition/8_class_mel/cnn32_5l_test_time.txt rename to benchmark_audition/8_class_mel/cnn32_5l_test_time.txt diff --git a/audition/8_class_mel/cnn32_5l_train_time.txt b/benchmark_audition/8_class_mel/cnn32_5l_train_time.txt similarity index 100% rename from audition/8_class_mel/cnn32_5l_train_time.txt rename to benchmark_audition/8_class_mel/cnn32_5l_train_time.txt diff --git a/audition/8_class_mel/cnn32_test_time.txt b/benchmark_audition/8_class_mel/cnn32_test_time.txt similarity index 100% rename from audition/8_class_mel/cnn32_test_time.txt rename to benchmark_audition/8_class_mel/cnn32_test_time.txt diff --git a/audition/8_class_mel/cnn32_train_time.txt b/benchmark_audition/8_class_mel/cnn32_train_time.txt similarity index 100% rename from audition/8_class_mel/cnn32_train_time.txt rename to benchmark_audition/8_class_mel/cnn32_train_time.txt diff --git a/audition/8_class_mel/naive_rf.txt b/benchmark_audition/8_class_mel/naive_rf.txt similarity index 100% rename from audition/8_class_mel/naive_rf.txt rename to benchmark_audition/8_class_mel/naive_rf.txt diff --git a/audition/8_class_mel/naive_rf_test_time.txt b/benchmark_audition/8_class_mel/naive_rf_test_time.txt similarity index 100% rename from audition/8_class_mel/naive_rf_test_time.txt rename to benchmark_audition/8_class_mel/naive_rf_test_time.txt diff --git a/audition/8_class_mel/naive_rf_train_time.txt b/benchmark_audition/8_class_mel/naive_rf_train_time.txt similarity index 100% rename from audition/8_class_mel/naive_rf_train_time.txt rename to benchmark_audition/8_class_mel/naive_rf_train_time.txt diff --git a/audition/8_class_mel/resnet18.txt b/benchmark_audition/8_class_mel/resnet18.txt similarity index 100% rename from audition/8_class_mel/resnet18.txt rename to benchmark_audition/8_class_mel/resnet18.txt diff --git a/audition/8_class_mel/resnet18_test_time.txt b/benchmark_audition/8_class_mel/resnet18_test_time.txt similarity index 100% rename from audition/8_class_mel/resnet18_test_time.txt rename to benchmark_audition/8_class_mel/resnet18_test_time.txt diff --git a/audition/8_class_mel/resnet18_train_time.txt b/benchmark_audition/8_class_mel/resnet18_train_time.txt similarity index 100% rename from audition/8_class_mel/resnet18_train_time.txt rename to benchmark_audition/8_class_mel/resnet18_train_time.txt diff --git a/audition/8_class_mel/svm.txt b/benchmark_audition/8_class_mel/svm.txt similarity index 100% rename from audition/8_class_mel/svm.txt rename to benchmark_audition/8_class_mel/svm.txt diff --git a/audition/8_class_mel/svm_test_time.txt b/benchmark_audition/8_class_mel/svm_test_time.txt similarity index 100% rename from audition/8_class_mel/svm_test_time.txt rename to benchmark_audition/8_class_mel/svm_test_time.txt diff --git a/audition/8_class_mel/svm_train_time.txt b/benchmark_audition/8_class_mel/svm_train_time.txt similarity index 100% rename from audition/8_class_mel/svm_train_time.txt rename to benchmark_audition/8_class_mel/svm_train_time.txt diff --git a/audition/8_class_mfcc/cnn32.txt b/benchmark_audition/8_class_mfcc/cnn32.txt similarity index 100% rename from audition/8_class_mfcc/cnn32.txt rename to benchmark_audition/8_class_mfcc/cnn32.txt diff --git a/audition/8_class_mfcc/cnn32_2l.txt b/benchmark_audition/8_class_mfcc/cnn32_2l.txt similarity index 100% rename from audition/8_class_mfcc/cnn32_2l.txt rename to benchmark_audition/8_class_mfcc/cnn32_2l.txt diff --git a/audition/8_class_mfcc/cnn32_2l_test_time.txt b/benchmark_audition/8_class_mfcc/cnn32_2l_test_time.txt similarity index 100% rename from audition/8_class_mfcc/cnn32_2l_test_time.txt rename to benchmark_audition/8_class_mfcc/cnn32_2l_test_time.txt diff --git a/audition/8_class_mfcc/cnn32_2l_train_time.txt b/benchmark_audition/8_class_mfcc/cnn32_2l_train_time.txt similarity index 100% rename from audition/8_class_mfcc/cnn32_2l_train_time.txt rename to benchmark_audition/8_class_mfcc/cnn32_2l_train_time.txt diff --git a/audition/8_class_mfcc/cnn32_5l.txt b/benchmark_audition/8_class_mfcc/cnn32_5l.txt similarity index 100% rename from audition/8_class_mfcc/cnn32_5l.txt rename to benchmark_audition/8_class_mfcc/cnn32_5l.txt diff --git a/audition/8_class_mfcc/cnn32_5l_test_time.txt b/benchmark_audition/8_class_mfcc/cnn32_5l_test_time.txt similarity index 100% rename from audition/8_class_mfcc/cnn32_5l_test_time.txt rename to benchmark_audition/8_class_mfcc/cnn32_5l_test_time.txt diff --git a/audition/8_class_mfcc/cnn32_5l_train_time.txt b/benchmark_audition/8_class_mfcc/cnn32_5l_train_time.txt similarity index 100% rename from audition/8_class_mfcc/cnn32_5l_train_time.txt rename to benchmark_audition/8_class_mfcc/cnn32_5l_train_time.txt diff --git a/audition/8_class_mfcc/cnn32_test_time.txt b/benchmark_audition/8_class_mfcc/cnn32_test_time.txt similarity index 100% rename from audition/8_class_mfcc/cnn32_test_time.txt rename to benchmark_audition/8_class_mfcc/cnn32_test_time.txt diff --git a/audition/8_class_mfcc/cnn32_train_time.txt b/benchmark_audition/8_class_mfcc/cnn32_train_time.txt similarity index 100% rename from audition/8_class_mfcc/cnn32_train_time.txt rename to benchmark_audition/8_class_mfcc/cnn32_train_time.txt diff --git a/audition/8_class_mfcc/naive_rf.txt b/benchmark_audition/8_class_mfcc/naive_rf.txt similarity index 100% rename from audition/8_class_mfcc/naive_rf.txt rename to benchmark_audition/8_class_mfcc/naive_rf.txt diff --git a/audition/8_class_mfcc/naive_rf_test_time.txt b/benchmark_audition/8_class_mfcc/naive_rf_test_time.txt similarity index 100% rename from audition/8_class_mfcc/naive_rf_test_time.txt rename to benchmark_audition/8_class_mfcc/naive_rf_test_time.txt diff --git a/audition/8_class_mfcc/naive_rf_train_time.txt b/benchmark_audition/8_class_mfcc/naive_rf_train_time.txt similarity index 100% rename from audition/8_class_mfcc/naive_rf_train_time.txt rename to benchmark_audition/8_class_mfcc/naive_rf_train_time.txt diff --git a/audition/8_class_mfcc/resnet18.txt b/benchmark_audition/8_class_mfcc/resnet18.txt similarity index 100% rename from audition/8_class_mfcc/resnet18.txt rename to benchmark_audition/8_class_mfcc/resnet18.txt diff --git a/audition/8_class_mfcc/resnet18_test_time.txt b/benchmark_audition/8_class_mfcc/resnet18_test_time.txt similarity index 100% rename from audition/8_class_mfcc/resnet18_test_time.txt rename to benchmark_audition/8_class_mfcc/resnet18_test_time.txt diff --git a/audition/8_class_mfcc/resnet18_train_time.txt b/benchmark_audition/8_class_mfcc/resnet18_train_time.txt similarity index 100% rename from audition/8_class_mfcc/resnet18_train_time.txt rename to benchmark_audition/8_class_mfcc/resnet18_train_time.txt diff --git a/audition/8_class_mfcc/svm.txt b/benchmark_audition/8_class_mfcc/svm.txt similarity index 100% rename from audition/8_class_mfcc/svm.txt rename to benchmark_audition/8_class_mfcc/svm.txt diff --git a/audition/8_class_mfcc/svm_test_time.txt b/benchmark_audition/8_class_mfcc/svm_test_time.txt similarity index 100% rename from audition/8_class_mfcc/svm_test_time.txt rename to benchmark_audition/8_class_mfcc/svm_test_time.txt diff --git a/audition/8_class_mfcc/svm_train_time.txt b/benchmark_audition/8_class_mfcc/svm_train_time.txt similarity index 100% rename from audition/8_class_mfcc/svm_train_time.txt rename to benchmark_audition/8_class_mfcc/svm_train_time.txt diff --git a/audition/audio_toolbox.py b/benchmark_audition/audio_toolbox.py similarity index 100% rename from audition/audio_toolbox.py rename to benchmark_audition/audio_toolbox.py diff --git a/audition/fsdd.py b/benchmark_audition/fsdd.py similarity index 100% rename from audition/fsdd.py rename to benchmark_audition/fsdd.py diff --git a/audition/fsdd_figures.ipynb b/benchmark_audition/fsdd_figures.ipynb similarity index 100% rename from audition/fsdd_figures.ipynb rename to benchmark_audition/fsdd_figures.ipynb diff --git a/tabular/cc18_figures.ipynb b/benchmark_tabular/cc18_figures.ipynb similarity index 100% rename from tabular/cc18_figures.ipynb rename to benchmark_tabular/cc18_figures.ipynb diff --git a/tabular/cc18_hyperparameter.py b/benchmark_tabular/cc18_hyperparameter.py similarity index 100% rename from tabular/cc18_hyperparameter.py rename to benchmark_tabular/cc18_hyperparameter.py diff --git a/tabular/cc18_kappa_ece.py b/benchmark_tabular/cc18_kappa_ece.py similarity index 100% rename from tabular/cc18_kappa_ece.py rename to benchmark_tabular/cc18_kappa_ece.py diff --git a/tabular/cc18_times.py b/benchmark_tabular/cc18_times.py similarity index 100% rename from tabular/cc18_times.py rename to benchmark_tabular/cc18_times.py diff --git a/tabular/metrics/cc18_all_parameters.txt b/benchmark_tabular/metrics/cc18_all_parameters.txt similarity index 100% rename from tabular/metrics/cc18_all_parameters.txt rename to benchmark_tabular/metrics/cc18_all_parameters.txt diff --git a/tabular/metrics/cc18_sample_sizes.txt b/benchmark_tabular/metrics/cc18_sample_sizes.txt similarity index 100% rename from tabular/metrics/cc18_sample_sizes.txt rename to benchmark_tabular/metrics/cc18_sample_sizes.txt diff --git a/tabular/results/cc18_dn_ece.txt b/benchmark_tabular/results/cc18_dn_ece.txt similarity index 100% rename from tabular/results/cc18_dn_ece.txt rename to benchmark_tabular/results/cc18_dn_ece.txt diff --git a/tabular/results/cc18_dn_kappa.txt b/benchmark_tabular/results/cc18_dn_kappa.txt similarity index 100% rename from tabular/results/cc18_dn_kappa.txt rename to benchmark_tabular/results/cc18_dn_kappa.txt diff --git a/tabular/results/cc18_dn_times_test.txt b/benchmark_tabular/results/cc18_dn_times_test.txt similarity index 100% rename from tabular/results/cc18_dn_times_test.txt rename to benchmark_tabular/results/cc18_dn_times_test.txt diff --git a/tabular/results/cc18_dn_times_train.txt b/benchmark_tabular/results/cc18_dn_times_train.txt similarity index 100% rename from tabular/results/cc18_dn_times_train.txt rename to benchmark_tabular/results/cc18_dn_times_train.txt diff --git a/tabular/results/cc18_rf_ece.txt b/benchmark_tabular/results/cc18_rf_ece.txt similarity index 100% rename from tabular/results/cc18_rf_ece.txt rename to benchmark_tabular/results/cc18_rf_ece.txt diff --git a/tabular/results/cc18_rf_kappa.txt b/benchmark_tabular/results/cc18_rf_kappa.txt similarity index 100% rename from tabular/results/cc18_rf_kappa.txt rename to benchmark_tabular/results/cc18_rf_kappa.txt diff --git a/tabular/results/cc18_rf_times_test.txt b/benchmark_tabular/results/cc18_rf_times_test.txt similarity index 100% rename from tabular/results/cc18_rf_times_test.txt rename to benchmark_tabular/results/cc18_rf_times_test.txt diff --git a/tabular/results/cc18_rf_times_train.txt b/benchmark_tabular/results/cc18_rf_times_train.txt similarity index 100% rename from tabular/results/cc18_rf_times_train.txt rename to benchmark_tabular/results/cc18_rf_times_train.txt diff --git a/vision/3_class/cnn32.txt b/benchmark_vision/3_class/cnn32.txt similarity index 100% rename from vision/3_class/cnn32.txt rename to benchmark_vision/3_class/cnn32.txt diff --git a/vision/3_class/cnn32_2l.txt b/benchmark_vision/3_class/cnn32_2l.txt similarity index 100% rename from vision/3_class/cnn32_2l.txt rename to benchmark_vision/3_class/cnn32_2l.txt diff --git a/vision/3_class/cnn32_2l_sc.txt b/benchmark_vision/3_class/cnn32_2l_sc.txt similarity index 100% rename from vision/3_class/cnn32_2l_sc.txt rename to benchmark_vision/3_class/cnn32_2l_sc.txt diff --git a/vision/3_class/cnn32_2l_st.txt b/benchmark_vision/3_class/cnn32_2l_st.txt similarity index 100% rename from vision/3_class/cnn32_2l_st.txt rename to benchmark_vision/3_class/cnn32_2l_st.txt diff --git a/vision/3_class/cnn32_2l_test_time.txt b/benchmark_vision/3_class/cnn32_2l_test_time.txt similarity index 100% rename from vision/3_class/cnn32_2l_test_time.txt rename to benchmark_vision/3_class/cnn32_2l_test_time.txt diff --git a/vision/3_class/cnn32_2l_test_time_sc.txt b/benchmark_vision/3_class/cnn32_2l_test_time_sc.txt similarity index 100% rename from vision/3_class/cnn32_2l_test_time_sc.txt rename to benchmark_vision/3_class/cnn32_2l_test_time_sc.txt diff --git a/vision/3_class/cnn32_2l_test_time_st.txt b/benchmark_vision/3_class/cnn32_2l_test_time_st.txt similarity index 100% rename from vision/3_class/cnn32_2l_test_time_st.txt rename to benchmark_vision/3_class/cnn32_2l_test_time_st.txt diff --git a/vision/3_class/cnn32_2l_train_time.txt b/benchmark_vision/3_class/cnn32_2l_train_time.txt similarity index 100% rename from vision/3_class/cnn32_2l_train_time.txt rename to benchmark_vision/3_class/cnn32_2l_train_time.txt diff --git a/vision/3_class/cnn32_2l_train_time_sc.txt b/benchmark_vision/3_class/cnn32_2l_train_time_sc.txt similarity index 100% rename from vision/3_class/cnn32_2l_train_time_sc.txt rename to benchmark_vision/3_class/cnn32_2l_train_time_sc.txt diff --git a/vision/3_class/cnn32_2l_train_time_st.txt b/benchmark_vision/3_class/cnn32_2l_train_time_st.txt similarity index 100% rename from vision/3_class/cnn32_2l_train_time_st.txt rename to benchmark_vision/3_class/cnn32_2l_train_time_st.txt diff --git a/vision/3_class/cnn32_5l.txt b/benchmark_vision/3_class/cnn32_5l.txt similarity index 100% rename from vision/3_class/cnn32_5l.txt rename to benchmark_vision/3_class/cnn32_5l.txt diff --git a/vision/3_class/cnn32_5l_sc.txt b/benchmark_vision/3_class/cnn32_5l_sc.txt similarity index 100% rename from vision/3_class/cnn32_5l_sc.txt rename to benchmark_vision/3_class/cnn32_5l_sc.txt diff --git a/vision/3_class/cnn32_5l_st.txt b/benchmark_vision/3_class/cnn32_5l_st.txt similarity index 100% rename from vision/3_class/cnn32_5l_st.txt rename to benchmark_vision/3_class/cnn32_5l_st.txt diff --git a/vision/3_class/cnn32_5l_test_time.txt b/benchmark_vision/3_class/cnn32_5l_test_time.txt similarity index 100% rename from vision/3_class/cnn32_5l_test_time.txt rename to benchmark_vision/3_class/cnn32_5l_test_time.txt diff --git a/vision/3_class/cnn32_5l_test_time_sc.txt b/benchmark_vision/3_class/cnn32_5l_test_time_sc.txt similarity index 100% rename from vision/3_class/cnn32_5l_test_time_sc.txt rename to benchmark_vision/3_class/cnn32_5l_test_time_sc.txt diff --git a/vision/3_class/cnn32_5l_test_time_st.txt b/benchmark_vision/3_class/cnn32_5l_test_time_st.txt similarity index 100% rename from vision/3_class/cnn32_5l_test_time_st.txt rename to benchmark_vision/3_class/cnn32_5l_test_time_st.txt diff --git a/vision/3_class/cnn32_5l_train_time.txt b/benchmark_vision/3_class/cnn32_5l_train_time.txt similarity index 100% rename from vision/3_class/cnn32_5l_train_time.txt rename to benchmark_vision/3_class/cnn32_5l_train_time.txt diff --git a/vision/3_class/cnn32_5l_train_time_sc.txt b/benchmark_vision/3_class/cnn32_5l_train_time_sc.txt similarity index 100% rename from vision/3_class/cnn32_5l_train_time_sc.txt rename to benchmark_vision/3_class/cnn32_5l_train_time_sc.txt diff --git a/vision/3_class/cnn32_5l_train_time_st.txt b/benchmark_vision/3_class/cnn32_5l_train_time_st.txt similarity index 100% rename from vision/3_class/cnn32_5l_train_time_st.txt rename to benchmark_vision/3_class/cnn32_5l_train_time_st.txt diff --git a/vision/3_class/cnn32_sc.txt b/benchmark_vision/3_class/cnn32_sc.txt similarity index 100% rename from vision/3_class/cnn32_sc.txt rename to benchmark_vision/3_class/cnn32_sc.txt diff --git a/vision/3_class/cnn32_st.txt b/benchmark_vision/3_class/cnn32_st.txt similarity index 100% rename from vision/3_class/cnn32_st.txt rename to benchmark_vision/3_class/cnn32_st.txt diff --git a/vision/3_class/cnn32_test_time.txt b/benchmark_vision/3_class/cnn32_test_time.txt similarity index 100% rename from vision/3_class/cnn32_test_time.txt rename to benchmark_vision/3_class/cnn32_test_time.txt diff --git a/vision/3_class/cnn32_test_time_sc.txt b/benchmark_vision/3_class/cnn32_test_time_sc.txt similarity index 100% rename from vision/3_class/cnn32_test_time_sc.txt rename to benchmark_vision/3_class/cnn32_test_time_sc.txt diff --git a/vision/3_class/cnn32_test_time_st.txt b/benchmark_vision/3_class/cnn32_test_time_st.txt similarity index 100% rename from vision/3_class/cnn32_test_time_st.txt rename to benchmark_vision/3_class/cnn32_test_time_st.txt diff --git a/vision/3_class/cnn32_train_time.txt b/benchmark_vision/3_class/cnn32_train_time.txt similarity index 100% rename from vision/3_class/cnn32_train_time.txt rename to benchmark_vision/3_class/cnn32_train_time.txt diff --git a/vision/3_class/cnn32_train_time_sc.txt b/benchmark_vision/3_class/cnn32_train_time_sc.txt similarity index 100% rename from vision/3_class/cnn32_train_time_sc.txt rename to benchmark_vision/3_class/cnn32_train_time_sc.txt diff --git a/vision/3_class/cnn32_train_time_st.txt b/benchmark_vision/3_class/cnn32_train_time_st.txt similarity index 100% rename from vision/3_class/cnn32_train_time_st.txt rename to benchmark_vision/3_class/cnn32_train_time_st.txt diff --git a/vision/3_class/naive_rf.txt b/benchmark_vision/3_class/naive_rf.txt similarity index 100% rename from vision/3_class/naive_rf.txt rename to benchmark_vision/3_class/naive_rf.txt diff --git a/vision/3_class/naive_rf_test_time.txt b/benchmark_vision/3_class/naive_rf_test_time.txt similarity index 100% rename from vision/3_class/naive_rf_test_time.txt rename to benchmark_vision/3_class/naive_rf_test_time.txt diff --git a/vision/3_class/naive_rf_test_time_lc.txt b/benchmark_vision/3_class/naive_rf_test_time_lc.txt similarity index 100% rename from vision/3_class/naive_rf_test_time_lc.txt rename to benchmark_vision/3_class/naive_rf_test_time_lc.txt diff --git a/vision/3_class/naive_rf_train_time.txt b/benchmark_vision/3_class/naive_rf_train_time.txt similarity index 100% rename from vision/3_class/naive_rf_train_time.txt rename to benchmark_vision/3_class/naive_rf_train_time.txt diff --git a/vision/3_class/naive_rf_train_time_lc.txt b/benchmark_vision/3_class/naive_rf_train_time_lc.txt similarity index 100% rename from vision/3_class/naive_rf_train_time_lc.txt rename to benchmark_vision/3_class/naive_rf_train_time_lc.txt diff --git a/vision/3_class/resnet18.txt b/benchmark_vision/3_class/resnet18.txt similarity index 100% rename from vision/3_class/resnet18.txt rename to benchmark_vision/3_class/resnet18.txt diff --git a/vision/3_class/resnet18_sc.txt b/benchmark_vision/3_class/resnet18_sc.txt similarity index 100% rename from vision/3_class/resnet18_sc.txt rename to benchmark_vision/3_class/resnet18_sc.txt diff --git a/vision/3_class/resnet18_st.txt b/benchmark_vision/3_class/resnet18_st.txt similarity index 100% rename from vision/3_class/resnet18_st.txt rename to benchmark_vision/3_class/resnet18_st.txt diff --git a/vision/3_class/resnet18_test_time.txt b/benchmark_vision/3_class/resnet18_test_time.txt similarity index 100% rename from vision/3_class/resnet18_test_time.txt rename to benchmark_vision/3_class/resnet18_test_time.txt diff --git a/vision/3_class/resnet18_test_time_sc.txt b/benchmark_vision/3_class/resnet18_test_time_sc.txt similarity index 100% rename from vision/3_class/resnet18_test_time_sc.txt rename to benchmark_vision/3_class/resnet18_test_time_sc.txt diff --git a/vision/3_class/resnet18_test_time_st.txt b/benchmark_vision/3_class/resnet18_test_time_st.txt similarity index 100% rename from vision/3_class/resnet18_test_time_st.txt rename to benchmark_vision/3_class/resnet18_test_time_st.txt diff --git a/vision/3_class/resnet18_train_time.txt b/benchmark_vision/3_class/resnet18_train_time.txt similarity index 100% rename from vision/3_class/resnet18_train_time.txt rename to benchmark_vision/3_class/resnet18_train_time.txt diff --git a/vision/3_class/resnet18_train_time_sc.txt b/benchmark_vision/3_class/resnet18_train_time_sc.txt similarity index 100% rename from vision/3_class/resnet18_train_time_sc.txt rename to benchmark_vision/3_class/resnet18_train_time_sc.txt diff --git a/vision/3_class/resnet18_train_time_st.txt b/benchmark_vision/3_class/resnet18_train_time_st.txt similarity index 100% rename from vision/3_class/resnet18_train_time_st.txt rename to benchmark_vision/3_class/resnet18_train_time_st.txt diff --git a/vision/3_class/svhn_cnn32.txt b/benchmark_vision/3_class/svhn_cnn32.txt similarity index 100% rename from vision/3_class/svhn_cnn32.txt rename to benchmark_vision/3_class/svhn_cnn32.txt diff --git a/vision/3_class/svhn_cnn32_2l.txt b/benchmark_vision/3_class/svhn_cnn32_2l.txt similarity index 100% rename from vision/3_class/svhn_cnn32_2l.txt rename to benchmark_vision/3_class/svhn_cnn32_2l.txt diff --git a/vision/3_class/svhn_cnn32_2l_test_time.txt b/benchmark_vision/3_class/svhn_cnn32_2l_test_time.txt similarity index 100% rename from vision/3_class/svhn_cnn32_2l_test_time.txt rename to benchmark_vision/3_class/svhn_cnn32_2l_test_time.txt diff --git a/vision/3_class/svhn_cnn32_2l_train_time.txt b/benchmark_vision/3_class/svhn_cnn32_2l_train_time.txt similarity index 100% rename from vision/3_class/svhn_cnn32_2l_train_time.txt rename to benchmark_vision/3_class/svhn_cnn32_2l_train_time.txt diff --git a/vision/3_class/svhn_cnn32_5l.txt b/benchmark_vision/3_class/svhn_cnn32_5l.txt similarity index 100% rename from vision/3_class/svhn_cnn32_5l.txt rename to benchmark_vision/3_class/svhn_cnn32_5l.txt diff --git a/vision/3_class/svhn_cnn32_5l_test_time.txt b/benchmark_vision/3_class/svhn_cnn32_5l_test_time.txt similarity index 100% rename from vision/3_class/svhn_cnn32_5l_test_time.txt rename to benchmark_vision/3_class/svhn_cnn32_5l_test_time.txt diff --git a/vision/3_class/svhn_cnn32_5l_train_time.txt b/benchmark_vision/3_class/svhn_cnn32_5l_train_time.txt similarity index 100% rename from vision/3_class/svhn_cnn32_5l_train_time.txt rename to benchmark_vision/3_class/svhn_cnn32_5l_train_time.txt diff --git a/vision/3_class/svhn_cnn32_test_time.txt b/benchmark_vision/3_class/svhn_cnn32_test_time.txt similarity index 100% rename from vision/3_class/svhn_cnn32_test_time.txt rename to benchmark_vision/3_class/svhn_cnn32_test_time.txt diff --git a/vision/3_class/svhn_cnn32_train_time.txt b/benchmark_vision/3_class/svhn_cnn32_train_time.txt similarity index 100% rename from vision/3_class/svhn_cnn32_train_time.txt rename to benchmark_vision/3_class/svhn_cnn32_train_time.txt diff --git a/vision/3_class/svhn_naive_rf.txt b/benchmark_vision/3_class/svhn_naive_rf.txt similarity index 100% rename from vision/3_class/svhn_naive_rf.txt rename to benchmark_vision/3_class/svhn_naive_rf.txt diff --git a/vision/3_class/svhn_naive_rf_test_time.txt b/benchmark_vision/3_class/svhn_naive_rf_test_time.txt similarity index 100% rename from vision/3_class/svhn_naive_rf_test_time.txt rename to benchmark_vision/3_class/svhn_naive_rf_test_time.txt diff --git a/vision/3_class/svhn_naive_rf_train_time.txt b/benchmark_vision/3_class/svhn_naive_rf_train_time.txt similarity index 100% rename from vision/3_class/svhn_naive_rf_train_time.txt rename to benchmark_vision/3_class/svhn_naive_rf_train_time.txt diff --git a/vision/3_class/svhn_resnet18.txt b/benchmark_vision/3_class/svhn_resnet18.txt similarity index 100% rename from vision/3_class/svhn_resnet18.txt rename to benchmark_vision/3_class/svhn_resnet18.txt diff --git a/vision/3_class/svhn_resnet18_test_time.txt b/benchmark_vision/3_class/svhn_resnet18_test_time.txt similarity index 100% rename from vision/3_class/svhn_resnet18_test_time.txt rename to benchmark_vision/3_class/svhn_resnet18_test_time.txt diff --git a/vision/3_class/svhn_resnet18_train_time.txt b/benchmark_vision/3_class/svhn_resnet18_train_time.txt similarity index 100% rename from vision/3_class/svhn_resnet18_train_time.txt rename to benchmark_vision/3_class/svhn_resnet18_train_time.txt diff --git a/vision/3_class/svhn_svm.txt b/benchmark_vision/3_class/svhn_svm.txt similarity index 100% rename from vision/3_class/svhn_svm.txt rename to benchmark_vision/3_class/svhn_svm.txt diff --git a/vision/3_class/svhn_svm_test_time.txt b/benchmark_vision/3_class/svhn_svm_test_time.txt similarity index 100% rename from vision/3_class/svhn_svm_test_time.txt rename to benchmark_vision/3_class/svhn_svm_test_time.txt diff --git a/vision/3_class/svhn_svm_train_time.txt b/benchmark_vision/3_class/svhn_svm_train_time.txt similarity index 100% rename from vision/3_class/svhn_svm_train_time.txt rename to benchmark_vision/3_class/svhn_svm_train_time.txt diff --git a/vision/3_class/svm.txt b/benchmark_vision/3_class/svm.txt similarity index 100% rename from vision/3_class/svm.txt rename to benchmark_vision/3_class/svm.txt diff --git a/vision/3_class/svm_test_time.txt b/benchmark_vision/3_class/svm_test_time.txt similarity index 100% rename from vision/3_class/svm_test_time.txt rename to benchmark_vision/3_class/svm_test_time.txt diff --git a/vision/3_class/svm_test_time_lc.txt b/benchmark_vision/3_class/svm_test_time_lc.txt similarity index 100% rename from vision/3_class/svm_test_time_lc.txt rename to benchmark_vision/3_class/svm_test_time_lc.txt diff --git a/vision/3_class/svm_train_time.txt b/benchmark_vision/3_class/svm_train_time.txt similarity index 100% rename from vision/3_class/svm_train_time.txt rename to benchmark_vision/3_class/svm_train_time.txt diff --git a/vision/3_class/svm_train_time_lc.txt b/benchmark_vision/3_class/svm_train_time_lc.txt similarity index 100% rename from vision/3_class/svm_train_time_lc.txt rename to benchmark_vision/3_class/svm_train_time_lc.txt diff --git a/vision/8_class/cnn32.txt b/benchmark_vision/8_class/cnn32.txt similarity index 100% rename from vision/8_class/cnn32.txt rename to benchmark_vision/8_class/cnn32.txt diff --git a/vision/8_class/cnn32_2l.txt b/benchmark_vision/8_class/cnn32_2l.txt similarity index 100% rename from vision/8_class/cnn32_2l.txt rename to benchmark_vision/8_class/cnn32_2l.txt diff --git a/vision/8_class/cnn32_2l_sc.txt b/benchmark_vision/8_class/cnn32_2l_sc.txt similarity index 100% rename from vision/8_class/cnn32_2l_sc.txt rename to benchmark_vision/8_class/cnn32_2l_sc.txt diff --git a/vision/8_class/cnn32_2l_st.txt b/benchmark_vision/8_class/cnn32_2l_st.txt similarity index 100% rename from vision/8_class/cnn32_2l_st.txt rename to benchmark_vision/8_class/cnn32_2l_st.txt diff --git a/vision/8_class/cnn32_2l_test_time.txt b/benchmark_vision/8_class/cnn32_2l_test_time.txt similarity index 100% rename from vision/8_class/cnn32_2l_test_time.txt rename to benchmark_vision/8_class/cnn32_2l_test_time.txt diff --git a/vision/8_class/cnn32_2l_test_time_sc.txt b/benchmark_vision/8_class/cnn32_2l_test_time_sc.txt similarity index 100% rename from vision/8_class/cnn32_2l_test_time_sc.txt rename to benchmark_vision/8_class/cnn32_2l_test_time_sc.txt diff --git a/vision/8_class/cnn32_2l_test_time_st.txt b/benchmark_vision/8_class/cnn32_2l_test_time_st.txt similarity index 100% rename from vision/8_class/cnn32_2l_test_time_st.txt rename to benchmark_vision/8_class/cnn32_2l_test_time_st.txt diff --git a/vision/8_class/cnn32_2l_train_time.txt b/benchmark_vision/8_class/cnn32_2l_train_time.txt similarity index 100% rename from vision/8_class/cnn32_2l_train_time.txt rename to benchmark_vision/8_class/cnn32_2l_train_time.txt diff --git a/vision/8_class/cnn32_2l_train_time_sc.txt b/benchmark_vision/8_class/cnn32_2l_train_time_sc.txt similarity index 100% rename from vision/8_class/cnn32_2l_train_time_sc.txt rename to benchmark_vision/8_class/cnn32_2l_train_time_sc.txt diff --git a/vision/8_class/cnn32_2l_train_time_st.txt b/benchmark_vision/8_class/cnn32_2l_train_time_st.txt similarity index 100% rename from vision/8_class/cnn32_2l_train_time_st.txt rename to benchmark_vision/8_class/cnn32_2l_train_time_st.txt diff --git a/vision/8_class/cnn32_5l.txt b/benchmark_vision/8_class/cnn32_5l.txt similarity index 100% rename from vision/8_class/cnn32_5l.txt rename to benchmark_vision/8_class/cnn32_5l.txt diff --git a/vision/8_class/cnn32_5l_sc.txt b/benchmark_vision/8_class/cnn32_5l_sc.txt similarity index 100% rename from vision/8_class/cnn32_5l_sc.txt rename to benchmark_vision/8_class/cnn32_5l_sc.txt diff --git a/vision/8_class/cnn32_5l_st.txt b/benchmark_vision/8_class/cnn32_5l_st.txt similarity index 100% rename from vision/8_class/cnn32_5l_st.txt rename to benchmark_vision/8_class/cnn32_5l_st.txt diff --git a/vision/8_class/cnn32_5l_test_time.txt b/benchmark_vision/8_class/cnn32_5l_test_time.txt similarity index 100% rename from vision/8_class/cnn32_5l_test_time.txt rename to benchmark_vision/8_class/cnn32_5l_test_time.txt diff --git a/vision/8_class/cnn32_5l_test_time_sc.txt b/benchmark_vision/8_class/cnn32_5l_test_time_sc.txt similarity index 100% rename from vision/8_class/cnn32_5l_test_time_sc.txt rename to benchmark_vision/8_class/cnn32_5l_test_time_sc.txt diff --git a/vision/8_class/cnn32_5l_test_time_st.txt b/benchmark_vision/8_class/cnn32_5l_test_time_st.txt similarity index 100% rename from vision/8_class/cnn32_5l_test_time_st.txt rename to benchmark_vision/8_class/cnn32_5l_test_time_st.txt diff --git a/vision/8_class/cnn32_5l_train_time.txt b/benchmark_vision/8_class/cnn32_5l_train_time.txt similarity index 100% rename from vision/8_class/cnn32_5l_train_time.txt rename to benchmark_vision/8_class/cnn32_5l_train_time.txt diff --git a/vision/8_class/cnn32_5l_train_time_sc.txt b/benchmark_vision/8_class/cnn32_5l_train_time_sc.txt similarity index 100% rename from vision/8_class/cnn32_5l_train_time_sc.txt rename to benchmark_vision/8_class/cnn32_5l_train_time_sc.txt diff --git a/vision/8_class/cnn32_5l_train_time_st.txt b/benchmark_vision/8_class/cnn32_5l_train_time_st.txt similarity index 100% rename from vision/8_class/cnn32_5l_train_time_st.txt rename to benchmark_vision/8_class/cnn32_5l_train_time_st.txt diff --git a/vision/8_class/cnn32_sc.txt b/benchmark_vision/8_class/cnn32_sc.txt similarity index 100% rename from vision/8_class/cnn32_sc.txt rename to benchmark_vision/8_class/cnn32_sc.txt diff --git a/vision/8_class/cnn32_st.txt b/benchmark_vision/8_class/cnn32_st.txt similarity index 100% rename from vision/8_class/cnn32_st.txt rename to benchmark_vision/8_class/cnn32_st.txt diff --git a/vision/8_class/cnn32_test_time.txt b/benchmark_vision/8_class/cnn32_test_time.txt similarity index 100% rename from vision/8_class/cnn32_test_time.txt rename to benchmark_vision/8_class/cnn32_test_time.txt diff --git a/vision/8_class/cnn32_test_time_sc.txt b/benchmark_vision/8_class/cnn32_test_time_sc.txt similarity index 100% rename from vision/8_class/cnn32_test_time_sc.txt rename to benchmark_vision/8_class/cnn32_test_time_sc.txt diff --git a/vision/8_class/cnn32_test_time_st.txt b/benchmark_vision/8_class/cnn32_test_time_st.txt similarity index 100% rename from vision/8_class/cnn32_test_time_st.txt rename to benchmark_vision/8_class/cnn32_test_time_st.txt diff --git a/vision/8_class/cnn32_train_time.txt b/benchmark_vision/8_class/cnn32_train_time.txt similarity index 100% rename from vision/8_class/cnn32_train_time.txt rename to benchmark_vision/8_class/cnn32_train_time.txt diff --git a/vision/8_class/cnn32_train_time_sc.txt b/benchmark_vision/8_class/cnn32_train_time_sc.txt similarity index 100% rename from vision/8_class/cnn32_train_time_sc.txt rename to benchmark_vision/8_class/cnn32_train_time_sc.txt diff --git a/vision/8_class/cnn32_train_time_st.txt b/benchmark_vision/8_class/cnn32_train_time_st.txt similarity index 100% rename from vision/8_class/cnn32_train_time_st.txt rename to benchmark_vision/8_class/cnn32_train_time_st.txt diff --git a/vision/8_class/naive_rf.txt b/benchmark_vision/8_class/naive_rf.txt similarity index 100% rename from vision/8_class/naive_rf.txt rename to benchmark_vision/8_class/naive_rf.txt diff --git a/vision/8_class/naive_rf_test_time.txt b/benchmark_vision/8_class/naive_rf_test_time.txt similarity index 100% rename from vision/8_class/naive_rf_test_time.txt rename to benchmark_vision/8_class/naive_rf_test_time.txt diff --git a/vision/8_class/naive_rf_test_time_lc.txt b/benchmark_vision/8_class/naive_rf_test_time_lc.txt similarity index 100% rename from vision/8_class/naive_rf_test_time_lc.txt rename to benchmark_vision/8_class/naive_rf_test_time_lc.txt diff --git a/vision/8_class/naive_rf_train_time.txt b/benchmark_vision/8_class/naive_rf_train_time.txt similarity index 100% rename from vision/8_class/naive_rf_train_time.txt rename to benchmark_vision/8_class/naive_rf_train_time.txt diff --git a/vision/8_class/naive_rf_train_time_lc.txt b/benchmark_vision/8_class/naive_rf_train_time_lc.txt similarity index 100% rename from vision/8_class/naive_rf_train_time_lc.txt rename to benchmark_vision/8_class/naive_rf_train_time_lc.txt diff --git a/vision/8_class/resnet18.txt b/benchmark_vision/8_class/resnet18.txt similarity index 100% rename from vision/8_class/resnet18.txt rename to benchmark_vision/8_class/resnet18.txt diff --git a/vision/8_class/resnet18_sc.txt b/benchmark_vision/8_class/resnet18_sc.txt similarity index 100% rename from vision/8_class/resnet18_sc.txt rename to benchmark_vision/8_class/resnet18_sc.txt diff --git a/vision/8_class/resnet18_st.txt b/benchmark_vision/8_class/resnet18_st.txt similarity index 100% rename from vision/8_class/resnet18_st.txt rename to benchmark_vision/8_class/resnet18_st.txt diff --git a/vision/8_class/resnet18_test_time.txt b/benchmark_vision/8_class/resnet18_test_time.txt similarity index 100% rename from vision/8_class/resnet18_test_time.txt rename to benchmark_vision/8_class/resnet18_test_time.txt diff --git a/vision/8_class/resnet18_test_time_sc.txt b/benchmark_vision/8_class/resnet18_test_time_sc.txt similarity index 100% rename from vision/8_class/resnet18_test_time_sc.txt rename to benchmark_vision/8_class/resnet18_test_time_sc.txt diff --git a/vision/8_class/resnet18_test_time_st.txt b/benchmark_vision/8_class/resnet18_test_time_st.txt similarity index 100% rename from vision/8_class/resnet18_test_time_st.txt rename to benchmark_vision/8_class/resnet18_test_time_st.txt diff --git a/vision/8_class/resnet18_train_time.txt b/benchmark_vision/8_class/resnet18_train_time.txt similarity index 100% rename from vision/8_class/resnet18_train_time.txt rename to benchmark_vision/8_class/resnet18_train_time.txt diff --git a/vision/8_class/resnet18_train_time_sc.txt b/benchmark_vision/8_class/resnet18_train_time_sc.txt similarity index 100% rename from vision/8_class/resnet18_train_time_sc.txt rename to benchmark_vision/8_class/resnet18_train_time_sc.txt diff --git a/vision/8_class/resnet18_train_time_st.txt b/benchmark_vision/8_class/resnet18_train_time_st.txt similarity index 100% rename from vision/8_class/resnet18_train_time_st.txt rename to benchmark_vision/8_class/resnet18_train_time_st.txt diff --git a/vision/8_class/svhn_cnn32.txt b/benchmark_vision/8_class/svhn_cnn32.txt similarity index 100% rename from vision/8_class/svhn_cnn32.txt rename to benchmark_vision/8_class/svhn_cnn32.txt diff --git a/vision/8_class/svhn_cnn32_2l.txt b/benchmark_vision/8_class/svhn_cnn32_2l.txt similarity index 100% rename from vision/8_class/svhn_cnn32_2l.txt rename to benchmark_vision/8_class/svhn_cnn32_2l.txt diff --git a/vision/8_class/svhn_cnn32_2l_test_time.txt b/benchmark_vision/8_class/svhn_cnn32_2l_test_time.txt similarity index 100% rename from vision/8_class/svhn_cnn32_2l_test_time.txt rename to benchmark_vision/8_class/svhn_cnn32_2l_test_time.txt diff --git a/vision/8_class/svhn_cnn32_2l_train_time.txt b/benchmark_vision/8_class/svhn_cnn32_2l_train_time.txt similarity index 100% rename from vision/8_class/svhn_cnn32_2l_train_time.txt rename to benchmark_vision/8_class/svhn_cnn32_2l_train_time.txt diff --git a/vision/8_class/svhn_cnn32_5l.txt b/benchmark_vision/8_class/svhn_cnn32_5l.txt similarity index 100% rename from vision/8_class/svhn_cnn32_5l.txt rename to benchmark_vision/8_class/svhn_cnn32_5l.txt diff --git a/vision/8_class/svhn_cnn32_5l_test_time.txt b/benchmark_vision/8_class/svhn_cnn32_5l_test_time.txt similarity index 100% rename from vision/8_class/svhn_cnn32_5l_test_time.txt rename to benchmark_vision/8_class/svhn_cnn32_5l_test_time.txt diff --git a/vision/8_class/svhn_cnn32_5l_train_time.txt b/benchmark_vision/8_class/svhn_cnn32_5l_train_time.txt similarity index 100% rename from vision/8_class/svhn_cnn32_5l_train_time.txt rename to benchmark_vision/8_class/svhn_cnn32_5l_train_time.txt diff --git a/vision/8_class/svhn_cnn32_test_time.txt b/benchmark_vision/8_class/svhn_cnn32_test_time.txt similarity index 100% rename from vision/8_class/svhn_cnn32_test_time.txt rename to benchmark_vision/8_class/svhn_cnn32_test_time.txt diff --git a/vision/8_class/svhn_cnn32_train_time.txt b/benchmark_vision/8_class/svhn_cnn32_train_time.txt similarity index 100% rename from vision/8_class/svhn_cnn32_train_time.txt rename to benchmark_vision/8_class/svhn_cnn32_train_time.txt diff --git a/vision/8_class/svhn_naive_rf.txt b/benchmark_vision/8_class/svhn_naive_rf.txt similarity index 100% rename from vision/8_class/svhn_naive_rf.txt rename to benchmark_vision/8_class/svhn_naive_rf.txt diff --git a/vision/8_class/svhn_naive_rf_test_time.txt b/benchmark_vision/8_class/svhn_naive_rf_test_time.txt similarity index 100% rename from vision/8_class/svhn_naive_rf_test_time.txt rename to benchmark_vision/8_class/svhn_naive_rf_test_time.txt diff --git a/vision/8_class/svhn_naive_rf_train_time.txt b/benchmark_vision/8_class/svhn_naive_rf_train_time.txt similarity index 100% rename from vision/8_class/svhn_naive_rf_train_time.txt rename to benchmark_vision/8_class/svhn_naive_rf_train_time.txt diff --git a/vision/8_class/svhn_resnet18.txt b/benchmark_vision/8_class/svhn_resnet18.txt similarity index 100% rename from vision/8_class/svhn_resnet18.txt rename to benchmark_vision/8_class/svhn_resnet18.txt diff --git a/vision/8_class/svhn_resnet18_test_time.txt b/benchmark_vision/8_class/svhn_resnet18_test_time.txt similarity index 100% rename from vision/8_class/svhn_resnet18_test_time.txt rename to benchmark_vision/8_class/svhn_resnet18_test_time.txt diff --git a/vision/8_class/svhn_resnet18_train_time.txt b/benchmark_vision/8_class/svhn_resnet18_train_time.txt similarity index 100% rename from vision/8_class/svhn_resnet18_train_time.txt rename to benchmark_vision/8_class/svhn_resnet18_train_time.txt diff --git a/vision/8_class/svhn_svm.txt b/benchmark_vision/8_class/svhn_svm.txt similarity index 100% rename from vision/8_class/svhn_svm.txt rename to benchmark_vision/8_class/svhn_svm.txt diff --git a/vision/8_class/svhn_svm_test_time.txt b/benchmark_vision/8_class/svhn_svm_test_time.txt similarity index 100% rename from vision/8_class/svhn_svm_test_time.txt rename to benchmark_vision/8_class/svhn_svm_test_time.txt diff --git a/vision/8_class/svhn_svm_train_time.txt b/benchmark_vision/8_class/svhn_svm_train_time.txt similarity index 100% rename from vision/8_class/svhn_svm_train_time.txt rename to benchmark_vision/8_class/svhn_svm_train_time.txt diff --git a/vision/8_class/svm.txt b/benchmark_vision/8_class/svm.txt similarity index 100% rename from vision/8_class/svm.txt rename to benchmark_vision/8_class/svm.txt diff --git a/vision/8_class/svm_test_time.txt b/benchmark_vision/8_class/svm_test_time.txt similarity index 100% rename from vision/8_class/svm_test_time.txt rename to benchmark_vision/8_class/svm_test_time.txt diff --git a/vision/8_class/svm_test_time_lc.txt b/benchmark_vision/8_class/svm_test_time_lc.txt similarity index 100% rename from vision/8_class/svm_test_time_lc.txt rename to benchmark_vision/8_class/svm_test_time_lc.txt diff --git a/vision/8_class/svm_train_time.txt b/benchmark_vision/8_class/svm_train_time.txt similarity index 100% rename from vision/8_class/svm_train_time.txt rename to benchmark_vision/8_class/svm_train_time.txt diff --git a/vision/8_class/svm_train_time_lc.txt b/benchmark_vision/8_class/svm_train_time_lc.txt similarity index 100% rename from vision/8_class/svm_train_time_lc.txt rename to benchmark_vision/8_class/svm_train_time_lc.txt diff --git a/vision/90_class/cnn32.txt b/benchmark_vision/90_class/cnn32.txt similarity index 100% rename from vision/90_class/cnn32.txt rename to benchmark_vision/90_class/cnn32.txt diff --git a/vision/90_class/cnn32_2l.txt b/benchmark_vision/90_class/cnn32_2l.txt similarity index 100% rename from vision/90_class/cnn32_2l.txt rename to benchmark_vision/90_class/cnn32_2l.txt diff --git a/vision/90_class/cnn32_2l_sc.txt b/benchmark_vision/90_class/cnn32_2l_sc.txt similarity index 100% rename from vision/90_class/cnn32_2l_sc.txt rename to benchmark_vision/90_class/cnn32_2l_sc.txt diff --git a/vision/90_class/cnn32_2l_st.txt b/benchmark_vision/90_class/cnn32_2l_st.txt similarity index 100% rename from vision/90_class/cnn32_2l_st.txt rename to benchmark_vision/90_class/cnn32_2l_st.txt diff --git a/vision/90_class/cnn32_2l_test_time.txt b/benchmark_vision/90_class/cnn32_2l_test_time.txt similarity index 100% rename from vision/90_class/cnn32_2l_test_time.txt rename to benchmark_vision/90_class/cnn32_2l_test_time.txt diff --git a/vision/90_class/cnn32_2l_test_time_sc.txt b/benchmark_vision/90_class/cnn32_2l_test_time_sc.txt similarity index 100% rename from vision/90_class/cnn32_2l_test_time_sc.txt rename to benchmark_vision/90_class/cnn32_2l_test_time_sc.txt diff --git a/vision/90_class/cnn32_2l_test_time_st.txt b/benchmark_vision/90_class/cnn32_2l_test_time_st.txt similarity index 100% rename from vision/90_class/cnn32_2l_test_time_st.txt rename to benchmark_vision/90_class/cnn32_2l_test_time_st.txt diff --git a/vision/90_class/cnn32_2l_train_time.txt b/benchmark_vision/90_class/cnn32_2l_train_time.txt similarity index 100% rename from vision/90_class/cnn32_2l_train_time.txt rename to benchmark_vision/90_class/cnn32_2l_train_time.txt diff --git a/vision/90_class/cnn32_2l_train_time_sc.txt b/benchmark_vision/90_class/cnn32_2l_train_time_sc.txt similarity index 100% rename from vision/90_class/cnn32_2l_train_time_sc.txt rename to benchmark_vision/90_class/cnn32_2l_train_time_sc.txt diff --git a/vision/90_class/cnn32_2l_train_time_st.txt b/benchmark_vision/90_class/cnn32_2l_train_time_st.txt similarity index 100% rename from vision/90_class/cnn32_2l_train_time_st.txt rename to benchmark_vision/90_class/cnn32_2l_train_time_st.txt diff --git a/vision/90_class/cnn32_5l.txt b/benchmark_vision/90_class/cnn32_5l.txt similarity index 100% rename from vision/90_class/cnn32_5l.txt rename to benchmark_vision/90_class/cnn32_5l.txt diff --git a/vision/90_class/cnn32_5l_sc.txt b/benchmark_vision/90_class/cnn32_5l_sc.txt similarity index 100% rename from vision/90_class/cnn32_5l_sc.txt rename to benchmark_vision/90_class/cnn32_5l_sc.txt diff --git a/vision/90_class/cnn32_5l_st.txt b/benchmark_vision/90_class/cnn32_5l_st.txt similarity index 100% rename from vision/90_class/cnn32_5l_st.txt rename to benchmark_vision/90_class/cnn32_5l_st.txt diff --git a/vision/90_class/cnn32_5l_test_time.txt b/benchmark_vision/90_class/cnn32_5l_test_time.txt similarity index 100% rename from vision/90_class/cnn32_5l_test_time.txt rename to benchmark_vision/90_class/cnn32_5l_test_time.txt diff --git a/vision/90_class/cnn32_5l_test_time_sc.txt b/benchmark_vision/90_class/cnn32_5l_test_time_sc.txt similarity index 100% rename from vision/90_class/cnn32_5l_test_time_sc.txt rename to benchmark_vision/90_class/cnn32_5l_test_time_sc.txt diff --git a/vision/90_class/cnn32_5l_test_time_st.txt b/benchmark_vision/90_class/cnn32_5l_test_time_st.txt similarity index 100% rename from vision/90_class/cnn32_5l_test_time_st.txt rename to benchmark_vision/90_class/cnn32_5l_test_time_st.txt diff --git a/vision/90_class/cnn32_5l_train_time.txt b/benchmark_vision/90_class/cnn32_5l_train_time.txt similarity index 100% rename from vision/90_class/cnn32_5l_train_time.txt rename to benchmark_vision/90_class/cnn32_5l_train_time.txt diff --git a/vision/90_class/cnn32_5l_train_time_sc.txt b/benchmark_vision/90_class/cnn32_5l_train_time_sc.txt similarity index 100% rename from vision/90_class/cnn32_5l_train_time_sc.txt rename to benchmark_vision/90_class/cnn32_5l_train_time_sc.txt diff --git a/vision/90_class/cnn32_5l_train_time_st.txt b/benchmark_vision/90_class/cnn32_5l_train_time_st.txt similarity index 100% rename from vision/90_class/cnn32_5l_train_time_st.txt rename to benchmark_vision/90_class/cnn32_5l_train_time_st.txt diff --git a/vision/90_class/cnn32_sc.txt b/benchmark_vision/90_class/cnn32_sc.txt similarity index 100% rename from vision/90_class/cnn32_sc.txt rename to benchmark_vision/90_class/cnn32_sc.txt diff --git a/vision/90_class/cnn32_st.txt b/benchmark_vision/90_class/cnn32_st.txt similarity index 100% rename from vision/90_class/cnn32_st.txt rename to benchmark_vision/90_class/cnn32_st.txt diff --git a/vision/90_class/cnn32_test_time.txt b/benchmark_vision/90_class/cnn32_test_time.txt similarity index 100% rename from vision/90_class/cnn32_test_time.txt rename to benchmark_vision/90_class/cnn32_test_time.txt diff --git a/vision/90_class/cnn32_test_time_sc.txt b/benchmark_vision/90_class/cnn32_test_time_sc.txt similarity index 100% rename from vision/90_class/cnn32_test_time_sc.txt rename to benchmark_vision/90_class/cnn32_test_time_sc.txt diff --git a/vision/90_class/cnn32_test_time_st.txt b/benchmark_vision/90_class/cnn32_test_time_st.txt similarity index 100% rename from vision/90_class/cnn32_test_time_st.txt rename to benchmark_vision/90_class/cnn32_test_time_st.txt diff --git a/vision/90_class/cnn32_train_time.txt b/benchmark_vision/90_class/cnn32_train_time.txt similarity index 100% rename from vision/90_class/cnn32_train_time.txt rename to benchmark_vision/90_class/cnn32_train_time.txt diff --git a/vision/90_class/cnn32_train_time_sc.txt b/benchmark_vision/90_class/cnn32_train_time_sc.txt similarity index 100% rename from vision/90_class/cnn32_train_time_sc.txt rename to benchmark_vision/90_class/cnn32_train_time_sc.txt diff --git a/vision/90_class/cnn32_train_time_st.txt b/benchmark_vision/90_class/cnn32_train_time_st.txt similarity index 100% rename from vision/90_class/cnn32_train_time_st.txt rename to benchmark_vision/90_class/cnn32_train_time_st.txt diff --git a/vision/90_class/naive_rf.txt b/benchmark_vision/90_class/naive_rf.txt similarity index 100% rename from vision/90_class/naive_rf.txt rename to benchmark_vision/90_class/naive_rf.txt diff --git a/vision/90_class/naive_rf_test_time.txt b/benchmark_vision/90_class/naive_rf_test_time.txt similarity index 100% rename from vision/90_class/naive_rf_test_time.txt rename to benchmark_vision/90_class/naive_rf_test_time.txt diff --git a/vision/90_class/naive_rf_test_time_lc.txt b/benchmark_vision/90_class/naive_rf_test_time_lc.txt similarity index 100% rename from vision/90_class/naive_rf_test_time_lc.txt rename to benchmark_vision/90_class/naive_rf_test_time_lc.txt diff --git a/vision/90_class/naive_rf_train_time.txt b/benchmark_vision/90_class/naive_rf_train_time.txt similarity index 100% rename from vision/90_class/naive_rf_train_time.txt rename to benchmark_vision/90_class/naive_rf_train_time.txt diff --git a/vision/90_class/naive_rf_train_time_lc.txt b/benchmark_vision/90_class/naive_rf_train_time_lc.txt similarity index 100% rename from vision/90_class/naive_rf_train_time_lc.txt rename to benchmark_vision/90_class/naive_rf_train_time_lc.txt diff --git a/vision/90_class/resnet18.txt b/benchmark_vision/90_class/resnet18.txt similarity index 100% rename from vision/90_class/resnet18.txt rename to benchmark_vision/90_class/resnet18.txt diff --git a/vision/90_class/resnet18_sc.txt b/benchmark_vision/90_class/resnet18_sc.txt similarity index 100% rename from vision/90_class/resnet18_sc.txt rename to benchmark_vision/90_class/resnet18_sc.txt diff --git a/vision/90_class/resnet18_st.txt b/benchmark_vision/90_class/resnet18_st.txt similarity index 100% rename from vision/90_class/resnet18_st.txt rename to benchmark_vision/90_class/resnet18_st.txt diff --git a/vision/90_class/resnet18_test_time.txt b/benchmark_vision/90_class/resnet18_test_time.txt similarity index 100% rename from vision/90_class/resnet18_test_time.txt rename to benchmark_vision/90_class/resnet18_test_time.txt diff --git a/vision/90_class/resnet18_test_time_sc.txt b/benchmark_vision/90_class/resnet18_test_time_sc.txt similarity index 100% rename from vision/90_class/resnet18_test_time_sc.txt rename to benchmark_vision/90_class/resnet18_test_time_sc.txt diff --git a/vision/90_class/resnet18_test_time_st.txt b/benchmark_vision/90_class/resnet18_test_time_st.txt similarity index 100% rename from vision/90_class/resnet18_test_time_st.txt rename to benchmark_vision/90_class/resnet18_test_time_st.txt diff --git a/vision/90_class/resnet18_train_time.txt b/benchmark_vision/90_class/resnet18_train_time.txt similarity index 100% rename from vision/90_class/resnet18_train_time.txt rename to benchmark_vision/90_class/resnet18_train_time.txt diff --git a/vision/90_class/resnet18_train_time_sc.txt b/benchmark_vision/90_class/resnet18_train_time_sc.txt similarity index 100% rename from vision/90_class/resnet18_train_time_sc.txt rename to benchmark_vision/90_class/resnet18_train_time_sc.txt diff --git a/vision/90_class/resnet18_train_time_st.txt b/benchmark_vision/90_class/resnet18_train_time_st.txt similarity index 100% rename from vision/90_class/resnet18_train_time_st.txt rename to benchmark_vision/90_class/resnet18_train_time_st.txt diff --git a/vision/90_class/svm.txt b/benchmark_vision/90_class/svm.txt similarity index 100% rename from vision/90_class/svm.txt rename to benchmark_vision/90_class/svm.txt diff --git a/vision/90_class/svm_test_time.txt b/benchmark_vision/90_class/svm_test_time.txt similarity index 100% rename from vision/90_class/svm_test_time.txt rename to benchmark_vision/90_class/svm_test_time.txt diff --git a/vision/90_class/svm_test_time_lc.txt b/benchmark_vision/90_class/svm_test_time_lc.txt similarity index 100% rename from vision/90_class/svm_test_time_lc.txt rename to benchmark_vision/90_class/svm_test_time_lc.txt diff --git a/vision/90_class/svm_train_time.txt b/benchmark_vision/90_class/svm_train_time.txt similarity index 100% rename from vision/90_class/svm_train_time.txt rename to benchmark_vision/90_class/svm_train_time.txt diff --git a/vision/90_class/svm_train_time_lc.txt b/benchmark_vision/90_class/svm_train_time_lc.txt similarity index 100% rename from vision/90_class/svm_train_time_lc.txt rename to benchmark_vision/90_class/svm_train_time_lc.txt diff --git a/vision/cifar_10.py b/benchmark_vision/cifar_10.py similarity index 100% rename from vision/cifar_10.py rename to benchmark_vision/cifar_10.py diff --git a/vision/cifar_100.py b/benchmark_vision/cifar_100.py similarity index 100% rename from vision/cifar_100.py rename to benchmark_vision/cifar_100.py diff --git a/vision/cifar_100_stc.py b/benchmark_vision/cifar_100_stc.py similarity index 100% rename from vision/cifar_100_stc.py rename to benchmark_vision/cifar_100_stc.py diff --git a/vision/cifar_10_stc.py b/benchmark_vision/cifar_10_stc.py similarity index 100% rename from vision/cifar_10_stc.py rename to benchmark_vision/cifar_10_stc.py diff --git a/vision/cifar_figures.ipynb b/benchmark_vision/cifar_figures.ipynb similarity index 100% rename from vision/cifar_figures.ipynb rename to benchmark_vision/cifar_figures.ipynb diff --git a/vision/svhn.py b/benchmark_vision/svhn.py similarity index 100% rename from vision/svhn.py rename to benchmark_vision/svhn.py diff --git a/vision/svhn_figure.ipynb b/benchmark_vision/svhn_figure.ipynb similarity index 100% rename from vision/svhn_figure.ipynb rename to benchmark_vision/svhn_figure.ipynb diff --git a/vision/svhn_toolbox.py b/benchmark_vision/svhn_toolbox.py similarity index 100% rename from vision/svhn_toolbox.py rename to benchmark_vision/svhn_toolbox.py diff --git a/vision/toolbox.py b/benchmark_vision/toolbox.py similarity index 100% rename from vision/toolbox.py rename to benchmark_vision/toolbox.py From 1dd00a93e869cb2900a4803177be6678b97c9484 Mon Sep 17 00:00:00 2001 From: Haoyin Xu Date: Thu, 23 Sep 2021 11:57:01 -0400 Subject: [PATCH 5/5] MAINT update code co-authors Co-Authored-By: Jayanta Dey <52499217+jdey4@users.noreply.github.com> --- benchmark_audition/audio_toolbox.py | 4 ++-- benchmark_audition/fsdd.py | 4 ++-- benchmark_tabular/cc18_kappa_ece.py | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/benchmark_audition/audio_toolbox.py b/benchmark_audition/audio_toolbox.py index ae658cfb..19613fc6 100644 --- a/benchmark_audition/audio_toolbox.py +++ b/benchmark_audition/audio_toolbox.py @@ -1,6 +1,6 @@ """ -Coauthors: Yu-Chung Peng - Haoyin Xu +Coauthors: Haoyin Xu + Yu-Chung Peng Madi Kusmanov """ import numpy as np diff --git a/benchmark_audition/fsdd.py b/benchmark_audition/fsdd.py index c836fb6d..b8bb9b57 100644 --- a/benchmark_audition/fsdd.py +++ b/benchmark_audition/fsdd.py @@ -1,6 +1,6 @@ """ -Coauthors: Yu-Chung Peng - Haoyin Xu +Coauthors: Haoyin Xu + Yu-Chung Peng Madi Kusmanov """ from audio_toolbox import * diff --git a/benchmark_tabular/cc18_kappa_ece.py b/benchmark_tabular/cc18_kappa_ece.py index 183110b1..68c40395 100644 --- a/benchmark_tabular/cc18_kappa_ece.py +++ b/benchmark_tabular/cc18_kappa_ece.py @@ -1,5 +1,6 @@ """ -Author: Michael Ainsworth +Coauthors: Michael Ainsworth + Jayanta Dey """ import numpy as np