Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Switch from accuracy to Cohen's Kappa for vision & auditory data #36

Merged
merged 5 commits into from
Sep 23, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
26 changes: 15 additions & 11 deletions audio/audio_toolbox.py → benchmark_audition/audio_toolbox.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
"""
Coauthors: Yu-Chung Peng
Haoyin Xu
Coauthors: Haoyin Xu
Yu-Chung Peng
Madi Kusmanov
"""
import numpy as np
from sklearn.metrics import accuracy_score
from sklearn.metrics import cohen_kappa_score
import time
import torch
import os
Expand Down Expand Up @@ -206,7 +206,7 @@ def run_rf_image_set(
end_time = time.perf_counter()
test_time = end_time - start_time

return accuracy_score(test_labels, test_preds), train_time, test_time
return cohen_kappa_score(test_labels, test_preds), train_time, test_time


def run_dn_image_es(
Expand Down Expand Up @@ -236,7 +236,7 @@ def run_dn_image_es(
flag = 0
start_time = time.perf_counter()
for epoch in range(epochs): # loop over the dataset multiple times

model.train()
for i in range(0, len(train_data), batch):
# get the inputs
inputs = train_data[i : i + batch].to(dev)
Expand All @@ -251,6 +251,7 @@ def run_dn_image_es(
optimizer.step()

# test generalization error for early stopping
model.eval()
cur_loss = 0
with torch.no_grad():
for i in range(0, len(valid_data), batch):
Expand All @@ -273,22 +274,25 @@ def run_dn_image_es(
break
end_time = time.perf_counter()
train_time = end_time - start_time

# test the model
correct = torch.tensor(0).to(dev)
total = torch.tensor(0).to(dev)
model.eval()
start_time = time.perf_counter()
test_preds = []
test_labels = []
with torch.no_grad():
for i in range(0, len(test_data), batch):
inputs = test_data[i : i + batch].to(dev)
labels = test_labels[i : i + batch].to(dev)
test_labels = np.concatenate((test_labels, labels.tolist()))

outputs = model(inputs)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels.view(-1)).sum().item()
test_preds = np.concatenate((test_preds, predicted.tolist()))

end_time = time.perf_counter()
test_time = end_time - start_time
accuracy = float(correct) / float(total)
return accuracy, train_time, test_time
return cohen_kappa_score(test_preds, test_labels), train_time, test_time


def prepare_data(
Expand Down
4 changes: 2 additions & 2 deletions audio/fsdd.py → benchmark_audition/fsdd.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""
Coauthors: Yu-Chung Peng
Haoyin Xu
Coauthors: Haoyin Xu
Yu-Chung Peng
Madi Kusmanov
"""
from audio_toolbox import *
Expand Down
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""
Author: Michael Ainsworth
Coauthors: Michael Ainsworth
Jayanta Dey
"""

import numpy as np
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
153 changes: 82 additions & 71 deletions image/cifar_10.py → benchmark_vision/cifar_10.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""
Coauthors: Yu-Chung Peng
Haoyin Xu
Coauthors: Haoyin Xu
Yu-Chung Peng
"""
from toolbox import *

Expand All @@ -14,50 +14,16 @@
import torchvision.transforms as transforms


# prepare CIFAR data
def main():
# Example usage: python cifar_10.py -m 3
parser = argparse.ArgumentParser()
parser.add_argument("-m", help="class number")
args = parser.parse_args()
n_classes = int(args.m)
prefix = args.m + "_class/"

nums = list(range(10))
random.shuffle(nums)
classes_space = list(combinations_45(nums, n_classes))

# normalize
scale = np.mean(np.arange(0, 256))
normalize = lambda x: (x - scale) / scale

# train data
cifar_trainset = datasets.CIFAR10(
root="./", train=True, download=True, transform=None
)
cifar_train_images = normalize(cifar_trainset.data)
cifar_train_labels = np.array(cifar_trainset.targets)

# test data
cifar_testset = datasets.CIFAR10(
root="./", train=False, download=True, transform=None
)
cifar_test_images = normalize(cifar_testset.data)
cifar_test_labels = np.array(cifar_testset.targets)

cifar_train_images = cifar_train_images.reshape(-1, 32 * 32 * 3)
cifar_test_images = cifar_test_images.reshape(-1, 32 * 32 * 3)

def run_naive_rf():
naive_rf_acc_vs_n = list()
naive_rf_train_time = list()
naive_rf_test_time = list()
for classes in classes_space:

# accuracy vs num training samples (naive_rf)
samples_space = np.geomspace(10, 10000, num=8, dtype=int)
# cohen_kappa vs num training samples (naive_rf)
for samples in samples_space:
RF = RandomForestClassifier(n_estimators=100, n_jobs=-1)
accuracy, train_time, test_time = run_rf_image_set(
cohen_kappa, train_time, test_time = run_rf_image_set(
RF,
cifar_train_images,
cifar_train_labels,
Expand All @@ -66,7 +32,7 @@ def main():
samples,
classes,
)
naive_rf_acc_vs_n.append(accuracy)
naive_rf_acc_vs_n.append(cohen_kappa)
naive_rf_train_time.append(train_time)
naive_rf_test_time.append(test_time)

Expand All @@ -75,16 +41,17 @@ def main():
write_result(prefix + "naive_rf_train_time.txt", naive_rf_train_time)
write_result(prefix + "naive_rf_test_time.txt", naive_rf_test_time)


def run_svm():
svm_acc_vs_n = list()
svm_train_time = list()
svm_test_time = list()
for classes in classes_space:

# accuracy vs num training samples (svm)
samples_space = np.geomspace(10, 10000, num=8, dtype=int)
# cohen_kappa vs num training samples (svm)
for samples in samples_space:
SVM = SVC()
accuracy, train_time, test_time = run_rf_image_set(
cohen_kappa, train_time, test_time = run_rf_image_set(
SVM,
cifar_train_images,
cifar_train_labels,
Expand All @@ -93,7 +60,7 @@ def main():
samples,
classes,
)
svm_acc_vs_n.append(accuracy)
svm_acc_vs_n.append(cohen_kappa)
svm_train_time.append(train_time)
svm_test_time.append(test_time)

Expand All @@ -102,17 +69,14 @@ def main():
write_result(prefix + "svm_train_time.txt", svm_train_time)
write_result(prefix + "svm_test_time.txt", svm_test_time)

data_transforms = transforms.Compose(
[transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]
)

def run_cnn32():
cnn32_acc_vs_n = list()
cnn32_train_time = list()
cnn32_test_time = list()
for classes in classes_space:

# accuracy vs num training samples (cnn32)
samples_space = np.geomspace(10, 10000, num=8, dtype=int)
# cohen_kappa vs num training samples (cnn32)
for samples in samples_space:
# train data
cifar_trainset = datasets.CIFAR10(
Expand All @@ -135,13 +99,13 @@ def main():
cifar_testset,
samples,
)
accuracy, train_time, test_time = run_dn_image_es(
cohen_kappa, train_time, test_time = run_dn_image_es(
cnn32,
train_loader,
valid_loader,
test_loader,
)
cnn32_acc_vs_n.append(accuracy)
cnn32_acc_vs_n.append(cohen_kappa)
cnn32_train_time.append(train_time)
cnn32_test_time.append(test_time)

Expand All @@ -150,13 +114,14 @@ def main():
write_result(prefix + "cnn32_train_time.txt", cnn32_train_time)
write_result(prefix + "cnn32_test_time.txt", cnn32_test_time)


def run_cnn32_2l():
cnn32_2l_acc_vs_n = list()
cnn32_2l_train_time = list()
cnn32_2l_test_time = list()
for classes in classes_space:

# accuracy vs num training samples (cnn32_2l)
samples_space = np.geomspace(10, 10000, num=8, dtype=int)
# cohen_kappa vs num training samples (cnn32_2l)
for samples in samples_space:
# train data
cifar_trainset = datasets.CIFAR10(
Expand All @@ -179,13 +144,13 @@ def main():
cifar_testset,
samples,
)
accuracy, train_time, test_time = run_dn_image_es(
cohen_kappa, train_time, test_time = run_dn_image_es(
cnn32_2l,
train_loader,
valid_loader,
test_loader,
)
cnn32_2l_acc_vs_n.append(accuracy)
cnn32_2l_acc_vs_n.append(cohen_kappa)
cnn32_2l_train_time.append(train_time)
cnn32_2l_test_time.append(test_time)

Expand All @@ -194,13 +159,14 @@ def main():
write_result(prefix + "cnn32_2l_train_time.txt", cnn32_2l_train_time)
write_result(prefix + "cnn32_2l_test_time.txt", cnn32_2l_test_time)


def run_cnn32_5l():
cnn32_5l_acc_vs_n = list()
cnn32_5l_train_time = list()
cnn32_5l_test_time = list()
for classes in classes_space:

# accuracy vs num training samples (cnn32_5l)
samples_space = np.geomspace(10, 10000, num=8, dtype=int)
# cohen_kappa vs num training samples (cnn32_5l)
for samples in samples_space:
# train data
cifar_trainset = datasets.CIFAR10(
Expand All @@ -223,13 +189,13 @@ def main():
cifar_testset,
samples,
)
accuracy, train_time, test_time = run_dn_image_es(
cohen_kappa, train_time, test_time = run_dn_image_es(
cnn32_5l,
train_loader,
valid_loader,
test_loader,
)
cnn32_5l_acc_vs_n.append(accuracy)
cnn32_5l_acc_vs_n.append(cohen_kappa)
cnn32_5l_train_time.append(train_time)
cnn32_5l_test_time.append(test_time)

Expand All @@ -238,21 +204,14 @@ def main():
write_result(prefix + "cnn32_5l_train_time.txt", cnn32_5l_train_time)
write_result(prefix + "cnn32_5l_test_time.txt", cnn32_5l_test_time)

# prepare CIFAR data
data_transforms = transforms.Compose(
[
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
]
)

def run_resnet18():
resnet18_acc_vs_n = list()
resnet18_train_time = list()
resnet18_test_time = list()
for classes in classes_space:

# accuracy vs num training samples (resnet18)
samples_space = np.geomspace(10, 10000, num=8, dtype=int)
# cohen_kappa vs num training samples (resnet18)
for samples in samples_space:
# train data
cifar_trainset = datasets.CIFAR10(
Expand All @@ -277,13 +236,13 @@ def main():
cifar_testset,
samples,
)
accuracy, train_time, test_time = run_dn_image_es(
cohen_kappa, train_time, test_time = run_dn_image_es(
res,
train_loader,
valid_loader,
test_loader,
)
resnet18_acc_vs_n.append(accuracy)
resnet18_acc_vs_n.append(cohen_kappa)
resnet18_train_time.append(train_time)
resnet18_test_time.append(test_time)

Expand All @@ -295,4 +254,56 @@ def main():

if __name__ == "__main__":
torch.multiprocessing.freeze_support()
main()

# Example usage: python cifar_10.py -m 3
parser = argparse.ArgumentParser()
parser.add_argument("-m", help="class number")
args = parser.parse_args()
n_classes = int(args.m)
prefix = args.m + "_class/"
samples_space = np.geomspace(10, 10000, num=8, dtype=int)

nums = list(range(10))
random.shuffle(nums)
classes_space = list(combinations_45(nums, n_classes))

# normalize
scale = np.mean(np.arange(0, 256))
normalize = lambda x: (x - scale) / scale

# train data
cifar_trainset = datasets.CIFAR10(
root="./", train=True, download=True, transform=None
)
cifar_train_images = normalize(cifar_trainset.data)
cifar_train_labels = np.array(cifar_trainset.targets)

# test data
cifar_testset = datasets.CIFAR10(
root="./", train=False, download=True, transform=None
)
cifar_test_images = normalize(cifar_testset.data)
cifar_test_labels = np.array(cifar_testset.targets)

cifar_train_images = cifar_train_images.reshape(-1, 32 * 32 * 3)
cifar_test_images = cifar_test_images.reshape(-1, 32 * 32 * 3)

run_naive_rf()
run_svm()

data_transforms = transforms.Compose(
[transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]
)

run_cnn32()
run_cnn32_2l()
run_cnn32_5l()

data_transforms = transforms.Compose(
[
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
]
)

run_resnet18()
Loading