-
Notifications
You must be signed in to change notification settings - Fork 0
/
summarise_all_results_OPT.py
112 lines (82 loc) · 5.28 KB
/
summarise_all_results_OPT.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
import pandas as pd
model_name="OPT6B"
hyper="top5_replace0.3_max5000_batch8"
def get_one_line_for_one_FA(model_name, FA_name,ratio_list):
eva_output_dir=f"evaluation_results/analogies/{model_name}_{FA_name}"
if FA_name == 'ours': eva_output_dir=f"evaluation_results/analogies/{model_name}_{FA_name}/{hyper}"
print(f"==>> eva_output_dir: {eva_output_dir}")
suff_list = [FA_name.replace('_', ' ').title()]
comp_list = [FA_name.replace('_', ' ').title()]
suff_mean = 0
comp_mean = 0
random_suff_list = [FA_name.replace('_', ' ').title()]
random_comp_list = [FA_name.replace('_', ' ').title()]
random_suff_mean = 0
random_comp_mean = 0
diff_ratio_len = int(len(ratio_list)-1)
for ratio in ratio_list:
faithful_results = pd.read_csv(eva_output_dir+f'/mean_{ratio}.csv')
if 0 < ratio < 1: # 0.05 - 0.3
suff_list.append(faithful_results['suff'][0])
comp_list.append(faithful_results['comp'][0])
suff_mean += faithful_results['suff'][0]
comp_mean += faithful_results['comp'][0]
random_suff_list.append(faithful_results['random_suff'][0])
random_comp_list.append(faithful_results['random_comp'][0])
random_suff_mean += faithful_results['random_suff'][0]
random_comp_mean += faithful_results['random_comp'][0]
else:
# mean
suff_list.append(suff_mean/diff_ratio_len)
comp_list.append(comp_mean/diff_ratio_len)
random_suff_list.append(random_suff_mean/diff_ratio_len)
random_comp_list.append(random_comp_mean/diff_ratio_len)
# soft
suff_list.append(faithful_results['suff'][0])
comp_list.append(faithful_results['comp'][0])
random_suff_list.append(faithful_results['random_suff'][0])
random_comp_list.append(faithful_results['random_comp'][0])
return suff_list, comp_list, random_suff_list, random_comp_list
ratio_list = [0.05, 0.1, 0.2, 0.3, 1.0] # 0 here for flexible len from greedy search and 1 for soft
rollout_suff, rollout_comp, random_rollout_suff, random_rollout_comp = get_one_line_for_one_FA(model_name, "rollout_attention", ratio_list)
last_suff, last_comp, random_last_suff, random_last_comp = get_one_line_for_one_FA(model_name, "last_attention", ratio_list)
all_suff, all_comp, random_all_suff, random_all_comp = get_one_line_for_one_FA(model_name, "all_attention", ratio_list)
norms_suff, norms_comp, random_norms_suff, random_norms_comp = get_one_line_for_one_FA(model_name, "inseq_ig", ratio_list)
signed_suff, signed_comp, random_signed_suff, random_signed_comp = get_one_line_for_one_FA(model_name, "inseq_EmbGrad", ratio_list)
integrated_suff, integrated_comp, random_integrated_suff, random_integrated_comp = get_one_line_for_one_FA(model_name, "inseq_signed", ratio_list)
ours_suff, ours_comp, ours_random_all_suff, ours_random_all_comp = get_one_line_for_one_FA(model_name, "ours", ratio_list)
suff_df = pd.DataFrame([norms_suff, signed_suff, integrated_suff, rollout_suff, last_suff, all_suff, ours_suff], columns=['Method','5% Suff', '10% Suff', '20% Suff', '30% Suff', 'Mean Suff', 'Soft Suff'])
comp_df = pd.DataFrame([norms_comp, signed_comp, integrated_comp, rollout_comp, last_comp, all_comp, ours_comp], columns=['Method','5% Comp', '10% Comp', '20% Comp', '30% Comp', 'Mean Comp', 'Soft Comp'])
#print(f"comp_df ==>> {comp_df}")
# suff_df = pd.DataFrame([rollout_suff, last_suff, all_suff, ours_suff], columns=['Method','fix len Suff'])
# comp_df = pd.DataFrame([rollout_comp, last_comp, all_comp, ours_comp], columns=['Method','fix len Comp'])
random_suff_df = pd.DataFrame([random_norms_suff, random_signed_suff, random_integrated_suff, random_rollout_suff, random_last_suff, random_all_suff, ours_random_all_suff], columns=['Method','5% Suff', '10% Suff', '20% Suff', '30% Suff', 'Mean Suff', 'Soft Suff'])
random_comp_df = pd.DataFrame([random_norms_comp, random_signed_comp, random_integrated_comp, random_rollout_comp, random_last_comp, random_all_comp, ours_random_all_comp], columns=['Method','5% Comp', '10% Comp', '20% Comp', '30% Comp', 'Mean Comp', 'Soft Comp'])
#print(f"random_comp_df ==>> {random_comp_df}")
print(' ========== SUFF =========')
print(suff_df)
print(' ========== RANDOM SUFF =========')
print(random_suff_df)
# print(' ')
# print(' ')
# print(' ========== Comp =========')
# print(comp_df)
# print(' ========== RANDOM Comp =========')
# print(random_comp_df)
# print(' ')
# print(' ')
def div_and_save(suff_df, random_suff_df, save_name):
final_suff_df = suff_df.copy()
for col in suff_df.columns:
for row in suff_df.index:
if isinstance(suff_df.at[row, col], float) and isinstance(random_suff_df.at[row, col], float):
final_suff_df.at[row, col] = suff_df.at[row, col] / random_suff_df.at[row, col]
print(' =======> final divided results =======')
print(final_suff_df)
final_suff_df.to_csv(f'evaluation_results/summary/{model_name}/{save_name}_{hyper}.csv')
return final_suff_df
final_suff_df = div_and_save(suff_df, random_suff_df, 'suff')
final_comp_df = div_and_save(comp_df, random_comp_df, 'comp')
stacked_df = pd.concat([final_suff_df, final_comp_df])
stacked_df.to_csv(f'evaluation_results/summary/{model_name}/{hyper}.csv')
print(' Done')