Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: DEV-2087: Fix agreement calculation for Taxonomy with label weights #59

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

fix: DEV-2087: Fix agreement calculation for Taxonomy with label weights #59

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
50d88e5
Merge branch 'master' of https://github.com/heartexlabs/label-studio-…
KonstantinKorotaev Mar 21, 2022
3384b01
Merge branch 'master' of https://github.com/heartexlabs/label-studio-…
KonstantinKorotaev Apr 6, 2022
b9b95d7
fix: DEV-2087: Fix agreement calculation for Taxonomy with label weights
KonstantinKorotaev Apr 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion evalme/tests/test_text.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -584,4 +584,16 @@ def test_htmltags_migration_per_label():
html_tags1 = HTMLTagsEvalItem(raw_data=item_old, shape_key="hypertextlabels")
html_tags2 = HTMLTagsEvalItem(raw_data=item_new, shape_key="hypertextlabels")
assert html_tags1.intersection(html_tags2, per_label=True) == {'Title': 0.9285714285714286}
assert html_tags2.intersection(html_tags1, per_label=True) == {'Title': 0.9285714285714286}
assert html_tags2.intersection(html_tags1, per_label=True) == {'Title': 0.9285714285714286}


def test_taxonomy_control_weights():
"""
Test taxonomy with label weights
:return:
"""
label_weights = {"B": 0.5, "B_A": 0.2}
pred = intersection_taxonomy(tree_subview_1, tree_subview_2, label_config=label_config_subview, label_weights=label_weights)
assert pred == 0.2
pred_vice = intersection_taxonomy(tree_subview_2, tree_subview_1, label_config=label_config_subview, label_weights=label_weights)
assert pred_vice == 0.05
10 changes: 9 additions & 1 deletion evalme/text/text.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ def spans_iou(self, prediction, per_label=False, label_config=None, label_weight
taxonomy_gt_list.extend(TaxonomyEvalItem._transform_tree(master_tree, item_gt_tx))
for item in taxonomy_pred_list:
if item in taxonomy_gt_list:
temp += 1
temp += label_weights.get(item[-1], 1)
matches += (temp / max(len(taxonomy_gt_list), 1))
tasks += 1
return matches / max(tasks, 1)
Expand Down Expand Up @@ -287,6 +287,8 @@ def path_matches(self, prediction, per_label=False, label_weights=dict()):
def _tree(label_config):
"""
Creating Tree from label_config
Example for default config:
{'Archaea': {}, 'Bacteria': {}, 'Eukarya': {'Human': {}, 'Oppossum': {}, 'Extraterrestial': {}}}
"""

def recursive_lookup(d, k='Taxonomy'):
Expand Down Expand Up @@ -366,6 +368,12 @@ def paths(self):

@staticmethod
def _compare_list(gt, pred):
"""
Compare taxonomy path in depth
:param gt: ground truth value
:param pred: predicted value
:return: score [0..1]
"""
score = 0
for p, g in zip(pred, gt):
if p == g:
Expand Down