From 927279f8fc49ffbdb9e24d647e0c5d8139fd9149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Pav=C3=A3o?= Date: Tue, 26 Sep 2023 14:28:20 +0200 Subject: [PATCH 1/2] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 590fb217b..7540db7ec 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ CodaLab is an open-source web-based platform that enables researchers, developer To see Codalab Competition's in action, visit [codalab.lisn.fr](https://codalab.lisn.fr/). +_[Codabench](https://github.com/codalab/codabench), the next-gen of CodaLab Competitions, is out. [Try it out](https://www.codabench.org/)!_ + ## Documentation - [CodaLab Wiki](https://github.com/codalab/codalab/wiki) From f06e051a6804714042e864eb60f91342d7d43d59 Mon Sep 17 00:00:00 2001 From: Carlo Date: Wed, 29 Nov 2023 17:53:00 +0100 Subject: [PATCH 2/2] added mean reciprocal rank calculation (MRR) --- codalab/apps/web/models.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/codalab/apps/web/models.py b/codalab/apps/web/models.py index 30ceb040f..607d8a38a 100644 --- a/codalab/apps/web/models.py +++ b/codalab/apps/web/models.py @@ -1238,9 +1238,13 @@ def sortkey(x): for result in results: for sdef in result['scoredefs']: if sdef.computed: - operation = getattr(models, sdef.computed_score.operation) + try: + operation = getattr(models, sdef.computed_score.operation) + operation_name = operation.name + except: + operation_name = sdef.computed_score.operation weights = sdef.computed_score.weights - if (operation.name == 'Avg'): + if (operation_name == 'Avg'): try: cnt = len(computed_deps[sdef.id]) if (cnt > 0): @@ -1263,6 +1267,18 @@ def sortkey(x): ranks[sdef.id] = self.rank_values(submission_ids, computed_values, sort_ascending=sdef.sorting=='asc') except KeyError: pass + elif (operation_name == 'MRR'): + try: + cnt = len(computed_deps[sdef.id]) + if (cnt > 0): + computed_values = {} + for id in submission_ids: + # Mean reciprocal rank computation + computed_values[id] = sum([1.0/ranks[d.id][id] for d in computed_deps[sdef.id]]) / float(cnt) + values[sdef.id] = computed_values + ranks[sdef.id] = self.rank_values(submission_ids, computed_values, sort_ascending=sdef.sorting=='asc') + except KeyError: + pass # format values for result in results: @@ -2398,7 +2414,8 @@ def save(self, *args, **kwargs): class SubmissionComputedScore(models.Model): scoredef = models.OneToOneField(SubmissionScoreDef, related_name='computed_score', on_delete=models.CASCADE) operation = models.CharField(max_length=10, choices=(('Max', 'Max'), - ('Avg', 'Average'))) + ('Avg', 'Average'), + ('MRR', 'MRR'))) weights = models.CharField(max_length=200, null=True, blank=True)