Merge pull request #3453 from codalab/develop

Merge develop into master
codalab · Dec 8, 2023 · b397bbd · b397bbd
2 parents db749ac + 6901945
commit b397bbd
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 3 deletions.
diff --git a/README.md b/README.md
@@ -9,6 +9,8 @@ CodaLab is an open-source web-based platform that enables researchers, developer
 
 To see Codalab Competition's in action, visit [codalab.lisn.fr](https://codalab.lisn.fr/).
 
+_[Codabench](https://github.com/codalab/codabench), the next-gen of CodaLab Competitions, is out. [Try it out](https://www.codabench.org/)!_
+
 ## Documentation
 
 - [CodaLab Wiki](https://github.com/codalab/codalab/wiki)

diff --git a/codalab/apps/web/models.py b/codalab/apps/web/models.py
@@ -1238,9 +1238,13 @@ def sortkey(x):
             for result in results:
                 for sdef in result['scoredefs']:
                     if sdef.computed:
-                        operation = getattr(models, sdef.computed_score.operation)
+                        try:
+                            operation = getattr(models, sdef.computed_score.operation)
+                            operation_name = operation.name
+                        except:
+                            operation_name = sdef.computed_score.operation
                         weights = sdef.computed_score.weights
-                        if (operation.name == 'Avg'):
+                        if (operation_name == 'Avg'):
                             try:
                                 cnt = len(computed_deps[sdef.id])
                                 if (cnt > 0):
@@ -1263,6 +1267,18 @@ def sortkey(x):
                                     ranks[sdef.id] = self.rank_values(submission_ids, computed_values, sort_ascending=sdef.sorting=='asc')
                             except KeyError:
                                 pass
+                        elif (operation_name == 'MRR'):
+                            try:
+                                cnt = len(computed_deps[sdef.id])
+                                if (cnt > 0):
+                                    computed_values = {}
+                                    for id in submission_ids:
+                                        # Mean reciprocal rank computation
+                                        computed_values[id] = sum([1.0/ranks[d.id][id] for d in computed_deps[sdef.id]]) / float(cnt)
+                                    values[sdef.id] = computed_values
+                                    ranks[sdef.id] = self.rank_values(submission_ids, computed_values, sort_ascending=sdef.sorting=='asc')
+                            except KeyError:
+                                pass
 
             # format values
             for result in results:
@@ -2398,7 +2414,8 @@ def save(self, *args, **kwargs):
 class SubmissionComputedScore(models.Model):
     scoredef = models.OneToOneField(SubmissionScoreDef, related_name='computed_score', on_delete=models.CASCADE)
     operation = models.CharField(max_length=10, choices=(('Max', 'Max'),
-                                                        ('Avg', 'Average')))
+                                                         ('Avg', 'Average'),
+                                                         ('MRR', 'MRR')))
     weights = models.CharField(max_length=200, null=True, blank=True)