diff --git a/README.md b/README.md index 590fb217b..7540db7ec 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ CodaLab is an open-source web-based platform that enables researchers, developer To see Codalab Competition's in action, visit [codalab.lisn.fr](https://codalab.lisn.fr/). +_[Codabench](https://github.com/codalab/codabench), the next-gen of CodaLab Competitions, is out. [Try it out](https://www.codabench.org/)!_ + ## Documentation - [CodaLab Wiki](https://github.com/codalab/codalab/wiki) diff --git a/codalab/apps/web/models.py b/codalab/apps/web/models.py index 30ceb040f..607d8a38a 100644 --- a/codalab/apps/web/models.py +++ b/codalab/apps/web/models.py @@ -1238,9 +1238,13 @@ def sortkey(x): for result in results: for sdef in result['scoredefs']: if sdef.computed: - operation = getattr(models, sdef.computed_score.operation) + try: + operation = getattr(models, sdef.computed_score.operation) + operation_name = operation.name + except: + operation_name = sdef.computed_score.operation weights = sdef.computed_score.weights - if (operation.name == 'Avg'): + if (operation_name == 'Avg'): try: cnt = len(computed_deps[sdef.id]) if (cnt > 0): @@ -1263,6 +1267,18 @@ def sortkey(x): ranks[sdef.id] = self.rank_values(submission_ids, computed_values, sort_ascending=sdef.sorting=='asc') except KeyError: pass + elif (operation_name == 'MRR'): + try: + cnt = len(computed_deps[sdef.id]) + if (cnt > 0): + computed_values = {} + for id in submission_ids: + # Mean reciprocal rank computation + computed_values[id] = sum([1.0/ranks[d.id][id] for d in computed_deps[sdef.id]]) / float(cnt) + values[sdef.id] = computed_values + ranks[sdef.id] = self.rank_values(submission_ids, computed_values, sort_ascending=sdef.sorting=='asc') + except KeyError: + pass # format values for result in results: @@ -2398,7 +2414,8 @@ def save(self, *args, **kwargs): class SubmissionComputedScore(models.Model): scoredef = models.OneToOneField(SubmissionScoreDef, related_name='computed_score', on_delete=models.CASCADE) operation = models.CharField(max_length=10, choices=(('Max', 'Max'), - ('Avg', 'Average'))) + ('Avg', 'Average'), + ('MRR', 'MRR'))) weights = models.CharField(max_length=200, null=True, blank=True)