feat: add more logs to wandb

commons/orm.py

@@ -521,3 +521,46 @@ async def get_validator_score() -> torch.Tensor | None:
             return None
 
         return torch.tensor(json.loads(score_record.score))
+
+    @staticmethod
+    async def get_completion_scores_and_models_by_dojo_task_id(
+        dojo_task_id: str,
+    ) -> dict[str, float | None]:
+        """
+        Fetch the scores and model IDs from Completion_Response_Model for a given Dojo task ID.
+
+        Args:
+            dojo_task_id (str): The Dojo task ID to search for.
+
+        Returns:
+            dict[str, Optional[float]]: A dictionary mapping model ID to score (which may be None) for the given Dojo task ID.
+        """
+        try:
+            # First, find the Feedback_Request_Model with the given dojo_task_id
+            feedback_request = await Feedback_Request_Model.prisma().find_first(
+                where=Feedback_Request_ModelWhereInput(dojo_task_id=dojo_task_id),
+                include={"completions": True},
+            )
+
+            if not feedback_request:
+                logger.warning(
+                    f"No Feedback_Request_Model found for dojo_task_id: {dojo_task_id}"
+                )
+                return {}
+
+            # Extract scores from the completions
+            scores_and_models = {
+                completion.model: completion.score
+                for completion in feedback_request.completions
+            }
+
+            logger.debug(
+                f"Found {len(scores_and_models)} scores and models for dojo_task_id: {dojo_task_id}"
+            )
+            return scores_and_models
+
+        except Exception as e:
+            logger.error(
+                f"Error fetching completion scores and models for dojo_task_id {dojo_task_id}: {e}"
+            )
+            return {}

neurons/validator.py

@@ -214,16 +214,35 @@ async def log_wandb():
                                 f"📝 Mean miner scores across different criteria: consensus shape:{mean_weighted_consensus_scores}, gt shape:{mean_weighted_gt_scores}"
                             )
 
+                            ground_truth_scores = [
+                                miner_scores.ground_truth.score
+                                for miner_scores in criteria_to_miner_score.values()
+                            ]
+
                             score_data = {}
                             # update the scores based on the rewards
                             score_data["scores_by_hotkey"] = hotkey_to_score
                             score_data["mean"] = {
                                 "consensus": mean_weighted_consensus_scores,
                                 "ground_truth": mean_weighted_gt_scores,
+                                "raw_ground_truth_scores": ground_truth_scores,
                             }
 
+                            dojo_task_scores = {}
+                            for miner_response in task.miner_responses:
+                                if miner_response.dojo_task_id is not None:
+                                    model_to_score_map = await ORM.get_completion_scores_and_models_by_dojo_task_id(
+                                        miner_response.dojo_task_id
+                                    )
+                                    dojo_task_scores[miner_response.dojo_task_id] = (
+                                        model_to_score_map
+                                    )
+
+                            score_data["dojo_task_scores"] = dojo_task_scores
+
                             wandb_data = jsonable_encoder(
                                 {
+                                    "request_id": task.request.request_id,
                                     "task": task.request.task_type,
                                     "criteria": task.request.criteria_types,
                                     "prompt": task.request.prompt,