Add missing grade_cells before autograding (#1770)

nbgrader/preprocessors/overwritecells.py

@@ -3,14 +3,84 @@
 from .. import utils
 from ..api import Gradebook, MissingEntry
 from . import NbGraderPreprocessor
+from ..nbgraderformat import MetadataValidator
 from nbconvert.exporters.exporter import ResourcesDict
 from nbformat.notebooknode import NotebookNode
+from traitlets import Bool, Unicode
 from typing import Tuple, Any
+from textwrap import dedent
 
 
 class OverwriteCells(NbGraderPreprocessor):
  """A preprocessor to overwrite information about grade and solution cells."""
 
+ add_missing_cells = Bool(
+ False,
+ help=dedent(
+ """
+ Whether or not missing grade_cells should be added back
+ to the notebooks being graded.
+ """
+ ),
+ ).tag(config=True)
+
+ missing_cell_notification = Unicode(
+ "This cell (id:{cell_id}) was missing from the submission. " +
+ "It was added back by nbgrader.\n\n", # Markdown requires two newlines
+ help=dedent(
+ """
+ A text to add at the beginning of every missing cell re-added to the notebook during autograding.
+ """
+ )
+ ).tag(config=True)
+
+ def missing_cell_transform(self, source_cell, max_score, is_solution=False, is_task=False):
+ """
+ Converts source_cell obtained from Gradebook into a cell that can be added to the notebook.
+ It is assumed that the cell is a grade_cell (unless is_task=True)
+ """
+
+ missing_cell_notification = self.missing_cell_notification.format(cell_id=source_cell.name)
+
+ cell = {
+ "cell_type": source_cell.cell_type,
+ "metadata": {
+ "deletable": False,
+ "editable": False,
+ "nbgrader": {
+ "grade": True,
+ "grade_id": source_cell.name,
+ "locked": source_cell.locked,
+ "checksum": source_cell.checksum,
+ "cell_type": source_cell.cell_type,
+ "points": max_score,
+ "solution": False
+ }
+ },
+ "source": missing_cell_notification + source_cell.source
+ }
+
+ # Code cell format is slightly different
+ if cell["cell_type"] == "code":
+ cell["execution_count"] = None
+ cell["outputs"] = []
+ cell["source"] = "# " + cell["source"] # make the notification we add a comment
+
+ # some grade cells are editable (manually graded answers)
+ if is_solution:
+ del cell["metadata"]["editable"]
+ cell["metadata"]["nbgrader"]["solution"] = True
+ # task cells are also a bit different
+ elif is_task:
+ cell["metadata"]["nbgrader"]["grade"] = False
+ cell["metadata"]["nbgrader"]["task"] = True
+ # this is when task cells were added so metadata validation should start from here
+ cell["metadata"]["nbgrader"]["schema_version"] = 3
+
+ cell = NotebookNode(cell)
+ cell = MetadataValidator().upgrade_cell_metadata(cell)
+ return cell
+
  def preprocess(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]:
  # pull information from the resources
  self.notebook_id = resources['nbgrader']['notebook']
@@ -22,6 +92,77 @@ def preprocess(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[Notebo
 
  with self.gradebook:
  nb, resources = super(OverwriteCells, self).preprocess(nb, resources)
+ if self.add_missing_cells:
+ nb, resources = self.add_missing_grade_cells(nb, resources)
+ nb, resources = self.add_missing_task_cells(nb, resources)
+
+ return nb, resources
+
+ def add_missing_grade_cells(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]:
+ """
+ Add missing grade cells back to the notebook.
+ If missing, find the previous solution/grade cell, and add the current cell after it.
+ It is assumed such a cell exists because
+ presumably the grade_cell exists to grade some work in the solution cell.
+ """
+ source_nb = self.gradebook.find_notebook(self.notebook_id, self.assignment_id)
+ source_cells = source_nb.source_cells
+ source_cell_ids = [cell.name for cell in source_cells]
+ grade_cells = {cell.name: cell for cell in source_nb.grade_cells}
+ solution_cell_ids = [cell.name for cell in source_nb.solution_cells]
+
+ # track indices of solution and grade cells in the submitted notebook
+ submitted_cell_idxs = dict()
+ for idx, cell in enumerate(nb.cells):
+ if utils.is_grade(cell) or utils.is_solution(cell):
+ submitted_cell_idxs[cell.metadata.nbgrader["grade_id"]] = idx
+
+ # Every time we add a cell, the idxs above get shifted
+ # We could process the notebook backwards, but that makes adding the cells in the right place more difficult
+ # So we keep track of how many we have added so far
+ added_count = 0
+
+ for grade_cell_id, grade_cell in grade_cells.items():
+ # If missing, find the previous solution/grade cell, and add the current cell after it.
+ if grade_cell_id not in submitted_cell_idxs:
+ self.log.warning(f"Missing grade cell {grade_cell_id} encountered, adding to notebook")
+ source_cell_idx = source_cell_ids.index(grade_cell_id)
+ cell_to_add = source_cells[source_cell_idx]
+ cell_to_add = self.missing_cell_transform(cell_to_add, grade_cell.max_score,
+ is_solution=grade_cell_id in solution_cell_ids)
+ # First cell was deleted, add it to start
+ if source_cell_idx == 0:
+ nb.cells.insert(0, cell_to_add)
+ submitted_cell_idxs[grade_cell_id] = 0
+ # Deleted cell is not the first, add it after the previous solution/grade cell
+ else:
+ prev_cell_id = source_cell_ids[source_cell_idx - 1]
+ prev_cell_idx = submitted_cell_idxs[prev_cell_id] + added_count
+ nb.cells.insert(prev_cell_idx + 1, cell_to_add) # +1 to add it after
+ submitted_cell_idxs[grade_cell_id] = submitted_cell_idxs[prev_cell_id]
+
+ # If the cell we just added is followed by other missing cells, we need to know its index in the nb
+ # However, no need to add `added_count` to avoid double-counting
+
+ added_count += 1 # shift idxs
+
+ return nb, resources
+
+ def add_missing_task_cells(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]:
+ """
+ Add missing task cells back to the notebook.
+ We can't figure out their original location, so they are added at the end, in their original order.
+ """
+ source_nb = self.gradebook.find_notebook(self.notebook_id, self.assignment_id)
+ source_cells = source_nb.source_cells
+ source_cell_ids = [cell.name for cell in source_cells]
+ submitted_ids = [cell["metadata"]["nbgrader"]["grade_id"] for cell in nb.cells if
+ "nbgrader" in cell["metadata"]]
+ for task_cell in source_nb.task_cells:
+ if task_cell.name not in submitted_ids:
+ cell_to_add = source_cells[source_cell_ids.index(task_cell.name)]
+ cell_to_add = self.missing_cell_transform(cell_to_add, task_cell.max_score, is_task=True)
+ nb.cells.append(cell_to_add)
 
  return nb, resources
 

nbgrader/tests/preprocessors/test_overwritecells.py

@@ -1,14 +1,14 @@
 import pytest
 
-from nbformat.v4 import new_notebook
+from nbformat.v4 import new_notebook, new_markdown_cell
 
 from ...preprocessors import SaveCells, OverwriteCells
 from ...api import Gradebook
 from ...utils import compute_checksum
 from .base import BaseTestPreprocessor
 from .. import (
  create_grade_cell, create_solution_cell, create_grade_and_solution_cell,
- create_locked_cell)
+ create_locked_cell, create_task_cell)
 
 
 @pytest.fixture
@@ -23,6 +23,7 @@ def gradebook(request, db):
 
  def fin():
  gb.close()
+
  request.addfinalizer(fin)
 
  return gb
@@ -197,7 +198,7 @@ def test_overwrite_locked_checksum(self, preprocessors, resources):
 
  assert cell.metadata.nbgrader["checksum"] == compute_checksum(cell)
 
- def test_nonexistant_grade_id(self, preprocessors, resources):
+ def test_nonexistent_grade_id(self, preprocessors, resources):
  """Are cells not in the database ignored?"""
  cell = create_grade_cell("", "code", "", 1)
  cell.metadata.nbgrader['grade'] = False
@@ -215,3 +216,52 @@ def test_nonexistant_grade_id(self, preprocessors, resources):
  nb, resources = preprocessors[1].preprocess(nb, resources)
  assert 'grade_id' not in cell.metadata.nbgrader
 
+ # Tests for adding missing cells back
+ def test_add_missing_cells(self, preprocessors, resources):
+ """
+ Note: This test will produce warnings (from OverwriteCells preprocessor) by default.
+ Current implementation of adding missing cells should:
+ - add missing cells right after the previous grade/solution cell, as the best approximation of their location
+ - add task cells at the end (because we can't detect their location in the notebook), in order of appearance
+ """
+
+ cells = [
+ create_solution_cell("Code assignment", "code", "code_solution"),
+ create_grade_cell("some tests", "code", "code_test1", 1),
+ create_grade_cell("more tests", "code", "code_test2", 1),
+ new_markdown_cell("some description"),
+ create_grade_and_solution_cell("write answer here", "markdown", "md_manually_graded1", 1),
+ create_grade_and_solution_cell("write answer here", "markdown", "md_manually_graded2", 1),
+ new_markdown_cell("some description"),
+ create_task_cell("some task description", "markdown", "task_cell1", 1),
+ new_markdown_cell("some description"),
+ create_task_cell("some task description", "markdown", "task_cell2", 1),
+ ]
+ # Add checksums to suppress warning
+ nbgrader_cells = [0, 1, 2, 4, 5, 7, 9]
+ for idx, cell in enumerate(cells):
+ if idx in nbgrader_cells:
+ cell.metadata.nbgrader["checksum"] = compute_checksum(cell)
+
+ expected_order = [0, 1, 2, 4, 5, 3, 6, 8, 7, 9]
+ expected = [cells[i].metadata.nbgrader["grade_id"] if "nbgrader" in cells[i].metadata else "markdown" for i in expected_order]
+
+ nb = new_notebook()
+ nb.cells = cells
+
+ # save to database
+ nb, resources = preprocessors[0].preprocess(nb, resources)
+
+ # remove grade/task cells to test their restoration
+ nb.cells.pop(9)
+ nb.cells.pop(7)
+ nb.cells.pop(5)
+ nb.cells.pop(4)
+ nb.cells.pop(2)
+ nb.cells.pop(1)
+
+ # restore
+ preprocessors[1].add_missing_cells = True
+ nb, resources = preprocessors[1].preprocess(nb, resources)
+ result = [cell["metadata"]["nbgrader"]["grade_id"] if "nbgrader" in cell["metadata"] else "markdown" for cell in nb.cells]
+ assert expected == result