New Feature: Automatic generation of test code

nbgrader/__init__.py

@@ -54,7 +54,7 @@ def _jupyter_nbextension_paths():
  src=os.path.join('nbextensions', 'validate_assignment'),
  dest="validate_assignment",
  require="validate_assignment/main"
- ),
+ )
  ]
 
  if sys.platform != 'win32':

nbgrader/apps/api.py

@@ -9,7 +9,7 @@
 from traitlets import Instance, Enum, Unicode, observe
 
 from ..coursedir import CourseDirectory
-from ..converters import GenerateAssignment, Autograde, GenerateFeedback, GenerateSolution
+from ..converters import GenerateAssignment, Autograde, GenerateFeedback, GenerateSolution, InstantiateTests
 from ..exchange import ExchangeFactory, ExchangeError
 from ..api import MissingEntry, Gradebook, Student, SubmittedAssignment
 from ..utils import parse_utc, temp_attrs, capture_log, as_timezone, to_numeric_tz
@@ -917,6 +917,32 @@ def generate_assignment(self, assignment_id, force=True, create=True):
  app.create_assignment = create
  return capture_log(app)
 
+ def instantiate_tests(self, assignment_id, force=True):
+ """Run ``nbgrader instantiate_tests`` for a particular assignment.
+
+ Arguments
+ ---------
+ assignment_id: string
+ The name of the assignment
+ force: bool
+ Whether to force creating the instantiated version, even if it already
+ exists.
+
+ Returns
+ -------
+ result: dict
+ A dictionary with the following keys (error and log may or may not be present):
+
+ - success (bool): whether or not the operation completed successfully
+ - error (string): formatted traceback
+ - log (string): captured log output
+
+ """
+ with temp_attrs(self.coursedir, assignment_id=assignment_id):
+ app = InstantiateTests(coursedir=self.coursedir, parent=self)
+ app.force = force
+ return capture_log(app)
+
  def unrelease(self, assignment_id):
  """Run ``nbgrader list --remove`` for a particular assignment.
 

nbgrader/apps/generateassignmentapp.py

@@ -2,10 +2,11 @@
 
 import sys
 
-from traitlets import default
+from traitlets import default, Bool
+from textwrap import dedent
 
 from .baseapp import NbGrader, nbgrader_aliases, nbgrader_flags
-from ..converters import BaseConverter, GenerateAssignment, NbGraderException
+from ..converters import BaseConverter, GenerateAssignment, NbGraderException, InstantiateTests
 from traitlets.traitlets import MetaHasTraits
 from typing import List, Any
 from traitlets.config.loader import Config
@@ -51,6 +52,13 @@
  {'BaseConverter': {'force': True}},
  "Overwrite an assignment/submission if it already exists."
  ),
+ 'generate_source_with_tests': (
+ {'GenerateAssignmentApp': {'instantiate_tests': True}},
+ "Generate an intermediate notebooks that contain the solutions as well as the tests "
+ "to be used for debugging purposes by the instructors\n"
+ "results saved according to\n"
+ "source_with_tests/./{assignment_id}/{notebook_id}.ipynb"
+ )
 })
 
 
@@ -62,6 +70,16 @@ class GenerateAssignmentApp(NbGrader):
  aliases = aliases
  flags = flags
 
+ instantiate_tests = Bool(
+ False,
+ help=dedent(
+ """
+ Generate an intermediate notebook that contains both solutions as well as the tests
+ to be used for debugging purposes"
+ """
+ )
+ ).tag(config=True)
+
  examples = """
  Produce the version of the assignment that is intended to be released to
  students. This performs several modifications to the original assignment:
@@ -112,7 +130,7 @@ class GenerateAssignmentApp(NbGrader):
  @default("classes")
  def _classes_default(self) -> List[MetaHasTraits]:
  classes = super(GenerateAssignmentApp, self)._classes_default()
- classes.extend([BaseConverter, GenerateAssignment])
+ classes.extend([BaseConverter, GenerateAssignment, InstantiateTests])
  return classes
 
  def _load_config(self, cfg: Config, **kwargs: Any) -> None:
@@ -141,6 +159,13 @@ def start(self) -> None:
  elif len(self.extra_args) == 1:
  self.coursedir.assignment_id = self.extra_args[0]
 
+ if self.instantiate_tests:
+ converter = InstantiateTests(coursedir=self.coursedir, parent=self)
+ try:
+ converter.start()
+ except NbGraderException:
+ sys.exit(1)
+
  converter = GenerateAssignment(coursedir=self.coursedir, parent=self)
  try:
  converter.start()

nbgrader/apps/nbgraderapp.py

@@ -111,6 +111,7 @@ class NbGraderApp(NbGrader):
  """
  ).strip()
  ),
+
  autograde=(
  AutogradeApp,
  dedent(

nbgrader/apps/quickstartapp.py

@@ -122,6 +122,11 @@ def start(self):
  ignore_html = shutil.ignore_patterns("*.html")
  shutil.copytree(example, os.path.join(course_path, "source"), ignore=ignore_html)
 
+ # copying the tests.yml file to the course directory
+ tests_file_path = os.path.abspath(os.path.join(
+ os.path.dirname(__file__), '..', 'docs', 'source', 'user_guide', 'tests.yml'))
+ shutil.copyfile(tests_file_path, os.path.join(course_path, 'tests.yml'))
+
  # create the config file
  self.log.info("Generating example config file...")
  currdir = os.getcwd()

nbgrader/converters/__init__.py

@@ -5,6 +5,7 @@
 from .feedback import Feedback
 from .generate_feedback import GenerateFeedback
 from .generate_solution import GenerateSolution
+from .instantiate_tests import InstantiateTests
 
 __all__ = [
  "BaseConverter",
@@ -14,5 +15,6 @@
  "Autograde",
  "Feedback",
  "GenerateFeedback",
- "GenerateSolution"
+ "GenerateSolution",
+ "InstantiateTests"
 ]

nbgrader/converters/generate_assignment.py

@@ -8,6 +8,7 @@
 from .base import BaseConverter, NbGraderException
 from ..preprocessors import (
  IncludeHeaderFooter,
+ InstantiateTests,
  ClearSolutions,
  LockCells,
  ComputeChecksums,
@@ -57,6 +58,7 @@ def _output_directory(self) -> str:
 
  preprocessors = List([
  IncludeHeaderFooter,
+ InstantiateTests,
  LockCells,
  ClearSolutions,
  ClearOutput,

nbgrader/converters/instantiate_tests.py

@@ -0,0 +1,51 @@
+import os
+import re
+from textwrap import dedent
+
+from traitlets import List, Bool, default
+
+from ..api import Gradebook, MissingEntry
+from .base import BaseConverter, NbGraderException
+from ..preprocessors import (
+ InstantiateTests,
+ ClearOutput,
+ CheckCellMetadata
+)
+from traitlets.config.loader import Config
+from typing import Any
+from ..coursedir import CourseDirectory
+
+
+class InstantiateTests(BaseConverter):
+
+ @default("permissions")
+ def _permissions_default(self) -> int:
+ return 664 if self.coursedir.groupshared else 644
+
+ @property
+ def _input_directory(self) -> str:
+ return self.coursedir.source_directory
+
+ @property
+ def _output_directory(self) -> str:
+ return self.coursedir.source_with_tests_directory
+
+ preprocessors = List([
+ InstantiateTests,
+ ClearOutput,
+ CheckCellMetadata,
+ ]).tag(config=True)
+
+ def _load_config(self, cfg: Config, **kwargs: Any) -> None:
+ super(InstantiateTests, self)._load_config(cfg, **kwargs)
+
+ def __init__(self, coursedir: CourseDirectory = None, **kwargs: Any) -> None:
+ super(InstantiateTests, self).__init__(coursedir=coursedir, **kwargs)
+
+ def start(self) -> None:
+ old_student_id = self.coursedir.student_id
+ self.coursedir.student_id = '.'
+ try:
+ super(InstantiateTests, self).start()
+ finally:
+ self.coursedir.student_id = old_student_id

nbgrader/coursedir.py

@@ -142,6 +142,19 @@ def _validate_notebook_id(self, proposal: Bunch) -> str:
  )
  ).tag(config=True)
 
+ source_with_tests_directory = Unicode(
+ 'source_with_tests',
+ help=dedent(
+ """
+ The name of the directory that contains the version of the
+ assignment that will contain both the solutions and the instantiated 
+ test cases to be used by instructors. This corresponds to
+ the `nbgrader_step` variable in the `directory_structure` config
+ option.
+ """
+ )
+ ).tag(config=True)
+
  submitted_directory = Unicode(
  'submitted',
  help=dedent(