diff --git a/nbgrader/apps/quickstartapp.py b/nbgrader/apps/quickstartapp.py index 0af1460e4..36f8b528f 100644 --- a/nbgrader/apps/quickstartapp.py +++ b/nbgrader/apps/quickstartapp.py @@ -122,10 +122,10 @@ def start(self): ignore_html = shutil.ignore_patterns("*.html") shutil.copytree(example, os.path.join(course_path, "source"), ignore=ignore_html) - # copying the tests.yml file to the course directory + # copying the autotests.yml file to the course directory tests_file_path = os.path.abspath(os.path.join( - os.path.dirname(__file__), '..', 'docs', 'source', 'user_guide', 'tests.yml')) - shutil.copyfile(tests_file_path, os.path.join(course_path, 'tests.yml')) + os.path.dirname(__file__), '..', 'docs', 'source', 'user_guide', 'autotests.yml')) + shutil.copyfile(tests_file_path, os.path.join(course_path, 'autotests.yml')) # create the config file self.log.info("Generating example config file...") diff --git a/nbgrader/docs/source/user_guide/tests.yml b/nbgrader/docs/source/user_guide/autotests.yml similarity index 100% rename from nbgrader/docs/source/user_guide/tests.yml rename to nbgrader/docs/source/user_guide/autotests.yml diff --git a/nbgrader/preprocessors/instantiatetests.py b/nbgrader/preprocessors/instantiatetests.py index 2089c5d48..7cb9a4d05 100644 --- a/nbgrader/preprocessors/instantiatetests.py +++ b/nbgrader/preprocessors/instantiatetests.py @@ -33,7 +33,7 @@ class InstantiateTests(NbGraderPreprocessor): tests = None autotest_filename = Unicode( - "tests.yml", + "autotests.yml", help="The filename where automatic testing code is stored" ).tag(config=True) @@ -115,7 +115,7 @@ def preprocess(self, nb, resources): raise ValueError(f"Kernel {kernel_name} has not been specified in InstantiateTests.comment_strs") if kernel_name not in self.sanitizers: raise ValueError(f"Kernel {kernel_name} has not been specified in InstantiateTests.sanitizers") - self.log.debug(f"Found kernel {kernel_name}") + self.log.debug("Found kernel %s", kernel_name) resources["kernel_name"] = kernel_name # load the template tests file @@ -124,10 +124,10 @@ def preprocess(self, nb, resources): self.global_tests_loaded = True # set up the sanitizer - self.log.debug('Setting sanitizer for kernel {kernel_name}') + self.log.debug('Setting sanitizer for kernel %s', kernel_name) self.sanitizer = self.sanitizers[kernel_name] #start the kernel - self.log.debug('Starting client for kernel {kernel_name}') + self.log.debug('Starting client for kernel %s', kernel_name) km, self.kc = start_new_kernel(kernel_name = kernel_name) # run the preprocessor @@ -214,7 +214,9 @@ def preprocess_cell(self, cell, resources, index): # appears in the line before the self.autotest_delimiter token use_hash = (self.hashed_delimiter in line[:line.find(self.autotest_delimiter)]) if use_hash: - self.log.debug('Hashing delimiter found, using template: ' + self.hash_template) + if self.hash_template is None: + raise ValueError('Found a hashing delimiter, but the hash property has not been set in autotests.yml') + self.log.debug('Hashing delimiter found, using template: %s', self.hash_template) else: self.log.debug('Hashing delimiter not found') @@ -233,18 +235,18 @@ def preprocess_cell(self, cell, resources, index): # generate the test for each snippet for snippet in snippets: - self.log.debug('Running autotest generation for snippet ' + snippet) + self.log.debug('Running autotest generation for snippet %s', snippet) # create a random salt for this test if use_hash: salt = secrets.token_hex(8) - self.log.debug('Using salt: ' + salt) + self.log.debug('Using salt: %s', salt) else: salt = None # get the normalized(/hashed) template tests for this code snippet self.log.debug( - 'Instantiating normalized' + ('/hashed ' if use_hash else ' ') + 'test templates based on type') + 'Instantiating normalized%s test templates based on type', ' & hashed' if use_hash else '') instantiated_tests, test_values, fail_messages = self._instantiate_tests(snippet, salt) # add all the lines to the cell @@ -253,21 +255,21 @@ def preprocess_cell(self, cell, resources, index): for i in range(len(instantiated_tests)): check_code = template.render(snippet=instantiated_tests[i], value=test_values[i], message=fail_messages[i]) - self.log.debug('Test: ' + check_code) + self.log.debug('Test: %s', check_code) new_lines.append(check_code) # add an empty line after this block of test code new_lines.append('') + # add the final success code and execute it + if is_grade_flag and self.global_tests_loaded and (self.autotest_delimiter in cell.source) and (self.success_code is not None): + new_lines.append(self.success_code) + non_autotest_code_lines.append(self.success_code) + # run the trailing non-autotest lines, if any remain if len(non_autotest_code_lines) > 0: self._execute_code_snippet("\n".join(non_autotest_code_lines)) - # add the final success message - if is_grade_flag and self.global_tests_loaded: - if self.autotest_delimiter in cell.source: - new_lines.append(self.success_code) - # replace the cell source cell.source = "\n".join(new_lines) @@ -279,36 +281,34 @@ def preprocess_cell(self, cell, resources, index): # ------------------------------------------------------------------------------------- def _load_test_template_file(self, resources): """ - attempts to load the tests.yml file within the assignment directory. In case such file is not found + attempts to load the autotests.yml file within the assignment directory. In case such file is not found or perhaps cannot be loaded, it will attempt to load the default_tests.yaml file with the course_directory """ - self.log.debug('loading template tests.yml...') - self.log.debug(f'kernel_name: {resources["kernel_name"]}') + self.log.debug('loading template autotests.yml...') + self.log.debug('kernel_name: %s', resources["kernel_name"]) try: with open(os.path.join(resources['metadata']['path'], self.autotest_filename), 'r') as tests_file: tests = yaml.safe_load(tests_file) self.log.debug(tests) except FileNotFoundError: - # if there is no tests file, just load a default tests dict + # if there is no tests file, try to load default tests dict self.log.warning( - 'No tests.yml file found in the assignment directory. Loading the default tests.yml file in the course root directory') - # tests = {} + 'No autotests.yml file found in the assignment directory. Loading the default autotests.yml file in the course root directory') try: with open(os.path.join(self.autotest_filename), 'r') as tests_file: tests = yaml.safe_load(tests_file) except FileNotFoundError: - # if there is no tests file, just create a default empty tests dict - self.log.warning( - 'No tests.yml file found. If AUTOTESTS appears in testing cells, an error will be thrown.') - tests = {} + # if there is not even a default tests file, re-raise the FileNotFound error + self.log.error('No autotests.yml file found, but there were autotest directives found in the notebook. ') + raise except yaml.parser.ParserError as e: - self.log.error('tests.yml contains invalid YAML code.') + self.log.error('autotests.yml contains invalid YAML code.') self.log.error(e.msg) raise except yaml.parser.ParserError as e: - self.log.error('tests.yml contains invalid YAML code.') + self.log.error('autotests.yml contains invalid YAML code.') self.log.error(e.msg) raise @@ -322,10 +322,10 @@ def _load_test_template_file(self, resources): self.dispatch_template = tests['dispatch'] # get the success message template - self.success_code = tests['success'] + self.success_code = tests.get('success', None) # get the hash code template - self.hash_template = tests['hash'] + self.hash_template = tests.get('hash', None) # get the hash code template self.check_template = tests['check'] @@ -342,19 +342,19 @@ def _instantiate_tests(self, snippet, salt=None): template = j2.Environment(loader=j2.BaseLoader).from_string(self.dispatch_template) dispatch_code = template.render(snippet=snippet) dispatch_result = self._execute_code_snippet(dispatch_code) - self.log.debug('Dispatch result returned by kernel: ', dispatch_result) + self.log.debug('Dispatch result returned by kernel: %s', dispatch_result) # get the test code; if the type isn't in our dict, just default to 'default' # if default isn't in the tests code, this will throw an error try: tests = self.test_templates_by_type.get(dispatch_result, self.test_templates_by_type['default']) except KeyError: - self.log.error('tests.yml must contain a top-level "default" key with corresponding test code') + self.log.error('autotests.yml must contain a top-level "default" key with corresponding test code') raise try: test_templs = [t['test'] for t in tests] fail_msgs = [t['fail'] for t in tests] except KeyError: - self.log.error('each type in tests.yml must have a list of dictionaries with a "test" and "fail" key') + self.log.error('each type in autotests.yml must have a list of dictionaries with a "test" and "fail" key') self.log.error('the "test" item should store the test template code, ' 'and the "fail" item should store a failure message') raise diff --git a/nbgrader/tests/apps/files/tests.yml b/nbgrader/tests/apps/files/autotests.yml similarity index 100% rename from nbgrader/tests/apps/files/tests.yml rename to nbgrader/tests/apps/files/autotests.yml diff --git a/nbgrader/tests/apps/test_nbgrader_autograde.py b/nbgrader/tests/apps/test_nbgrader_autograde.py index 1f00939f8..7bd3eacdc 100644 --- a/nbgrader/tests/apps/test_nbgrader_autograde.py +++ b/nbgrader/tests/apps/test_nbgrader_autograde.py @@ -102,7 +102,7 @@ def test_grade_autotest(self, db, course_dir): run_nbgrader(["db", "student", "add", "bar", "--db", db]) self._copy_file(join("files", "autotest-simple.ipynb"), join(course_dir, "source", "ps1", "p1.ipynb")) - self._copy_file(join("files", "tests.yml"), join(course_dir, "tests.yml")) + self._copy_file(join("files", "autotests.yml"), join(course_dir, "autotests.yml")) run_nbgrader(["generate_assignment", "ps1", "--db", db]) self._copy_file(join("files", "autotest-simple-unchanged.ipynb"), join(course_dir, "submitted", "foo", "ps1", "p1.ipynb")) @@ -132,7 +132,7 @@ def test_grade_hashed_autotest(self, db, course_dir): run_nbgrader(["db", "student", "add", "bar", "--db", db]) self._copy_file(join("files", "autotest-hashed.ipynb"), join(course_dir, "source", "ps1", "p1.ipynb")) - self._copy_file(join("files", "tests.yml"), join(course_dir, "tests.yml")) + self._copy_file(join("files", "autotests.yml"), join(course_dir, "autotests.yml")) run_nbgrader(["generate_assignment", "ps1", "--db", db]) self._copy_file(join("files", "autotest-hashed-unchanged.ipynb"), join(course_dir, "submitted", "foo", "ps1", "p1.ipynb")) @@ -162,7 +162,7 @@ def test_grade_complex_autotest(self, db, course_dir): run_nbgrader(["db", "student", "add", "bar", "--db", db]) self._copy_file(join("files", "autotest-multi.ipynb"), join(course_dir, "source", "ps1", "p1.ipynb")) - self._copy_file(join("files", "tests.yml"), join(course_dir, "tests.yml")) + self._copy_file(join("files", "autotests.yml"), join(course_dir, "autotests.yml")) run_nbgrader(["generate_assignment", "ps1", "--db", db]) self._copy_file(join("files", "autotest-multi-unchanged.ipynb"), join(course_dir, "submitted", "foo", "ps1", "p1.ipynb")) @@ -944,7 +944,7 @@ def test_hidden_tests_autotest(self, db, course_dir): fh.write("""c.ClearSolutions.code_stub=dict(python="# YOUR CODE HERE")""") self._copy_file(join("files", "autotest-hidden.ipynb"), join(course_dir, "source", "ps1", "p1.ipynb")) - self._copy_file(join("files", "tests.yml"), join(course_dir, "tests.yml")) + self._copy_file(join("files", "autotests.yml"), join(course_dir, "autotests.yml")) run_nbgrader(["generate_assignment", "ps1", "--db", db]) self._copy_file(join("files", "autotest-hidden-unchanged.ipynb"), join(course_dir, "submitted", "foo", "ps1", "p1.ipynb")) diff --git a/nbgrader/tests/apps/test_nbgrader_generate_assignment.py b/nbgrader/tests/apps/test_nbgrader_generate_assignment.py index 444f1f08f..cb93d0f30 100644 --- a/nbgrader/tests/apps/test_nbgrader_generate_assignment.py +++ b/nbgrader/tests/apps/test_nbgrader_generate_assignment.py @@ -50,11 +50,11 @@ def test_single_file(self, course_dir, temp_cwd): def test_autotests_simple(self, course_dir, temp_cwd): """Can a notebook with simple autotests be generated with a default yaml location, and is autotest code removed?""" self._copy_file(join("files", "autotest-simple.ipynb"), join(course_dir, "source", "ps1", "foo.ipynb")) - self._copy_file(join("files", "tests.yml"), join(course_dir, "tests.yml")) + self._copy_file(join("files", "autotests.yml"), join(course_dir, "autotests.yml")) run_nbgrader(["db", "assignment", "add", "ps1"]) run_nbgrader(["generate_assignment", "ps1"]) assert os.path.isfile(join(course_dir, "release", "ps1", "foo.ipynb")) - assert not os.path.isfile(join(course_dir, "release", "ps1", "tests.yml")) + assert not os.path.isfile(join(course_dir, "release", "ps1", "autotests.yml")) foo = self._file_contents(join(course_dir, "release", "ps1", "foo.ipynb")) assert "AUTOTEST" not in foo @@ -62,11 +62,11 @@ def test_autotests_simple(self, course_dir, temp_cwd): def test_autotests_simple(self, course_dir, temp_cwd): """Can a notebook with simple autotests be generated with an assignment-specific yaml, and is autotest code removed?""" self._copy_file(join("files", "autotest-simple.ipynb"), join(course_dir, "source", "ps1", "foo.ipynb")) - self._copy_file(join("files", "tests.yml"), join(course_dir, "source", "ps1", "tests.yml")) + self._copy_file(join("files", "autotests.yml"), join(course_dir, "source", "ps1", "autotests.yml")) run_nbgrader(["db", "assignment", "add", "ps1"]) run_nbgrader(["generate_assignment", "ps1"]) assert os.path.isfile(join(course_dir, "release", "ps1", "foo.ipynb")) - assert os.path.isfile(join(course_dir, "release", "ps1", "tests.yml")) + assert os.path.isfile(join(course_dir, "release", "ps1", "autotests.yml")) foo = self._file_contents(join(course_dir, "release", "ps1", "foo.ipynb")) assert "AUTOTEST" not in foo @@ -81,7 +81,7 @@ def test_autotests_needs_yaml(self, course_dir, temp_cwd): def test_autotests_fancy(self, course_dir, temp_cwd): """Can a more complicated autotests notebook be generated, and is autotest code removed?""" self._copy_file(join("files", "autotest-multi.ipynb"), join(course_dir, "source", "ps1", "foo.ipynb")) - self._copy_file(join("files", "tests.yml"), join(course_dir, "tests.yml")) + self._copy_file(join("files", "autotests.yml"), join(course_dir, "autotests.yml")) run_nbgrader(["db", "assignment", "add", "ps1"]) run_nbgrader(["generate_assignment", "ps1"]) assert os.path.isfile(join(course_dir, "release", "ps1", "foo.ipynb")) @@ -92,7 +92,7 @@ def test_autotests_fancy(self, course_dir, temp_cwd): def test_autotests_hidden(self, course_dir, temp_cwd): """Can a notebook with hidden autotest be generated, and is autotest/hidden sections removed?""" self._copy_file(join("files", "autotest-hidden.ipynb"), join(course_dir, "source", "ps1", "foo.ipynb")) - self._copy_file(join("files", "tests.yml"), join(course_dir, "tests.yml")) + self._copy_file(join("files", "autotests.yml"), join(course_dir, "autotests.yml")) run_nbgrader(["db", "assignment", "add", "ps1"]) run_nbgrader(["generate_assignment", "ps1"]) assert os.path.isfile(join(course_dir, "release", "ps1", "foo.ipynb")) @@ -104,7 +104,7 @@ def test_autotests_hidden(self, course_dir, temp_cwd): def test_autotests_hashed(self, course_dir, temp_cwd): """Can a notebook with hashed autotests be generated, and is hashed autotest code removed?""" self._copy_file(join("files", "autotest-hashed.ipynb"), join(course_dir, "source", "ps1", "foo.ipynb")) - self._copy_file(join("files", "tests.yml"), join(course_dir, "tests.yml")) + self._copy_file(join("files", "autotests.yml"), join(course_dir, "autotests.yml")) run_nbgrader(["db", "assignment", "add", "ps1"]) run_nbgrader(["generate_assignment", "ps1"]) assert os.path.isfile(join(course_dir, "release", "ps1", "foo.ipynb")) @@ -117,7 +117,7 @@ def test_generate_source_with_tests_flag(self, course_dir, temp_cwd): """Does setting the flag --source_with_tests also create a notebook with solution and tests in the source_with_tests directory""" self._copy_file(join("files", "autotest-simple.ipynb"), join(course_dir, "source", "ps1", "foo.ipynb")) - self._copy_file(join("files", "tests.yml"), join(course_dir, "source", "ps1", "tests.yml")) + self._copy_file(join("files", "autotests.yml"), join(course_dir, "source", "ps1", "autotests.yml")) run_nbgrader(["db", "assignment", "add", "ps1"]) run_nbgrader(["generate_assignment", "ps1", "--source_with_tests"]) assert os.path.isfile(join(course_dir, "release", "ps1", "foo.ipynb")) diff --git a/nbgrader/tests/apps/test_nbgrader_generate_feedback.py b/nbgrader/tests/apps/test_nbgrader_generate_feedback.py index 6ff20a70b..cb76d3e13 100644 --- a/nbgrader/tests/apps/test_nbgrader_generate_feedback.py +++ b/nbgrader/tests/apps/test_nbgrader_generate_feedback.py @@ -333,7 +333,7 @@ def test_autotests(self, course_dir): self._copy_file(join("files", "autotest-simple.ipynb"), join(course_dir, "source", "ps1", "p1.ipynb")) self._copy_file(join("files", "autotest-simple.ipynb"), join(course_dir, "source", "ps1", "p2.ipynb")) - self._copy_file(join("files", "tests.yml"), join(course_dir, "tests.yml")) + self._copy_file(join("files", "autotests.yml"), join(course_dir, "autotests.yml")) run_nbgrader(["generate_assignment", "ps1"]) self._copy_file(join("files", "autotest-simple.ipynb"), join(course_dir, "submitted", "foo", "ps1", "p1.ipynb"))