From 0c894cc8bfb066b8e45daa7b2c6ebde480b1eccf Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 1 Mar 2016 21:27:36 -0600 Subject: [PATCH] Pull more logic out of pep8 --- flake8/checker.py | 98 +++++++++++++++++++++++++++++++++++++++++++---- flake8/utils.py | 18 +++++++++ 2 files changed, 109 insertions(+), 7 deletions(-) diff --git a/flake8/checker.py b/flake8/checker.py index 38a46349..a6c14ecb 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -201,40 +201,112 @@ def _make_processor(self): return None def report(self, error_code, line_number, column, text): - # type: (str, int, int, str) -> NoneType + # type: (str, int, int, str) -> str """Report an error by storing it in the results list.""" + if error_code is None: + error_code, text = text.split(' ', 1) error = (error_code, self.filename, line_number, column, text) self.results.append(error) + return error_code def run_check(self, plugin, **arguments): """Run the check in a single plugin.""" self.processor.keyword_arguments_for(plugin.parameters, arguments) return plugin.execute(**arguments) + def run_logical_checks(self): + """Run all checks expecting a logical line.""" + for plugin in self.checks.logical_line_plugins: + result = self.run_check(plugin) # , logical_line=logical_line) + if result is not None: + column_offset, text = result + self.report( + error_code=None, + line_number=self.processor.line_number, + column=column_offset, + text=text, + ) + def run_physical_checks(self, physical_line): + """Run all checks for a given physical line.""" for plugin in self.checks.physical_line_plugins: result = self.run_check(plugin, physical_line=physical_line) if result is not None: column_offset, text = result - error_code, error_text = text.split(' ', 1) - self.report( - error_code=error_code, + error_code = self.report( + error_code=None, line_number=self.processor.line_number, column=column_offset, - text=error_text, + text=text, ) self.processor.check_physical_error(error_code, physical_line) + def _log_token(self, token): + if token[2][0] == token[3][0]: + pos = '[%s:%s]' % (token[2][1] or '', token[3][1]) + else: + pos = 'l.%s' % token[3][0] + LOG.debug('l.%s\t%s\t%s\t%r' % + (token[2][0], pos, tokenize.tok_name[token[0]], + token[1])) + + def process_tokens(self): + """Process tokens and trigger checks. + + This can raise a :class:`flake8.exceptions.InvalidSyntax` exception. + Instead of using this directly, you should use + :meth:`flake8.checker.FileChecker.run_checks`. + """ + parens = 0 + processor = self.processor + for token in processor.generate_tokens(): + self.check_physical_eol(token) + token_type, text = token[0:2] + self._log_token(token) + if token_type == tokenize.OP: + parens = utils.count_parentheses(parens, text) + elif parens == 0: + if utils.token_is_newline(token): + self.handle_newline(token_type) + elif (utils.token_is_comment(token) and + len(processor.tokens) == 1): + self.handle_comment(token, text) + + if processor.tokens: + # If any tokens are left over, process them + self.run_physical_checks(processor.lines[-1]) + self.run_logical_checks() + def run_checks(self): """Run checks against the file.""" try: - for token in self.processor.generate_tokens(): - self.check_physical_eol(token) + self.process_tokens() except exceptions.InvalidSyntax as exc: self.report(exc.error_code, exc.line_number, exc.column_number, exc.error_message) + def handle_comment(self, token, token_text): + """Handle the logic when encountering a comment token.""" + # The comment also ends a physical line + token = list(token) + token[1] = token_text.rstrip('\r\n') + token[3] = (token[2][0], token[2][1] + len(token[1])) + self.processor.tokens = [tuple(token)] + self.run_logical_checks() + + def handle_newline(self, token_type): + """Handle the logic when encountering a newline token.""" + if token_type == tokenize.NEWLINE: + self.check_logical() + self.processor.reset_blank_before() + elif len(self.processor.tokens) == 1: + # The physical line contains only this token. + self.processor.visited_new_blank_line() + self.processor.delete_first_token() + else: + self.run_logical_checks() + def check_physical_eol(self, token): """Run physical checks if and only if it is at the end of the line.""" if utils.is_eol_token(token): @@ -331,6 +403,18 @@ def inside_multiline(self, line_number): yield self.multiline = False + def reset_blank_before(self): + """Reset the blank_before attribute to zero.""" + self.blank_before = 0 + + def delete_first_token(self): + """Delete the first token in the list of tokens.""" + del self.tokens[0] + + def visited_new_blank_line(self): + """Note that we visited a new blank line.""" + self.blank_lines += 1 + def split_line(self, token): """Split a physical line's line based on new-lines. diff --git a/flake8/utils.py b/flake8/utils.py index a3be0eaf..97b766af 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -205,3 +205,21 @@ def is_eol_token(token, _is_eol_token=is_eol_token): def is_multiline_string(token): """Check if this is a multiline string.""" return token[0] == tokenize.STRING and '\n' in token[1] + + +def token_is_newline(token): + """Check if the token type is a newline token type.""" + return token[0] in NEWLINE + + +def token_is_comment(token): + """Check if the token type is a comment.""" + return COMMENT_WITH_NL and token[0] == tokenize.COMMENT + + +def count_parentheses(current_parentheses_count, token_text): + """Count the number of parentheses.""" + if token_text in '([{': + return current_parentheses_count + 1 + elif token_text in '}])': + return current_parentheses_count - 1