diff --git a/flake8/checker.py b/flake8/checker.py index a6c14ecb..82f9e796 100644 --- a/flake8/checker.py +++ b/flake8/checker.py @@ -17,6 +17,9 @@ LOG = logging.getLogger(__name__) +SKIP_TOKENS = frozenset([tokenize.NL, tokenize.NEWLINE, tokenize.INDENT, + tokenize.DEDENT]) + class Manager(object): """Manage the parallelism and checker instances for each plugin and file. @@ -216,8 +219,15 @@ def run_check(self, plugin, **arguments): def run_logical_checks(self): """Run all checks expecting a logical line.""" + comments, logical_line, mapping = self.processor.build_logical_line() + if not mapping: + return + self.processor.update_state(mapping) + + LOG.debug('Logical line: "%s"', logical_line.rstrip()) + for plugin in self.checks.logical_line_plugins: - result = self.run_check(plugin) # , logical_line=logical_line) + result = self.run_check(plugin, logical_line=logical_line) if result is not None: column_offset, text = result self.report( @@ -415,6 +425,45 @@ def visited_new_blank_line(self): """Note that we visited a new blank line.""" self.blank_lines += 1 + def build_logical_line_tokens(self): + """Build the mapping, comments, and logical line lists.""" + logical = [] + comments = [] + length = 0 + previous_row = previous_column = mapping = None + for token_type, text, start, end, line in self.tokens: + if token_type in SKIP_TOKENS: + continue + if not mapping: + mapping = [(0, start)] + if token_type == tokenize.COMMENT: + comments.append(text) + continue + if token_type == tokenize.STRING: + text = utils.mutate_string(text) + if previous_row: + (start_row, start_column) = start + if previous_row != start_row: + row_index = previous_row - 1 + column_index = previous_column - 1 + previous_text = self.lines[row_index][column_index] + if (previous_text == ',' or + (previous_text not in '{[(' and + text not in '}])')): + text = ' ' + text + elif previous_column != start_column: + text = line[previous_column:start_column] + text + logical.append(text) + length += len(text) + mapping.append((length, end)) + (previous_row, previous_column) = end + return comments, logical, mapping + + def build_logical_line(self): + """Build a logical line from the current tokens list.""" + comments, logical, mapping_list = self.build_logical_line_tokens() + return ''.join(comments), ''.join(logical), mapping_list + def split_line(self, token): """Split a physical line's line based on new-lines. diff --git a/flake8/utils.py b/flake8/utils.py index 97b766af..fe024d1f 100644 --- a/flake8/utils.py +++ b/flake8/utils.py @@ -223,3 +223,23 @@ def count_parentheses(current_parentheses_count, token_text): return current_parentheses_count + 1 elif token_text in '}])': return current_parentheses_count - 1 + + +def mutate_string(text): + """Replace contents with 'xxx' to prevent syntax matching. + + >>> mute_string('"abc"') + '"xxx"' + >>> mute_string("'''abc'''") + "'''xxx'''" + >>> mute_string("r'abc'") + "r'xxx'" + """ + # String modifiers (e.g. u or r) + start = text.index(text[-1]) + 1 + end = len(text) - 1 + # Triple quotes + if text[-3:] in ('"""', "'''"): + start += 2 + end -= 2 + return text[:start] + 'x' * (end - start) + text[end:]