Use flake8 builtin file_tokens

flake8_eradicate.py

@@ -2,7 +2,7 @@
 
 import ast
 import tokenize
-from typing import Iterable, Iterator, Tuple, Type
+from typing import Iterable, Iterator, List, Tuple, Type
 
 import pkg_resources
 from eradicate import Eradicator
@@ -26,16 +26,21 @@ class Checker(object):
 
     options = None
 
-    def __init__(self, tree: ast.AST, filename: str):
+    def __init__(
+        self, tree: ast.AST,
+        filename: str,
+        file_tokens: List[tokenize.TokenInfo],
+    ):
         """
         ``flake8`` plugin constructor.
 
         Arguments:
             tree: the file abstract syntax tree.
             filename: the name of the file to process
-
+            file_tokens: file tokens
         """
         self.filename = filename
+        self.file_tokens = file_tokens
 
         self._options = {
             'aggressive': self.options.eradicate_aggressive,  # type: ignore
@@ -133,10 +138,9 @@ def _lines_with_commented_out_code(self) -> Iterable[int]:
                 # rewind as we don't want to skip it during tokenization
                 f.seek(0)
 
-            file_tokens = tokenize.generate_tokens(f.readline)
             comment_in_file = any(
                 token.type == tokenize.COMMENT
-                for token in file_tokens
+                for token in self.file_tokens
             )
 
             if comment_in_file:

tests/fixtures/correct_no_comment.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 class Some(object):
 
     property_name = 1

tests/test_comments.py

@@ -2,13 +2,19 @@
 
 import subprocess
 import sys
+import tokenize
 from collections import namedtuple
 
 from flake8_eradicate import Checker
 
 PY_GTE_36 = sys.version_info >= (3, 6)
 
 
+def _get_file_tokens(filename: str):
+    with open(filename) as f:
+        return list(tokenize.generate_tokens(f.readline))
+
+
 def test_correct_fixture(absolute_path):
     """End-to-End test to check that correct code works."""
     filename = absolute_path('fixtures', 'correct.py')
@@ -144,15 +150,19 @@ def test_lines_with_commented_out_code_incorrect_fixture_output(absolute_path):
         eradicate_whitelist_extend=False,
     )
 
-    checker = Checker(tree=None, filename=filename)
+    checker = Checker(
+        tree=None,
+        filename=filename,
+        file_tokens=_get_file_tokens(filename),
+    )
     output = list(checker._lines_with_commented_out_code())
     if PY_GTE_36:
         assert output == [3, 4, 9, 10, 14, 15, 16, 18, 19, 21, 22, 24, 25]
     else:
         assert output == [3, 9, 10, 14, 15, 16, 18, 19, 21, 22, 24, 25]
 
 
-def test_lines_with_commented_out_code_file_without_comment_output(
+def test_lines_with_commented_out_code_file_no_comment(
     absolute_path,
 ):
     """Make sure file without comment are ignored."""
@@ -168,6 +178,10 @@ def test_lines_with_commented_out_code_file_without_comment_output(
         eradicate_whitelist_extend=False,
     )
 
-    checker = Checker(tree=None, filename=filename)
+    checker = Checker(
+        tree=None,
+        filename=filename,
+        file_tokens=_get_file_tokens(filename),
+    )
     output = list(checker._lines_with_commented_out_code())
     assert output == []