add --exclude-regex and --no-make-paths-absolute to exclude specific …

…file paths (#115)

* add --exclude-glob argument to exclude file paths from detection

* add --no-make-paths-absolute to allow relative globs

NOTE: tests are broken here because [!/] doesn't work the way I thought it would. The next commit
will change the glob option to use a regex.

* change --exclude-glob to --exclude-regex

- tests pass now
- interface more familiar for windows users

* match using re.search and use anchors

* memoize a "master regex" instead of looping through regex patterns

Add pylint overrides for tests which access the private memoized master regex. I think these tests
are necessary if we think memoizing the regex is a useful optimization, but this may well be
premature optimization.

* Revert "memoize a "master regex" instead of looping through regex patterns"

This reverts commit 27302a3.

* respond to review comments!

* add tests for config parsing

- also add 'yes'/'no' test cases for other boolean config flags

* compare exclusion_regex pattern strings

* fix os path sep for regex on windows

sample.vermin.ini

@@ -77,6 +77,30 @@
 #exclusions =
 #  email.parser.FeedParser
 #  argparse.ArgumentParser(allow_abbrev)
+#
+#
+# Exclude specific file paths from being crawled by matching against a regular expression. Useful to
+# ignore files that are not executed under the same python interpreter, such as .pyi files.
+#
+# Exclude any '.pyi' file:                \.pyi$
+#
+# (Note: the below examples require `make_paths_absolute = no`, or prefixing the patterns with the
+# regex-escaped path to the current directory.)
+#
+# Exclude the directory 'a/b/':           ^a/b$
+# Exclude '.pyi' files under 'a/b/':      ^a/b/.+\.pyi$
+# Exclude '.pyi' files in exactly 'a/b/': ^a/b/[^/]+\.pyi$
+#
+# Example regex exclusions:
+#exclusion_regex =
+#  \.pyi$
+
+### Absolute Path Resolution
+# Convert any relative paths from the command line into absolute paths. This affects the path
+# printed to the terminal if a file fails a check, and and requires exclusion_regex patterns to
+# match absolute paths.
+#
+#make_paths_absolute = yes
 
 ### Backports ###
 # Some features are sometimes backported into packages, in repositories such as PyPi, that are

tests/arguments.py

@@ -226,6 +226,58 @@ def test_exclude_file(self):
     self.assertContainsDict({"code": 0}, self.parse_args(["--exclude-file", fn]))
     self.assertEmpty(self.config.exclusions())
 
+  def test_exclude_regex(self):
+    self.assertContainsDict({"code": 1}, self.parse_args(["--exclude-regex"]))  # Needs <rx> part.
+    self.assertEmpty(self.config.exclusion_regex())
+
+    args = ["--exclude-regex", r"\.pyi$",
+            "--exclude-regex", "^a/b$"]
+    self.assertContainsDict({"code": 0}, self.parse_args(args))
+    expected = [r"\.pyi$", "^a/b$"]
+    self.assertEqual(expected, self.config.exclusion_regex())  # Expect it sorted.
+    self.assertFalse(self.config.is_excluded_by_regex("a/b.py"))
+    self.assertTrue(self.config.is_excluded_by_regex("asdf.pyi"))
+    self.assertTrue(self.config.is_excluded_by_regex("a/m.pyi"))
+    self.assertTrue(self.config.is_excluded_by_regex("a/b"))
+
+    # Regex patterns are applied at each level of directory traversal. If 'a/' is provided on the
+    # command line, then the regex 'a/b' will match the recursive traversal when it encounters the
+    # directory 'a/b', and avoid recursing into that directory. This makes it more efficient to use
+    # when possible, but more difficult to test in isolation. test_exclude_regex_relative() in
+    # general.py tests that 'a/b' excludes e.g. 'a/b/c.py'.
+    self.assertFalse(self.config.is_excluded_by_regex("a/b/c.py"))
+
+    self.config.reset()
+    self.assertEmpty(self.config.exclusion_regex())
+    args = ["--exclude-regex", "^a/b/.+$",
+            "--exclude-regex", r"^a/.+/.+\.pyi$"]
+    self.assertContainsDict({"code": 0}, self.parse_args(args))
+    self.assertTrue(self.config.is_excluded_by_regex("a/b/c.py"))
+    self.assertTrue(self.config.is_excluded_by_regex("a/b/c/d.py"))
+    # '.+/.+\.pyi' does not match .pyi files in the top-level directory.
+    self.assertFalse(self.config.is_excluded_by_regex("a/m.pyi"))
+    self.assertTrue(self.config.is_excluded_by_regex("a/d/m.pyi"))
+    self.assertFalse(self.config.is_excluded_by_regex("m.pyi"))
+
+    self.config.reset()
+    # Use '[^/]+' instead of '.+' to force only matching files in the top-level.
+    self.assertContainsDict({"code": 0}, self.parse_args(["--exclude-regex", r"^a/b/[^/]+\.pyi$"]))
+    self.assertTrue(self.config.is_excluded_by_regex("a/b/c.pyi"))
+    self.assertFalse(self.config.is_excluded_by_regex("a/b/c.py"))
+    self.assertFalse(self.config.is_excluded_by_regex("a/b/c/d.pyi"))
+
+    self.assertContainsDict({"code": 0}, self.parse_args(["--no-exclude-regex"]))
+    self.assertEmpty(self.config.exclusion_regex())
+
+  def test_make_paths_absolute(self):
+    self.assertTrue(self.config.make_paths_absolute())
+
+    self.assertContainsDict({"code": 0}, self.parse_args(["--no-make-paths-absolute"]))
+    self.assertFalse(self.config.make_paths_absolute())
+
+    self.assertContainsDict({"code": 0}, self.parse_args(["--make-paths-absolute"]))
+    self.assertTrue(self.config.make_paths_absolute())
+
   def test_backport(self):
     # Needs <name> part.
     self.assertContainsDict({"code": 1}, self.parse_args(["--backport"]))

tests/config.py

@@ -78,6 +78,8 @@ def test_repr(self):
   show_tips = {}
   analyze_hidden = {}
   exclusions = {}
+  exclusion_regex = {}
+  make_paths_absolute = {}
   backports = {}
   features = {}
   targets = {}
@@ -89,8 +91,9 @@ def test_repr(self):
 )""".format(self.config.__class__.__name__, self.config.quiet(), self.config.verbose(),
             self.config.print_visits(), self.config.processes(), self.config.ignore_incomp(),
             self.config.pessimistic(), self.config.show_tips(), self.config.analyze_hidden(),
-            self.config.exclusions(), list(self.config.backports()), list(self.config.features()),
-            self.config.targets(), self.config.eval_annotations(),
+            self.config.exclusions(), self.config.exclusion_regex(),
+            self.config.make_paths_absolute(), list(self.config.backports()),
+            list(self.config.features()), self.config.targets(), self.config.eval_annotations(),
             self.config.only_show_violations(), self.config.parse_comments(),
             self.config.scan_symlink_folders(), self.config.format().name()))
 
@@ -192,6 +195,12 @@ def test_parse_invalid_verbose(self):
 """, True],
     [u"""[vermin]
 print_visits = False
+""", False],
+    [u"""[vermin]
+print_visits = yes
+""", True],
+    [u"""[vermin]
+print_visits = no
 """, False],
   ])
   def test_parse_print_visits(self, data, expected):
@@ -238,6 +247,12 @@ def test_parse_invalid_processes(self):
 """, True],
     [u"""[vermin]
 ignore_incomp = False
+""", False],
+    [u"""[vermin]
+ignore_incomp = yes
+""", True],
+    [u"""[vermin]
+ignore_incomp = no
 """, False],
   ])
   def test_parse_ignore_incomp(self, data, expected):
@@ -257,6 +272,12 @@ def test_parse_ignore_incomp(self, data, expected):
 """, True],
     [u"""[vermin]
 pessimistic = False
+""", False],
+    [u"""[vermin]
+pessimistic = yes
+""", True],
+    [u"""[vermin]
+pessimistic = no
 """, False],
   ])
   def test_parse_pessimistic(self, data, expected):
@@ -276,6 +297,12 @@ def test_parse_pessimistic(self, data, expected):
 """, False],
     [u"""[vermin]
 show_tips = True
+""", True],
+    [u"""[vermin]
+show_tips = no
+""", False],
+    [u"""[vermin]
+show_tips = yes
 """, True],
   ])
   def test_parse_show_tips(self, data, expected):
@@ -308,6 +335,56 @@ def test_parse_exclusions(self, data, expected):
     self.assertIsNotNone(config)
     self.assertEqual(config.exclusions(), expected)
 
+  @VerminTest.parameterized_args([
+    [u"""[vermin]
+exclusion_regex =
+""", []],
+    [u"""[vermin]
+#exclusion_regex = \\.pyi$
+""", []],
+    [u"""[vermin]
+exclusion_regex = \\.pyi$
+""", [r"\.pyi$"]],
+    [u"""[vermin]
+exclusion_regex = \\.pyi$
+  ^a/b$
+""", [r"\.pyi$", r"^a/b$"]],
+    [u"""[vermin]
+exclusion_regex =
+  ^a/b$
+  \\.pyi$
+""", [r"\.pyi$", r"^a/b$"]],
+  ])
+  def test_parse_exclusion_regex(self, data, expected):
+    config = Config.parse_data(data)
+    self.assertIsNotNone(config)
+    self.assertEqual(config.exclusion_regex(), expected)
+
+  @VerminTest.parameterized_args([
+    [u"""[vermin]
+make_paths_absolute =
+""", True],
+    [u"""[vermin]
+#make_paths_absolute = False
+""", True],
+    [u"""[vermin]
+make_paths_absolute = yes
+""", True],
+    [u"""[vermin]
+make_paths_absolute = no
+""", False],
+    [u"""[vermin]
+make_paths_absolute = True
+""", True],
+    [u"""[vermin]
+make_paths_absolute = False
+""", False],
+  ])
+  def test_parse_make_paths_absolute(self, data, expected):
+    config = Config.parse_data(data)
+    self.assertIsNotNone(config)
+    self.assertEqual(config.make_paths_absolute(), expected)
+
   def test_parse_backports(self):
     bps = Backports.modules()
     config = Config.parse_data(u"""[vermin]

tests/general.py

@@ -1,5 +1,6 @@
 import sys
 import os
+import re
 import io
 from os.path import abspath, basename, join, splitext
 from tempfile import NamedTemporaryFile, mkdtemp
@@ -13,7 +14,8 @@
 from vermin.formats import ParsableFormat
 from vermin.utility import open_wrapper
 
-from .testutils import VerminTest, current_version, ScopedTemporaryFile, detect, visit, touch
+from .testutils import VerminTest, current_version, ScopedTemporaryFile, detect, visit, touch, \
+  working_dir
 
 class VerminGeneralTests(VerminTest):
   def test_detect_without_config(self):
@@ -314,6 +316,73 @@ def test_detect_vermin_paths_no_invalid_exts(self):
 
     rmtree(tmp_fld)
 
+  def test_exclude_pyi_regex(self):
+    tmp_fld = mkdtemp()
+
+    # With the default of --make-paths-absolute, this will match .pyi files in any subdirectory. The
+    # most common use case for --exclude-regex is expected to be for file extensions, so it's great
+    # that will work regardless of the --make-paths-absolute setting.
+    self.config.add_exclusion_regex(r"\.pyi$")
+
+    f = touch(tmp_fld, "code.pyi")
+    with open_wrapper(f, mode="w", encoding="utf-8") as fp:
+      fp.write("print('this is code')")
+
+    paths = detect_paths([tmp_fld], config=self.config)
+    self.assertEmpty(paths)
+
+    rmtree(tmp_fld)
+
+  def test_exclude_directory_regex(self):
+    tmp_fld = mkdtemp()
+
+    # Excluding the directory .../a should exclude any files recursively beneath it as well.
+    self.config.add_exclusion_regex('^' + re.escape(join(tmp_fld, "a")) + '$')
+
+    # Create .../a and .../a/b directories.
+    os.mkdir(join(tmp_fld, "a"))
+    os.mkdir(join(tmp_fld, "a/b"))
+
+    paths = ["code.py", "a/code.py", "a/b/code.py"]
+    for p in paths:
+      f = touch(tmp_fld, p)
+      with open_wrapper(f, mode="w", encoding="utf-8") as fp:
+        fp.write("print('this is code')")
+
+    paths = detect_paths([tmp_fld], config=self.config)
+    self.assertEqual(paths, [join(tmp_fld, "code.py")])
+
+    rmtree(tmp_fld)
+
+  def test_exclude_regex_relative(self):
+    tmp_fld = mkdtemp()
+
+    # Keep paths relative, and provide patterns matching relative paths.
+    self.config.set_make_paths_absolute(False)
+    self.config.add_exclusion_regex("^a{0}b$".format(re.escape(os.path.sep)))
+    self.config.add_exclusion_regex("^a{0}.+pyi$".format(re.escape(os.path.sep)))
+
+    # Create .../a and .../a/b directories.
+    os.mkdir(join(tmp_fld, "a"))
+    os.mkdir(join(tmp_fld, "a", "b"))
+
+    paths = [
+      join("a", "code.py"),
+      join("a", "code.pyi"),
+      join("a", "b", "code.py"),
+    ]
+    for p in paths:
+      f = touch(tmp_fld, p)
+      with open_wrapper(f, mode="w", encoding="utf-8") as fp:
+        fp.write("print('this is code')")
+
+    # Temporarily modify the working directory.
+    with working_dir(tmp_fld):
+      paths = detect_paths(["a"], config=self.config)
+      self.assertEqual(paths, [join("a", "code.py")])
+
+    rmtree(tmp_fld)
+
   def test_detect_vermin_min_versions(self):
     paths = detect_paths([abspath("vermin")], config=self.config)
     processor = Processor()

tests/testutils.py

@@ -1,6 +1,7 @@
 import unittest
 import sys
 import os
+from contextlib import contextmanager
 from os.path import join
 from tempfile import NamedTemporaryFile
 
@@ -18,6 +19,15 @@ def touch(fld, name, contents=None):
   fp.close()
   return filename
 
+@contextmanager
+def working_dir(path):
+  prev_wd = os.getcwd()
+  try:
+    os.chdir(path)
+    yield
+  finally:
+    os.chdir(prev_wd)
+
 class VerminTest(unittest.TestCase):
   """General test case class for all Vermin tests."""