Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GH-110109: Move tests for pathlib.Path.walk() into main test classes. #110655

Merged
merged 3 commits into from
Oct 28, 2023
Merged

GH-110109: Move tests for pathlib.Path.walk() into main test classes. #110655

Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
17511b3
GH-110109: Move tests for `pathlib.Path.walk()` into main test classes.
barneygale Oct 10, 2023
47a81dd
Restore directory with no permissions.
barneygale Oct 10, 2023
9db41d0
Add check for path existence in `test_walk_symlink_location`
barneygale Oct 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
336 changes: 167 additions & 169 deletions Lib/test/test_pathlib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2614,6 +2614,162 @@ def test_complex_symlinks_relative(self):
def test_complex_symlinks_relative_dot_dot(self):
self._check_complex_symlinks(os.path.join('dirA', '..'))

def setUpWalk(self):
# Build:
# TESTFN/
# TEST1/ a file kid and two directory kids
# tmp1
# SUB1/ a file kid and a directory kid
# tmp2
# SUB11/ no kids
# SUB2/ a file kid and a dirsymlink kid
# tmp3
# link/ a symlink to TEST2
# broken_link
# broken_link2
# TEST2/
# tmp4 a lone file
self.walk_path = self.cls(BASE, "TEST1")
self.sub1_path = self.walk_path / "SUB1"
self.sub11_path = self.sub1_path / "SUB11"
self.sub2_path = self.walk_path / "SUB2"
tmp1_path = self.walk_path / "tmp1"
tmp2_path = self.sub1_path / "tmp2"
tmp3_path = self.sub2_path / "tmp3"
self.link_path = self.sub2_path / "link"
t2_path = self.cls(BASE, "TEST2")
tmp4_path = self.cls(BASE, "TEST2", "tmp4")
broken_link_path = self.sub2_path / "broken_link"
broken_link2_path = self.sub2_path / "broken_link2"

self.sub11_path.mkdir(parents=True)
self.sub2_path.mkdir(parents=True)
t2_path.mkdir(parents=True)

for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path:
with path.open("w", encoding='utf-8') as f:
f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n")

if self.can_symlink:
self.link_path.symlink_to(t2_path)
broken_link_path.symlink_to('broken')
broken_link2_path.symlink_to(self.cls('tmp3', 'broken'))
self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"])
else:
self.sub2_tree = (self.sub2_path, [], ["tmp3"])

def test_walk_topdown(self):
self.setUpWalk()
walker = self.walk_path.walk()
entry = next(walker)
entry[1].sort() # Ensure we visit SUB1 before SUB2
self.assertEqual(entry, (self.walk_path, ["SUB1", "SUB2"], ["tmp1"]))
entry = next(walker)
self.assertEqual(entry, (self.sub1_path, ["SUB11"], ["tmp2"]))
entry = next(walker)
self.assertEqual(entry, (self.sub11_path, [], []))
entry = next(walker)
entry[1].sort()
entry[2].sort()
self.assertEqual(entry, self.sub2_tree)
with self.assertRaises(StopIteration):
next(walker)

def test_walk_prune(self):
self.setUpWalk()
# Prune the search.
all = []
for root, dirs, files in self.walk_path.walk():
all.append((root, dirs, files))
if 'SUB1' in dirs:
# Note that this also mutates the dirs we appended to all!
dirs.remove('SUB1')

self.assertEqual(len(all), 2)
self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"]))

all[1][-1].sort()
all[1][1].sort()
self.assertEqual(all[1], self.sub2_tree)

def test_walk_bottom_up(self):
self.setUpWalk()
seen_testfn = seen_sub1 = seen_sub11 = seen_sub2 = False
for path, dirnames, filenames in self.walk_path.walk(top_down=False):
if path == self.walk_path:
self.assertFalse(seen_testfn)
self.assertTrue(seen_sub1)
self.assertTrue(seen_sub2)
self.assertEqual(sorted(dirnames), ["SUB1", "SUB2"])
self.assertEqual(filenames, ["tmp1"])
seen_testfn = True
elif path == self.sub1_path:
self.assertFalse(seen_testfn)
self.assertFalse(seen_sub1)
self.assertTrue(seen_sub11)
self.assertEqual(dirnames, ["SUB11"])
self.assertEqual(filenames, ["tmp2"])
seen_sub1 = True
elif path == self.sub11_path:
self.assertFalse(seen_sub1)
self.assertFalse(seen_sub11)
self.assertEqual(dirnames, [])
self.assertEqual(filenames, [])
seen_sub11 = True
elif path == self.sub2_path:
self.assertFalse(seen_testfn)
self.assertFalse(seen_sub2)
self.assertEqual(sorted(dirnames), sorted(self.sub2_tree[1]))
self.assertEqual(sorted(filenames), sorted(self.sub2_tree[2]))
seen_sub2 = True
else:
raise AssertionError(f"Unexpected path: {path}")
self.assertTrue(seen_testfn)

def test_walk_follow_symlinks(self):
if not self.can_symlink:
self.skipTest("symlinks required")
self.setUpWalk()
walk_it = self.walk_path.walk(follow_symlinks=True)
for root, dirs, files in walk_it:
if root == self.link_path:
self.assertEqual(dirs, [])
self.assertEqual(files, ["tmp4"])
break
else:
self.fail("Didn't follow symlink with follow_symlinks=True")

def test_walk_symlink_location(self):
if not self.can_symlink:
self.skipTest("symlinks required")
self.setUpWalk()
# Tests whether symlinks end up in filenames or dirnames depending
# on the `follow_symlinks` argument.
walk_it = self.walk_path.walk(follow_symlinks=False)
for root, dirs, files in walk_it:
if root == self.sub2_path:
self.assertIn("link", files)
break
else:
self.fail("symlink not found")

walk_it = self.walk_path.walk(follow_symlinks=True)
for root, dirs, files in walk_it:
if root == self.sub2_path:
self.assertIn("link", dirs)
break
barneygale marked this conversation as resolved.
Show resolved Hide resolved

def test_walk_above_recursion_limit(self):
recursion_limit = 40
# directory_depth > recursion_limit
directory_depth = recursion_limit + 10
base = self.cls(BASE, 'deep')
path = self.cls(base, *(['d'] * directory_depth))
path.mkdir(parents=True)

with set_recursion_limit(recursion_limit):
list(base.walk())
list(base.walk(top_down=False))

class DummyPathWithSymlinks(DummyPath):
def readlink(self):
Expand Down Expand Up @@ -3187,178 +3343,32 @@ def test_passing_kwargs_deprecated(self):
with self.assertWarns(DeprecationWarning):
self.cls(foo="bar")


class WalkTests(unittest.TestCase):

def setUp(self):
self.addCleanup(os_helper.rmtree, os_helper.TESTFN)

# Build:
# TESTFN/
# TEST1/ a file kid and two directory kids
# tmp1
# SUB1/ a file kid and a directory kid
# tmp2
# SUB11/ no kids
# SUB2/ a file kid and a dirsymlink kid
# tmp3
# SUB21/ not readable
# tmp5
# link/ a symlink to TEST2
# broken_link
# broken_link2
# broken_link3
# TEST2/
# tmp4 a lone file
self.walk_path = pathlib.Path(os_helper.TESTFN, "TEST1")
self.sub1_path = self.walk_path / "SUB1"
self.sub11_path = self.sub1_path / "SUB11"
self.sub2_path = self.walk_path / "SUB2"
def setUpWalk(self):
super().setUpWalk()
sub21_path= self.sub2_path / "SUB21"
tmp1_path = self.walk_path / "tmp1"
tmp2_path = self.sub1_path / "tmp2"
tmp3_path = self.sub2_path / "tmp3"
tmp5_path = sub21_path / "tmp3"
self.link_path = self.sub2_path / "link"
t2_path = pathlib.Path(os_helper.TESTFN, "TEST2")
tmp4_path = pathlib.Path(os_helper.TESTFN, "TEST2", "tmp4")
broken_link_path = self.sub2_path / "broken_link"
broken_link2_path = self.sub2_path / "broken_link2"
broken_link3_path = self.sub2_path / "broken_link3"

os.makedirs(self.sub11_path)
os.makedirs(self.sub2_path)
os.makedirs(sub21_path)
os.makedirs(t2_path)

for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path:
with open(path, "x", encoding='utf-8') as f:
f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n")

if os_helper.can_symlink():
os.symlink(os.path.abspath(t2_path), self.link_path)
os.symlink('broken', broken_link_path, True)
os.symlink(pathlib.Path('tmp3', 'broken'), broken_link2_path, True)
os.symlink(pathlib.Path('SUB21', 'tmp5'), broken_link3_path, True)
self.sub2_tree = (self.sub2_path, ["SUB21"],
["broken_link", "broken_link2", "broken_link3",
"link", "tmp3"])
else:
self.sub2_tree = (self.sub2_path, ["SUB21"], ["tmp3"])

tmp5_path.write_text("I am tmp5, blame test_pathlib.")
if self.can_symlink:
os.symlink(tmp5_path, broken_link3_path)
self.sub2_tree[2].append('broken_link3')
self.sub2_tree[2].sort()
if not is_emscripten:
# Emscripten fails with inaccessible directories.
os.chmod(sub21_path, 0)
try:
os.listdir(sub21_path)
except PermissionError:
self.addCleanup(os.chmod, sub21_path, stat.S_IRWXU)
self.sub2_tree[1].append('SUB21')
else:
os.chmod(sub21_path, stat.S_IRWXU)
os.unlink(tmp5_path)
os.rmdir(sub21_path)
del self.sub2_tree[1][:1]

def test_walk_topdown(self):
walker = self.walk_path.walk()
entry = next(walker)
entry[1].sort() # Ensure we visit SUB1 before SUB2
self.assertEqual(entry, (self.walk_path, ["SUB1", "SUB2"], ["tmp1"]))
entry = next(walker)
self.assertEqual(entry, (self.sub1_path, ["SUB11"], ["tmp2"]))
entry = next(walker)
self.assertEqual(entry, (self.sub11_path, [], []))
entry = next(walker)
entry[1].sort()
entry[2].sort()
self.assertEqual(entry, self.sub2_tree)
with self.assertRaises(StopIteration):
next(walker)

def test_walk_prune(self, walk_path=None):
if walk_path is None:
walk_path = self.walk_path
# Prune the search.
all = []
for root, dirs, files in walk_path.walk():
all.append((root, dirs, files))
if 'SUB1' in dirs:
# Note that this also mutates the dirs we appended to all!
dirs.remove('SUB1')

self.assertEqual(len(all), 2)
self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"]))

all[1][-1].sort()
all[1][1].sort()
self.assertEqual(all[1], self.sub2_tree)

def test_file_like_path(self):
self.test_walk_prune(FakePath(self.walk_path).__fspath__())

def test_walk_bottom_up(self):
seen_testfn = seen_sub1 = seen_sub11 = seen_sub2 = False
for path, dirnames, filenames in self.walk_path.walk(top_down=False):
if path == self.walk_path:
self.assertFalse(seen_testfn)
self.assertTrue(seen_sub1)
self.assertTrue(seen_sub2)
self.assertEqual(sorted(dirnames), ["SUB1", "SUB2"])
self.assertEqual(filenames, ["tmp1"])
seen_testfn = True
elif path == self.sub1_path:
self.assertFalse(seen_testfn)
self.assertFalse(seen_sub1)
self.assertTrue(seen_sub11)
self.assertEqual(dirnames, ["SUB11"])
self.assertEqual(filenames, ["tmp2"])
seen_sub1 = True
elif path == self.sub11_path:
self.assertFalse(seen_sub1)
self.assertFalse(seen_sub11)
self.assertEqual(dirnames, [])
self.assertEqual(filenames, [])
seen_sub11 = True
elif path == self.sub2_path:
self.assertFalse(seen_testfn)
self.assertFalse(seen_sub2)
self.assertEqual(sorted(dirnames), sorted(self.sub2_tree[1]))
self.assertEqual(sorted(filenames), sorted(self.sub2_tree[2]))
seen_sub2 = True
else:
raise AssertionError(f"Unexpected path: {path}")
self.assertTrue(seen_testfn)

@os_helper.skip_unless_symlink
def test_walk_follow_symlinks(self):
walk_it = self.walk_path.walk(follow_symlinks=True)
for root, dirs, files in walk_it:
if root == self.link_path:
self.assertEqual(dirs, [])
self.assertEqual(files, ["tmp4"])
break
else:
self.fail("Didn't follow symlink with follow_symlinks=True")

@os_helper.skip_unless_symlink
def test_walk_symlink_location(self):
# Tests whether symlinks end up in filenames or dirnames depending
# on the `follow_symlinks` argument.
walk_it = self.walk_path.walk(follow_symlinks=False)
for root, dirs, files in walk_it:
if root == self.sub2_path:
self.assertIn("link", files)
break
else:
self.fail("symlink not found")

walk_it = self.walk_path.walk(follow_symlinks=True)
for root, dirs, files in walk_it:
if root == self.sub2_path:
self.assertIn("link", dirs)
break

def test_walk_bad_dir(self):
self.setUpWalk()
errors = []
walk_it = self.walk_path.walk(on_error=errors.append)
root, dirs, files = next(walk_it)
Expand All @@ -3380,8 +3390,8 @@ def test_walk_bad_dir(self):

def test_walk_many_open_files(self):
depth = 30
base = pathlib.Path(os_helper.TESTFN, 'deep')
path = pathlib.Path(base, *(['d']*depth))
base = self.cls(BASE, 'deep')
path = self.cls(base, *(['d']*depth))
path.mkdir(parents=True)

iters = [base.walk(top_down=False) for _ in range(100)]
Expand All @@ -3399,18 +3409,6 @@ def test_walk_many_open_files(self):
self.assertEqual(next(it), expected)
path = path / 'd'

def test_walk_above_recursion_limit(self):
recursion_limit = 40
# directory_depth > recursion_limit
directory_depth = recursion_limit + 10
base = pathlib.Path(os_helper.TESTFN, 'deep')
path = pathlib.Path(base, *(['d'] * directory_depth))
path.mkdir(parents=True)

with set_recursion_limit(recursion_limit):
list(base.walk())
list(base.walk(top_down=False))


@only_posix
class PosixPathTest(PathTest):
Expand Down