bpo-25643: Refactor the C tokenizer into smaller, logical units (GH-2…

…5050)
python · Mar 28, 2021 · 261a452 · 261a452
1 parent fb1d01b
commit 261a452
Show file tree

Hide file tree

Showing 4 changed files with 353 additions and 359 deletions.
diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py
@@ -55,13 +55,13 @@ def test_line_continuation_EOF_from_file_bpo2180(self):
             file_name = script_helper.make_script(temp_dir, 'foo', '\\')
             rc, out, err = script_helper.assert_python_failure(file_name)
             self.assertIn(b'unexpected EOF while parsing', err)
-            self.assertIn(b'line 2', err)
+            self.assertIn(b'line 1', err)
             self.assertIn(b'\\', err)
 
             file_name = script_helper.make_script(temp_dir, 'foo', 'y = 6\\')
             rc, out, err = script_helper.assert_python_failure(file_name)
             self.assertIn(b'unexpected EOF while parsing', err)
-            self.assertIn(b'line 2', err)
+            self.assertIn(b'line 1', err)
             self.assertIn(b'y = 6\\', err)
 
 if __name__ == "__main__":

diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py
@@ -205,6 +205,23 @@ def test_utf8_bom_and_utf8_coding_line(self):
                b'print(ascii("\xc3\xa4"))\n')
         self.check_script_output(src, br"'\xe4'")
 
+    def test_crlf(self):
+        src = (b'print(ascii("""\r\n"""))\n')
+        out = self.check_script_output(src, br"'\n'")
+
+    def test_crcrlf(self):
+        src = (b'print(ascii("""\r\r\n"""))\n')
+        out = self.check_script_output(src, br"'\n\n'")
+
+    def test_crcrcrlf(self):
+        src = (b'print(ascii("""\r\r\r\n"""))\n')
+        out = self.check_script_output(src, br"'\n\n\n'")
+
+    def test_crcrcrlf2(self):
+        src = (b'#coding:iso-8859-1\n'
+               b'print(ascii("""\r\r\r\n"""))\n')
+        out = self.check_script_output(src, br"'\n\n\n'")
+
 
 class BytesSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase):