pythongh-99581: Fix a buffer overflow in the tokenizer when copying l…

…ines that fill the available buffer

Lib/test/test_tokenize.py

@@ -10,6 +10,7 @@
 from unittest import TestCase, mock
 from test.test_grammar import (VALID_UNDERSCORE_LITERALS,
                                INVALID_UNDERSCORE_LITERALS)
+from test.support.script_helper import assert_python_ok
 import os
 import token
 
@@ -1026,7 +1027,7 @@ def test_utf8_coding_cookie_and_utf8_bom(self):
     def test_bad_coding_cookie(self):
         self.assertRaises(SyntaxError, self._testFile, 'bad_coding.py')
         self.assertRaises(SyntaxError, self._testFile, 'bad_coding2.py')
-
+    
 
 class Test_Tokenize(TestCase):
 
@@ -2631,5 +2632,13 @@ def fib(n):
         self.assertEqual(get_tokens(code), get_tokens(code_no_cont))
 
 
+class CTokenizerBufferTests(unittest.TestCase):
+    def test_newline_at_the_end_of_buffer(self):
+        # See issue 99581
+        filename = 'tokenize_tests-reflow-buffer-missing-newline.txt'
+        path = os.path.join(os.path.dirname(__file__), filename)
+        assert_python_ok(path)
+
+
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()

Misc/NEWS.d/next/Core and Builtins/2022-11-19-22-27-52.gh-issue-99581.yKYPbf.rst

@@ -0,0 +1,3 @@
+Fixed a bug that was causing a buffer overflow if the tokenizer copies a
+line missing the newline caracter from a file that is as long as the
+available tokenizer buffer. Patch by Pablo galindo

Parser/tokenizer.c

@@ -413,7 +413,11 @@ tok_readline_recode(struct tok_state *tok) {
         error_ret(tok);
         goto error;
     }
-    if (!tok_reserve_buf(tok, buflen + 1)) {
+    // Make room for the null terminator *and* potentially
+    // an extra newline character that we may need to artificially
+    // add.
+    size_t buffer_size = buflen + 2;
+    if (!tok_reserve_buf(tok, buffer_size)) {
         goto error;
     }
     memcpy(tok->inp, buf, buflen);
@@ -1000,6 +1004,7 @@ tok_underflow_file(struct tok_state *tok) {
         return 0;
     }
     if (tok->inp[-1] != '\n') {
+        assert(tok->inp + 1 < tok->end);
         /* Last line does not end in \n, fake one */
         *tok->inp++ = '\n';
         *tok->inp = '\0';