diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 1272e1e9be002e..47f2c06685bcaa 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3,7 +3,7 @@ from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP, STRING, ENDMARKER, ENCODING, tok_name, detect_encoding, open as tokenize_open, Untokenizer, generate_tokens, - NEWLINE, _generate_tokens_from_c_tokenizer) + NEWLINE, _generate_tokens_from_c_tokenizer, DEDENT) from io import BytesIO, StringIO import unittest from textwrap import dedent @@ -2512,6 +2512,26 @@ def get_tokens(string): self.assertRaises(SyntaxError, get_tokens, "("*1000+"a"+")"*1000) self.assertRaises(SyntaxError, get_tokens, "]") + def test_max_indent(self): + MAXINDENT = 100 + + def generate_source(indents): + source = ''.join((' ' * x) + 'if True:\n' for x in range(indents)) + source += ' ' * indents + 'pass\n' + return source + + valid = generate_source(MAXINDENT - 1) + tokens = list(_generate_tokens_from_c_tokenizer(valid)) + self.assertEqual(tokens[-1].type, DEDENT) + compile(valid, "", "exec") + + invalid = generate_source(MAXINDENT) + tokens = list(_generate_tokens_from_c_tokenizer(invalid)) + self.assertEqual(tokens[-1].type, NEWLINE) + self.assertRaises( + IndentationError, compile, invalid, "", "exec" + ) + def test_continuation_lines_indentation(self): def get_tokens(string): return [(kind, string) for (kind, string, *_) in _generate_tokens_from_c_tokenizer(string)]