pythongh-92728: Restore re.template, but deprecate it (pythonGH-93161)

Revert "bpo-47211: Remove function re.template() and flag re.TEMPLATE (pythonGH-32300)"

This reverts commit b09184b.
(cherry picked from commit 16a7e4a)

Co-authored-by: Miro Hrončok <miro@hroncok.cz>

Doc/whatsnew/3.11.rst

@@ -1273,6 +1273,11 @@ Deprecated
   is now deprecated. Support will be removed in Python 3.13. (Contributed by
   Jingchen Ye in :gh:`90224`.)
 
+* The :func:`re.template` function and the corresponding :const:`re.TEMPLATE`
+  and :const:`re.T` flags are deprecated, as they were undocumented and
+  lacked an obvious purpose. They will be removed in Python 3.13.
+  (Contributed by Serhiy Storchaka and Miro Hrončok in :gh:`92728`.)
+
 
 Pending Removal in Python 3.12
 ==============================

Lib/re/__init__.py

@@ -129,7 +129,7 @@
 # public symbols
 __all__ = [
     "match", "fullmatch", "search", "sub", "subn", "split",
-    "findall", "finditer", "compile", "purge", "escape",
+    "findall", "finditer", "compile", "purge", "template", "escape",
     "error", "Pattern", "Match", "A", "I", "L", "M", "S", "X", "U",
     "ASCII", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE",
     "UNICODE", "NOFLAG", "RegexFlag",
@@ -148,6 +148,8 @@ class RegexFlag:
     MULTILINE = M = _compiler.SRE_FLAG_MULTILINE # make anchors look for newline
     DOTALL = S = _compiler.SRE_FLAG_DOTALL # make dot match newline
     VERBOSE = X = _compiler.SRE_FLAG_VERBOSE # ignore whitespace and comments
+    # sre extensions (experimental, don't rely on these)
+    TEMPLATE = T = _compiler.SRE_FLAG_TEMPLATE # unknown purpose, deprecated
     DEBUG = _compiler.SRE_FLAG_DEBUG # dump pattern after compilation
     __str__ = object.__str__
     _numeric_repr_ = hex
@@ -229,6 +231,18 @@ def purge():
     _cache.clear()
     _compile_repl.cache_clear()
 
+def template(pattern, flags=0):
+    "Compile a template pattern, returning a Pattern object, deprecated"
+    import warnings
+    warnings.warn("The re.template() function is deprecated "
+                  "as it is an undocumented function "
+                  "without an obvious purpose. "
+                  "Use re.compile() instead.",
+                  DeprecationWarning)
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", DeprecationWarning)  # warn just once
+        return _compile(pattern, flags|T)
+
 # SPECIAL_CHARS
 # closing ')', '}' and ']'
 # '-' (a range in character set)
@@ -270,6 +284,13 @@ def _compile(pattern, flags):
         return pattern
     if not _compiler.isstring(pattern):
         raise TypeError("first argument must be string or compiled pattern")
+    if flags & T:
+        import warnings
+        warnings.warn("The re.TEMPLATE/re.T flag is deprecated "
+                  "as it is an undocumented flag "
+                  "without an obvious purpose. "
+                  "Don't use it.",
+                  DeprecationWarning)
     p = _compiler.compile(pattern, flags)
     if not (flags & DEBUG):
         if len(_cache) >= _MAXCACHE:

Lib/re/_compiler.py

@@ -108,6 +108,8 @@ def _compile(data, pattern, flags):
             else:
                 emit(ANY)
         elif op in REPEATING_CODES:
+            if flags & SRE_FLAG_TEMPLATE:
+                raise error("internal: unsupported template operator %r" % (op,))
             if _simple(av[2]):
                 emit(REPEATING_CODES[op][2])
                 skip = _len(code); emit(0)

Lib/re/_constants.py

@@ -204,6 +204,7 @@ def _makecodes(*names):
 }
 
 # flags
+SRE_FLAG_TEMPLATE = 1 # template mode (unknown purpose, deprecated)
 SRE_FLAG_IGNORECASE = 2 # case insensitive
 SRE_FLAG_LOCALE = 4 # honour system locale
 SRE_FLAG_MULTILINE = 8 # treat target as multiline string

Lib/re/_parser.py

@@ -61,11 +61,12 @@
     "x": SRE_FLAG_VERBOSE,
     # extensions
     "a": SRE_FLAG_ASCII,
+    "t": SRE_FLAG_TEMPLATE,
     "u": SRE_FLAG_UNICODE,
 }
 
 TYPE_FLAGS = SRE_FLAG_ASCII | SRE_FLAG_LOCALE | SRE_FLAG_UNICODE
-GLOBAL_FLAGS = SRE_FLAG_DEBUG
+GLOBAL_FLAGS = SRE_FLAG_DEBUG | SRE_FLAG_TEMPLATE
 
 class State:
     # keeps track of state for parsing

Lib/test/test_re.py

@@ -2417,6 +2417,30 @@ def test_bug_gh91616(self):
         self.assertTrue(re.fullmatch(r'(?s:(?>.*?\.).*)\Z', "a.txt")) # reproducer
         self.assertTrue(re.fullmatch(r'(?s:(?=(?P<g0>.*?\.))(?P=g0).*)\Z', "a.txt"))
 
+    def test_template_function_and_flag_is_deprecated(self):
+        with self.assertWarns(DeprecationWarning) as cm:
+            template_re1 = re.template(r'a')
+        self.assertIn('re.template()', str(cm.warning))
+        self.assertIn('is deprecated', str(cm.warning))
+        self.assertIn('function', str(cm.warning))
+        self.assertNotIn('flag', str(cm.warning))
+
+        with self.assertWarns(DeprecationWarning) as cm:
+            # we deliberately use more flags here to test that that still
+            # triggers the warning
+            # if paranoid, we could test multiple different combinations,
+            # but it's probably not worth it
+            template_re2 = re.compile(r'a', flags=re.TEMPLATE|re.UNICODE)
+        self.assertIn('re.TEMPLATE', str(cm.warning))
+        self.assertIn('is deprecated', str(cm.warning))
+        self.assertIn('flag', str(cm.warning))
+        self.assertNotIn('function', str(cm.warning))
+
+        # while deprecated, is should still function
+        self.assertEqual(template_re1, template_re2)
+        self.assertTrue(template_re1.match('ahoy'))
+        self.assertFalse(template_re1.match('nope'))
+
 
 def get_debug_out(pat):
     with captured_stdout() as out:
@@ -2611,11 +2635,11 @@ def test_flags_repr(self):
                          "re.IGNORECASE|re.DOTALL|re.VERBOSE|0x100000")
         self.assertEqual(
                 repr(~re.I),
-                "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.DOTALL|re.VERBOSE|re.DEBUG|0x1")
+                "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.DOTALL|re.VERBOSE|re.TEMPLATE|re.DEBUG")
         self.assertEqual(repr(~(re.I|re.S|re.X)),
-                         "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.DEBUG|0x1")
+                         "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.TEMPLATE|re.DEBUG")
         self.assertEqual(repr(~(re.I|re.S|re.X|(1<<20))),
-                         "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.DEBUG|0xffe01")
+                         "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.TEMPLATE|re.DEBUG|0xffe00")
 
 
 class ImplementationTest(unittest.TestCase):

Misc/NEWS.d/3.11.0b1.rst

@@ -1373,6 +1373,7 @@ Suppress expression chaining for more :mod:`re` parsing errors.
 
 Remove undocumented and never working function ``re.template()`` and flag
 ``re.TEMPLATE``.
+This was later reverted in 3.11.0b2 and deprecated instead.
 
 ..
 

Misc/NEWS.d/next/Library/2022-05-24-10-59-02.gh-issue-92728.zxTifq.rst

@@ -0,0 +1,3 @@
+The :func:`re.template` function and the corresponding :const:`re.TEMPLATE`
+and :const:`re.T` flags are restored after they were removed in 3.11.0b1,
+but they are now deprecated, so they might be removed from Python 3.13.

Modules/_sre/sre.c

@@ -1323,6 +1323,7 @@ pattern_repr(PatternObject *obj)
         const char *name;
         int value;
     } flag_names[] = {
+        {"re.TEMPLATE", SRE_FLAG_TEMPLATE},
         {"re.IGNORECASE", SRE_FLAG_IGNORECASE},
         {"re.LOCALE", SRE_FLAG_LOCALE},
         {"re.MULTILINE", SRE_FLAG_MULTILINE},

Modules/_sre/sre_constants.h

@@ -85,6 +85,7 @@
 #define SRE_CATEGORY_UNI_NOT_WORD 15
 #define SRE_CATEGORY_UNI_LINEBREAK 16
 #define SRE_CATEGORY_UNI_NOT_LINEBREAK 17
+#define SRE_FLAG_TEMPLATE 1
 #define SRE_FLAG_IGNORECASE 2
 #define SRE_FLAG_LOCALE 4
 #define SRE_FLAG_MULTILINE 8