gh-99108: Refactor _sha256 & _sha512 into _sha2. (#101924)

This merges their code. They're backed by the same single HACL* static library, having them be a single module simplifies maintenance.

This should unbreak the wasm enscripten builds that currently fail due to linking in --whole-archive mode and the HACL* library appearing twice.

Long unnoticed error fixed: _sha512.SHA384Type was doubly assigned and was actually SHA512Type. Nobody depends on those internal names.

Also rename LIBHACL_ make vars to LIBHACL_SHA2_ in preperation for other future HACL things.

Lib/hashlib.py

@@ -92,13 +92,13 @@ def __get_builtin_constructor(name):
             import _md5
             cache['MD5'] = cache['md5'] = _md5.md5
         elif name in {'SHA256', 'sha256', 'SHA224', 'sha224'}:
-            import _sha256
-            cache['SHA224'] = cache['sha224'] = _sha256.sha224
-            cache['SHA256'] = cache['sha256'] = _sha256.sha256
+            import _sha2
+            cache['SHA224'] = cache['sha224'] = _sha2.sha224
+            cache['SHA256'] = cache['sha256'] = _sha2.sha256
         elif name in {'SHA512', 'sha512', 'SHA384', 'sha384'}:
-            import _sha512
-            cache['SHA384'] = cache['sha384'] = _sha512.sha384
-            cache['SHA512'] = cache['sha512'] = _sha512.sha512
+            import _sha2
+            cache['SHA384'] = cache['sha384'] = _sha2.sha384
+            cache['SHA512'] = cache['sha512'] = _sha2.sha512
         elif name in {'blake2b', 'blake2s'}:
             import _blake2
             cache['blake2b'] = _blake2.blake2b

Lib/test/test_hashlib.py

@@ -1,6 +1,4 @@
-# Test hashlib module
-#
-# $Id$
+# Test the hashlib module.
 #
 #  Copyright (C) 2005-2010   Gregory P. Smith (greg@krypto.org)
 #  Licensed to PSF under a Contributor Agreement.
@@ -28,7 +26,6 @@
 from http.client import HTTPException
 
 
-# default builtin hash module
 default_builtin_hashes = {'md5', 'sha1', 'sha256', 'sha512', 'sha3', 'blake2'}
 # --with-builtin-hashlib-hashes override
 builtin_hashes = sysconfig.get_config_var("PY_BUILTIN_HASHLIB_HASHES")
@@ -66,6 +63,7 @@ def get_fips_mode():
 requires_blake2 = unittest.skipUnless(_blake2, 'requires _blake2')
 
 # bpo-46913: Don't test the _sha3 extension on a Python UBSAN build
+# TODO(gh-99108): Revisit this after _sha3 uses HACL*.
 SKIP_SHA3 = support.check_sanitizer(ub=True)
 requires_sha3 = unittest.skipUnless(not SKIP_SHA3, 'requires _sha3')
 
@@ -107,7 +105,7 @@ class HashLibTestCase(unittest.TestCase):
 
     shakes = {'shake_128', 'shake_256'}
 
-    # Issue #14693: fallback modules are always compiled under POSIX
+    # gh-58898: Fallback modules are always compiled under POSIX.
     _warn_on_extension_import = (os.name == 'posix' or support.Py_DEBUG)
 
     def _conditional_import_module(self, module_name):
@@ -116,7 +114,7 @@ def _conditional_import_module(self, module_name):
             return importlib.import_module(module_name)
         except ModuleNotFoundError as error:
             if self._warn_on_extension_import and module_name in builtin_hashes:
-                warnings.warn('Did a C extension fail to compile? %s' % error)
+                warnings.warn(f'Did a C extension fail to compile? {error}')
         return None
 
     def __init__(self, *args, **kwargs):
@@ -147,7 +145,7 @@ def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs):
         _hashlib = self._conditional_import_module('_hashlib')
         self._hashlib = _hashlib
         if _hashlib:
-            # These two algorithms should always be present when this module
+            # These algorithms should always be present when this module
             # is compiled.  If not, something was compiled wrong.
             self.assertTrue(hasattr(_hashlib, 'openssl_md5'))
             self.assertTrue(hasattr(_hashlib, 'openssl_sha1'))
@@ -172,12 +170,10 @@ def add_builtin_constructor(name):
         _sha1 = self._conditional_import_module('_sha1')
         if _sha1:
             add_builtin_constructor('sha1')
-        _sha256 = self._conditional_import_module('_sha256')
-        if _sha256:
+        _sha2 = self._conditional_import_module('_sha2')
+        if _sha2:
             add_builtin_constructor('sha224')
             add_builtin_constructor('sha256')
-        _sha512 = self._conditional_import_module('_sha512')
-        if _sha512:
             add_builtin_constructor('sha384')
             add_builtin_constructor('sha512')
         if _blake2:
@@ -460,9 +456,9 @@ def check_blocksize_name(self, name, block_size=0, digest_size=0,
                 self.assertEqual(len(m.hexdigest()), 2*digest_size)
             self.assertEqual(m.name, name)
             # split for sha3_512 / _sha3.sha3 object
-            self.assertIn(name.split("_")[0], repr(m))
+            self.assertIn(name.split("_")[0], repr(m).lower())
 
-    def test_blocksize_name(self):
+    def test_blocksize_and_name(self):
         self.check_blocksize_name('md5', 64, 16)
         self.check_blocksize_name('sha1', 64, 20)
         self.check_blocksize_name('sha224', 64, 28)

Makefile.pre.in

@@ -207,7 +207,7 @@ ENSUREPIP=      @ENSUREPIP@
 # Internal static libraries
 LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a
 LIBEXPAT_A= Modules/expat/libexpat.a
-LIBHACL_A= Modules/_hacl/libHacl_Streaming_SHA2.a
+LIBHACL_SHA2_A= Modules/_hacl/libHacl_Streaming_SHA2.a
 
 # Module state, compiler flags and linker flags
 # Empty CFLAGS and LDFLAGS are omitted.
@@ -575,10 +575,10 @@ LIBEXPAT_HEADERS= \
 ##########################################################################
 # hashlib's HACL* library
 
-LIBHACL_OBJS= \
+LIBHACL_SHA2_OBJS= \
                 Modules/_hacl/Hacl_Streaming_SHA2.o
 
-LIBHACL_HEADERS= \
+LIBHACL_SHA2_HEADERS= \
                 Modules/_hacl/Hacl_Streaming_SHA2.h \
                 Modules/_hacl/include/krml/FStar_UInt128_Verified.h \
                 Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h \
@@ -912,12 +912,12 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS)
 # Build HACL* static libraries for hashlib: libHacl_Streaming_SHA2.a
 LIBHACL_CFLAGS=-I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED)
 
-Modules/_hacl/Hacl_Streaming_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c $(LIBHACL_HEADERS)
+Modules/_hacl/Hacl_Streaming_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c $(LIBHACL_SHA2_HEADERS)
 	$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c
 
-$(LIBHACL_A): $(LIBHACL_OBJS)
+$(LIBHACL_SHA2_A): $(LIBHACL_SHA2_OBJS)
 	-rm -f $@
-	$(AR) $(ARFLAGS) $@ $(LIBHACL_OBJS)
+	$(AR) $(ARFLAGS) $@ $(LIBHACL_SHA2_OBJS)
 
 # create relative links from build/lib.platform/egg.so to Modules/egg.so
 # pybuilddir.txt is created too late. We cannot use it in Makefile
@@ -2635,9 +2635,8 @@ MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h
 MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h
 MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h
 MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h
-MODULE__SHA256_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) $(LIBHACL_A)
+MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_A)
 MODULE__SHA3_DEPS=$(srcdir)/Modules/_sha3/sha3.c $(srcdir)/Modules/_sha3/sha3.h $(srcdir)/Modules/hashlib.h
-MODULE__SHA512_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) $(LIBHACL_A)
 MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c
 MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
 MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h

Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst

@@ -0,0 +1,3 @@
+The built-in extension modules for :mod:`hashlib` SHA2 algorithms, used when
+OpenSSL does not provide them, now live in a single internal ``_sha2`` module
+instead of separate ``_sha256`` and ``_sha512`` modules.

Modules/Setup

@@ -165,8 +165,7 @@ PYTHONPATH=$(COREPYTHONPATH)
 #_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c
 #_md5 md5module.c
 #_sha1 sha1module.c
-#_sha256 sha256module.c
-#_sha512 sha512module.c
+#_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a
 #_sha3 _sha3/sha3module.c
 
 # text encodings and unicode

Modules/Setup.stdlib.in

@@ -79,8 +79,7 @@
 # hashing builtins, can be disabled with --without-builtin-hashlib-hashes
 @MODULE__MD5_TRUE@_md5 md5module.c
 @MODULE__SHA1_TRUE@_sha1 sha1module.c
-@MODULE__SHA256_TRUE@_sha256 sha256module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a
-@MODULE__SHA512_TRUE@_sha512 sha512module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a
+@MODULE__SHA2_TRUE@_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a
 @MODULE__SHA3_TRUE@_sha3 _sha3/sha3module.c
 @MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c