add blake3 to hashlib

blake3_impl.c and blake3module.c are adapted from the existing BLAKE2
module. This involves a lot of copy-paste, and hopefully someone who
knows this code better can help me clean them up. (In particular, BLAKE2
relies on clinic codegen to share code between BLAKE2b and BLAKE2s, but
BLAKE3 has no need for that.)

blake3_dispatch.c, which is vendored from upstream, includes runtime CPU
feature detection to choose the appropriate SIMD instruction set for the
current platform (x86 only). In this model, the build should include all
instruction sets, and here I unconditionally include the Unix assembly
files (*_unix.S) as `extra_objects` in setup.py. This "works on my box",
but is currently incomplete in several ways:

- It needs some Windows-specific build logic. There are two additional
  assembly flavors included for each instruction set, *_windows_gnu.S
  and *_windows_msvc.asm. I need to figure out how to include the right
  flavor based on the target OS/ABI.
- I need to figure out how to omit these files on non-x86-64 platforms.
  x86-32 will require some explicit preprocessor definitions to restrict
  blake3_dispatch.c to portable code. (Unless we vendor intrinsics-based
  implementations for 32-bit support. More on this below.)
- It's not going to work on compilers that are too old to recognize
  these instruction sets, particularly AVX-512. (Question: What's the
  oldest GCC version that CPython supports?) Maybe compiler feature
  detection could be added to ./configure and somehow plumbed through to
  setup.py.

I'm hoping someone more experienced with the build system can help me
narrow down the best solution for each of those.

This also raises the higher level question of whether the CPython
project feels comfortable about including assembly files in general. As
a possible alternative, the upstream BLAKE3 project also provides
intrinsics-based implementations of the same optimizations. The upsides
of these are 1) that they don't require Unix/Windows platform detection,
2) that they support 32-bit x86 targets, and 3) that C is easier to
audit than assembly. However, the downsides of these are 1) that they're
~10% slower than the hand-written assembly, 2) that their performance is
less consistent and worse on older compilers, and 3) that they take
noticeably longer to compile. We recommend the assembly implementations
for these reasons, but intrinsics are a viable option if assembly
violates CPython's requirements.

Author: oconnor663

Lib/hashlib.py

@@ -12,7 +12,7 @@
 than using new(name):
 
 md5(), sha1(), sha224(), sha256(), sha384(), sha512(), blake2b(), blake2s(),
-sha3_224, sha3_256, sha3_384, sha3_512, shake_128, and shake_256.
+blake3(), sha3_224, sha3_256, sha3_384, sha3_512, shake_128, and shake_256.
 
 More algorithms may be available on your platform but the above are guaranteed
 to exist.  See the algorithms_guaranteed and algorithms_available attributes
@@ -56,7 +56,7 @@
 # This tuple and __get_builtin_constructor() must be modified if a new
 # always available algorithm is added.
 __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512',
-                      'blake2b', 'blake2s',
+                      'blake2b', 'blake2s', 'blake3',
                       'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
                       'shake_128', 'shake_256')
 
@@ -75,8 +75,9 @@
 # implementations neither support keyed blake2 (blake2 MAC) nor advanced
 # features like salt, personalization, or tree hashing. OpenSSL hash-only
 # variants are available as 'blake2b512' and 'blake2s256', though.
+# OpenSSL 1.1.0 does not support blake3.
 __block_openssl_constructor = {
-    'blake2b', 'blake2s',
+    'blake2b', 'blake2s', 'blake3'
 }
 
 def __get_builtin_constructor(name):
@@ -103,6 +104,9 @@ def __get_builtin_constructor(name):
             import _blake2
             cache['blake2b'] = _blake2.blake2b
             cache['blake2s'] = _blake2.blake2s
+        elif name in {'blake3'}:
+            import _blake3
+            cache['blake3'] = _blake3.blake3
         elif name in {'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512'}:
             import _sha3
             cache['sha3_224'] = _sha3.sha3_224

Lib/test/test_hashlib.py

@@ -92,7 +92,7 @@ class HashLibTestCase(unittest.TestCase):
     supported_hash_names = ( 'md5', 'MD5', 'sha1', 'SHA1',
                              'sha224', 'SHA224', 'sha256', 'SHA256',
                              'sha384', 'SHA384', 'sha512', 'SHA512',
-                             'blake2b', 'blake2s',
+                             'blake2b', 'blake2s', 'blake3',
                              'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
                              'shake_128', 'shake_256')
 
@@ -172,6 +172,9 @@ def add_builtin_constructor(name):
         if _blake2:
             add_builtin_constructor('blake2s')
             add_builtin_constructor('blake2b')
+        _blake3 = self._conditional_import_module('_blake3')
+        if _blake3:
+            add_builtin_constructor('blake3')
 
         _sha3 = self._conditional_import_module('_sha3')
         if _sha3:
@@ -342,25 +345,25 @@ def test_large_update(self):
             self.assertEqual(m1.digest(*args), m4_copy.digest(*args))
             self.assertEqual(m4.digest(*args), m4_digest)
 
-    def check(self, name, data, hexdigest, shake=False, **kwargs):
+    def check(self, name, data, hexdigest, set_length=False, **kwargs):
         length = len(hexdigest)//2
         hexdigest = hexdigest.lower()
         constructors = self.constructors_to_test[name]
         # 2 is for hashlib.name(...) and hashlib.new(name, ...)
         self.assertGreaterEqual(len(constructors), 2)
         for hash_object_constructor in constructors:
             m = hash_object_constructor(data, **kwargs)
-            computed = m.hexdigest() if not shake else m.hexdigest(length)
+            computed = m.hexdigest() if not set_length else m.hexdigest(length)
             self.assertEqual(
                     computed, hexdigest,
                     "Hash algorithm %s constructed using %s returned hexdigest"
                     " %r for %d byte input data that should have hashed to %r."
                     % (name, hash_object_constructor,
                        computed, len(data), hexdigest))
-            computed = m.digest() if not shake else m.digest(length)
+            computed = m.digest() if not set_length else m.digest(length)
             digest = bytes.fromhex(hexdigest)
             self.assertEqual(computed, digest)
-            if not shake:
+            if not set_length:
                 self.assertEqual(len(digest), m.digest_size)
 
     def check_no_unicode(self, algorithm_name):
@@ -776,6 +779,73 @@ def test_blake2s_vectors(self):
             key = bytes.fromhex(key)
             self.check('blake2s', msg, md, key=key)
 
+    def test_case_blake3_0(self):
+        self.check('blake3', b"",
+          "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262")
+
+    def test_case_blake3_1(self):
+        self.check('blake3', b"abc",
+          "6437b3ac38465133ffb63b75273a8db548c558465d79db03fd359c6cd5bd9d85")
+
+    def test_case_blake3_keyed(self):
+        self.check('blake3', b"abc",
+          "6da54495d8152f2bcba87bd7282df70901cdb66b4448ed5f4c7bd2852b8b5532",
+          key=bytes(range(32)))
+
+    def test_case_blake3_derive_key(self):
+        self.check('blake3', b"super secret key material",
+          "dbf0a1433e0137fb11b71d3ae3c138bff46445936dd5d4f01f403c23abd5660a",
+          derive_key_context="hardcoded, globally unique, application-specific context string")
+
+    def test_case_blake3_length(self):
+        self.check('blake3', b"",
+          "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" +
+          "e00f03e7b69af26b7faaf09fcd333050338ddfe085b8cc869ca98b206c08243a" +
+          "26f5487789e8f660afe6c99ef9e0c52b92e7393024a80459cf91f476f9ffdbda" +
+          "7001c22e159b402631f277ca96f2defdf1078282314e763699a31c5363165421" +
+          "cce14d",
+          # True here means that digest() and hexdigest() get a length
+          # argument. This shares the variable length test codepath with
+          # shake_128 and shake_256.
+          True)
+
+    def test_case_blake3_seek(self):
+        # None of the other hashes support a seek parameter. Rather than
+        # hacking this into self.check(), just invoke blake3() explicitly.
+        output_hex = (
+            "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" +
+            "e00f03e7b69af26b7faaf09fcd333050338ddfe085b8cc869ca98b206c08243a" +
+            "26f5487789e8f660afe6c99ef9e0c52b92e7393024a80459cf91f476f9ffdbda" +
+            "7001c22e159b402631f277ca96f2defdf1078282314e763699a31c5363165421" +
+            "cce14d")
+        output_bytes = unhexlify(output_hex)
+        # Test a few interesting seek points, including one with length=0.
+        for seek in [0, 1, len(output_bytes)//2, len(output_bytes)-1, len(output_bytes)]:
+            length = len(output_bytes) - seek
+            expected_bytes = output_bytes[seek:]
+            expected_hex = output_hex[2*seek:]
+            # positional
+            assert expected_bytes == hashlib.blake3().digest(length, seek)
+            assert expected_hex == hashlib.blake3().hexdigest(length, seek)
+            # keywords
+            assert expected_bytes == hashlib.blake3().digest(length=length, seek=seek)
+            assert expected_hex == hashlib.blake3().hexdigest(length=length, seek=seek)
+
+    def test_case_blake3_key_must_be_32_bytes(self):
+        for length in [0, 1, 31, 33]:
+            try:
+                hashlib.blake3(key=b"\0"*31)
+                assert False, "the line above should raise an exception"
+            except ValueError as e:
+                assert str(e) == "key must be 32 bytes"
+
+    def test_case_blake3_keyed_derive_key_exclusive(self):
+        try:
+            hashlib.blake3(key=b"\0"*32, derive_key_context="foo")
+            assert False, "the line above should raise an exception"
+        except ValueError as e:
+            assert str(e) == "key and derive_key_context can't be used together"
+
     def test_case_sha3_224_0(self):
         self.check('sha3_224', b"",
           "6b4e03423667dbb73b6e15454f0eb1abd4597f9a1b078e3f5b5a6bc7")