-
-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Apply upstream patch to soften
crc32c
dependency
- Loading branch information
Showing
3 changed files
with
251 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
From 4b3dc36f8e96d91e4efcfb8f231863505f9a1594 Mon Sep 17 00:00:00 2001 | ||
From: jakirkham <jakirkham@gmail.com> | ||
Date: Wed, 13 Nov 2024 10:37:04 -0800 | ||
Subject: [PATCH] Skip `checksum32` tests if `crc32` is missing | ||
|
||
--- | ||
numcodecs/tests/test_checksum32.py | 6 +++++- | ||
1 file changed, 5 insertions(+), 1 deletion(-) | ||
|
||
diff --git a/numcodecs/tests/test_checksum32.py b/numcodecs/tests/test_checksum32.py | ||
index 7aad9da3..8acec3fc 100644 | ||
--- a/numcodecs/tests/test_checksum32.py | ||
+++ b/numcodecs/tests/test_checksum32.py | ||
@@ -3,7 +3,11 @@ | ||
import numpy as np | ||
import pytest | ||
|
||
-from numcodecs.checksum32 import CRC32, CRC32C, Adler32 | ||
+try: | ||
+ from numcodecs.checksum32 import CRC32, CRC32C, Adler32 | ||
+except ImportError: # pragma: no cover | ||
+ pytest.skip("numcodecs.checksum32 not available", allow_module_level=True) | ||
+ | ||
from numcodecs.tests.common import ( | ||
check_backwards_compatibility, | ||
check_config, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,216 @@ | ||
From afce73117651b160e041a0c20e2b52b0f95dac18 Mon Sep 17 00:00:00 2001 | ||
From: jakirkham <jakirkham@gmail.com> | ||
Date: Wed, 13 Nov 2024 18:27:03 -0800 | ||
Subject: [PATCH] Cleanup `crc32c` soft dependency | ||
|
||
--- | ||
.pre-commit-config.yaml | 2 +- | ||
numcodecs/__init__.py | 8 ++++-- | ||
numcodecs/checksum32.py | 46 +++++++++++++++--------------- | ||
numcodecs/tests/test_checksum32.py | 38 +++++++++++++++--------- | ||
4 files changed, 55 insertions(+), 39 deletions(-) | ||
|
||
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml | ||
index bf6a751b..1712efa7 100644 | ||
--- a/.pre-commit-config.yaml | ||
+++ b/.pre-commit-config.yaml | ||
@@ -31,4 +31,4 @@ repos: | ||
hooks: | ||
- id: mypy | ||
args: [--config-file, pyproject.toml] | ||
- additional_dependencies: [numpy, pytest, zfpy, 'zarr==3.0.0b1'] | ||
+ additional_dependencies: [numpy, pytest, crc32c, zfpy, 'zarr==3.0.0b1'] | ||
diff --git a/numcodecs/__init__.py b/numcodecs/__init__.py | ||
index c7401665..8b949d15 100644 | ||
--- a/numcodecs/__init__.py | ||
+++ b/numcodecs/__init__.py | ||
@@ -117,13 +117,17 @@ | ||
|
||
register_codec(MsgPack) | ||
|
||
-from numcodecs.checksum32 import CRC32, CRC32C, Adler32, JenkinsLookup3 | ||
+from numcodecs.checksum32 import CRC32, Adler32, JenkinsLookup3 | ||
|
||
register_codec(CRC32) | ||
-register_codec(CRC32C) | ||
register_codec(Adler32) | ||
register_codec(JenkinsLookup3) | ||
|
||
+with suppress(ImportError): | ||
+ from numcodecs.checksum32 import CRC32C | ||
+ | ||
+ register_codec(CRC32C) | ||
+ | ||
from numcodecs.json import JSON | ||
|
||
register_codec(JSON) | ||
diff --git a/numcodecs/checksum32.py b/numcodecs/checksum32.py | ||
index d5f6a8da..041f9e9f 100644 | ||
--- a/numcodecs/checksum32.py | ||
+++ b/numcodecs/checksum32.py | ||
@@ -1,7 +1,9 @@ | ||
import struct | ||
import zlib | ||
from collections.abc import Callable | ||
-from typing import TYPE_CHECKING, Literal | ||
+from contextlib import suppress | ||
+from types import ModuleType | ||
+from typing import TYPE_CHECKING, Literal, Optional | ||
|
||
import numpy as np | ||
|
||
@@ -9,6 +11,10 @@ | ||
from .compat import ensure_contiguous_ndarray, ndarray_copy | ||
from .jenkins import jenkins_lookup3 | ||
|
||
+_crc32c: Optional[ModuleType] = None | ||
+with suppress(ImportError): | ||
+ import crc32c as _crc32c # type: ignore[no-redef] | ||
+ | ||
if TYPE_CHECKING: | ||
from typing_extensions import Buffer | ||
|
||
@@ -76,28 +82,6 @@ class CRC32(Checksum32): | ||
location = 'start' | ||
|
||
|
||
-class CRC32C(Checksum32): | ||
- """Codec add a crc32c checksum to the buffer. | ||
- | ||
- Parameters | ||
- ---------- | ||
- location : 'start' or 'end' | ||
- Where to place the checksum in the buffer. | ||
- """ | ||
- | ||
- codec_id = 'crc32c' | ||
- | ||
- def checksum(self, buf): | ||
- try: | ||
- from crc32c import crc32c as crc32c_ | ||
- | ||
- return crc32c_(buf) | ||
- except ImportError: # pragma: no cover | ||
- raise ImportError("crc32c must be installed to use the CRC32C checksum codec.") | ||
- | ||
- location = 'end' | ||
- | ||
- | ||
class Adler32(Checksum32): | ||
"""Codec add a adler32 checksum to the buffer. | ||
|
||
@@ -168,3 +152,19 @@ def decode(self, buf, out=None): | ||
out.view("uint8")[:] = b[:-4] | ||
return out | ||
return memoryview(b[:-4]) | ||
+ | ||
+ | ||
+if _crc32c: | ||
+ | ||
+ class CRC32C(Checksum32): | ||
+ """Codec add a crc32c checksum to the buffer. | ||
+ | ||
+ Parameters | ||
+ ---------- | ||
+ location : 'start' or 'end' | ||
+ Where to place the checksum in the buffer. | ||
+ """ | ||
+ | ||
+ codec_id = 'crc32c' | ||
+ checksum = _crc32c.crc32c # type: ignore[union-attr] | ||
+ location = 'end' | ||
diff --git a/numcodecs/tests/test_checksum32.py b/numcodecs/tests/test_checksum32.py | ||
index 8acec3fc..9bdc25cb 100644 | ||
--- a/numcodecs/tests/test_checksum32.py | ||
+++ b/numcodecs/tests/test_checksum32.py | ||
@@ -1,13 +1,10 @@ | ||
import itertools | ||
+from contextlib import suppress | ||
|
||
import numpy as np | ||
import pytest | ||
|
||
-try: | ||
- from numcodecs.checksum32 import CRC32, CRC32C, Adler32 | ||
-except ImportError: # pragma: no cover | ||
- pytest.skip("numcodecs.checksum32 not available", allow_module_level=True) | ||
- | ||
+from numcodecs.checksum32 import CRC32, Adler32 | ||
from numcodecs.tests.common import ( | ||
check_backwards_compatibility, | ||
check_config, | ||
@@ -17,6 +14,12 @@ | ||
check_repr, | ||
) | ||
|
||
+has_crc32c = False | ||
+with suppress(ImportError): | ||
+ from numcodecs.checksum32 import CRC32C | ||
+ | ||
+ has_crc32c = True | ||
+ | ||
# mix of dtypes: integer, float, bool, string | ||
# mix of shapes: 1D, 2D, 3D | ||
# mix of orders: C, F | ||
@@ -39,11 +42,16 @@ | ||
codecs = [ | ||
CRC32(), | ||
CRC32(location="end"), | ||
- CRC32C(location="start"), | ||
- CRC32C(), | ||
Adler32(), | ||
Adler32(location="end"), | ||
] | ||
+if has_crc32c: | ||
+ codecs.extend( | ||
+ [ | ||
+ CRC32C(location="start"), | ||
+ CRC32C(), | ||
+ ] | ||
+ ) | ||
|
||
|
||
@pytest.mark.parametrize(("codec", "arr"), itertools.product(codecs, arrays)) | ||
@@ -88,25 +96,28 @@ def test_err_encode_list(codec): | ||
def test_err_location(): | ||
with pytest.raises(ValueError): | ||
CRC32(location="foo") | ||
- with pytest.raises(ValueError): | ||
- CRC32C(location="foo") | ||
with pytest.raises(ValueError): | ||
Adler32(location="foo") | ||
+ if has_crc32c: | ||
+ with pytest.raises(ValueError): | ||
+ CRC32C(location="foo") | ||
|
||
|
||
def test_repr(): | ||
check_repr("CRC32(location='start')") | ||
- check_repr("CRC32C(location='start')") | ||
- check_repr("Adler32(location='start')") | ||
check_repr("CRC32(location='end')") | ||
- check_repr("CRC32C(location='end')") | ||
+ check_repr("Adler32(location='start')") | ||
check_repr("Adler32(location='end')") | ||
+ if has_crc32c: | ||
+ check_repr("CRC32C(location='start')") | ||
+ check_repr("CRC32C(location='end')") | ||
|
||
|
||
def test_backwards_compatibility(): | ||
check_backwards_compatibility(CRC32.codec_id, arrays, [CRC32()]) | ||
check_backwards_compatibility(Adler32.codec_id, arrays, [Adler32()]) | ||
- check_backwards_compatibility(CRC32C.codec_id, arrays, [CRC32C()]) | ||
+ if has_crc32c: | ||
+ check_backwards_compatibility(CRC32C.codec_id, arrays, [CRC32C()]) | ||
|
||
|
||
@pytest.mark.parametrize("codec", codecs) | ||
@@ -127,6 +138,7 @@ def test_err_out_too_small(codec): | ||
codec.decode(codec.encode(arr), out) | ||
|
||
|
||
+@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed") | ||
def test_crc32c_checksum(): | ||
arr = np.arange(0, 64, dtype="uint8") | ||
buf = CRC32C(location="end").encode(arr) |