Skip to content

Commit

Permalink
Apply upstream patch to soften crc32c dependency
Browse files Browse the repository at this point in the history
  • Loading branch information
jakirkham committed Nov 14, 2024
1 parent b540060 commit f2f587c
Show file tree
Hide file tree
Showing 3 changed files with 251 additions and 0 deletions.
9 changes: 9 additions & 0 deletions recipe/meta.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,15 @@ package:
source:
url: https://pypi.org/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz
sha256: 3f138f5e4f574a94a4a8ba1c8be9323ec89bce363f4c65801d8e29e5064c6599
patches:
################################################################
# Fix softening of `crc32c` test dependency #
# #
# xref: https://github.com/zarr-developers/numcodecs/pull/635 #
# xref: https://github.com/zarr-developers/numcodecs/pull/637 #
################################################################
- patches/PR_635.patch
- patches/PR_637.patch

build:
number: 0
Expand Down
26 changes: 26 additions & 0 deletions recipe/patches/PR_635.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
From 4b3dc36f8e96d91e4efcfb8f231863505f9a1594 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 13 Nov 2024 10:37:04 -0800
Subject: [PATCH] Skip `checksum32` tests if `crc32` is missing

---
numcodecs/tests/test_checksum32.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/numcodecs/tests/test_checksum32.py b/numcodecs/tests/test_checksum32.py
index 7aad9da3..8acec3fc 100644
--- a/numcodecs/tests/test_checksum32.py
+++ b/numcodecs/tests/test_checksum32.py
@@ -3,7 +3,11 @@
import numpy as np
import pytest

-from numcodecs.checksum32 import CRC32, CRC32C, Adler32
+try:
+ from numcodecs.checksum32 import CRC32, CRC32C, Adler32
+except ImportError: # pragma: no cover
+ pytest.skip("numcodecs.checksum32 not available", allow_module_level=True)
+
from numcodecs.tests.common import (
check_backwards_compatibility,
check_config,
216 changes: 216 additions & 0 deletions recipe/patches/PR_637.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,216 @@
From afce73117651b160e041a0c20e2b52b0f95dac18 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 13 Nov 2024 18:27:03 -0800
Subject: [PATCH] Cleanup `crc32c` soft dependency

---
.pre-commit-config.yaml | 2 +-
numcodecs/__init__.py | 8 ++++--
numcodecs/checksum32.py | 46 +++++++++++++++---------------
numcodecs/tests/test_checksum32.py | 38 +++++++++++++++---------
4 files changed, 55 insertions(+), 39 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index bf6a751b..1712efa7 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -31,4 +31,4 @@ repos:
hooks:
- id: mypy
args: [--config-file, pyproject.toml]
- additional_dependencies: [numpy, pytest, zfpy, 'zarr==3.0.0b1']
+ additional_dependencies: [numpy, pytest, crc32c, zfpy, 'zarr==3.0.0b1']
diff --git a/numcodecs/__init__.py b/numcodecs/__init__.py
index c7401665..8b949d15 100644
--- a/numcodecs/__init__.py
+++ b/numcodecs/__init__.py
@@ -117,13 +117,17 @@

register_codec(MsgPack)

-from numcodecs.checksum32 import CRC32, CRC32C, Adler32, JenkinsLookup3
+from numcodecs.checksum32 import CRC32, Adler32, JenkinsLookup3

register_codec(CRC32)
-register_codec(CRC32C)
register_codec(Adler32)
register_codec(JenkinsLookup3)

+with suppress(ImportError):
+ from numcodecs.checksum32 import CRC32C
+
+ register_codec(CRC32C)
+
from numcodecs.json import JSON

register_codec(JSON)
diff --git a/numcodecs/checksum32.py b/numcodecs/checksum32.py
index d5f6a8da..041f9e9f 100644
--- a/numcodecs/checksum32.py
+++ b/numcodecs/checksum32.py
@@ -1,7 +1,9 @@
import struct
import zlib
from collections.abc import Callable
-from typing import TYPE_CHECKING, Literal
+from contextlib import suppress
+from types import ModuleType
+from typing import TYPE_CHECKING, Literal, Optional

import numpy as np

@@ -9,6 +11,10 @@
from .compat import ensure_contiguous_ndarray, ndarray_copy
from .jenkins import jenkins_lookup3

+_crc32c: Optional[ModuleType] = None
+with suppress(ImportError):
+ import crc32c as _crc32c # type: ignore[no-redef]
+
if TYPE_CHECKING:
from typing_extensions import Buffer

@@ -76,28 +82,6 @@ class CRC32(Checksum32):
location = 'start'


-class CRC32C(Checksum32):
- """Codec add a crc32c checksum to the buffer.
-
- Parameters
- ----------
- location : 'start' or 'end'
- Where to place the checksum in the buffer.
- """
-
- codec_id = 'crc32c'
-
- def checksum(self, buf):
- try:
- from crc32c import crc32c as crc32c_
-
- return crc32c_(buf)
- except ImportError: # pragma: no cover
- raise ImportError("crc32c must be installed to use the CRC32C checksum codec.")
-
- location = 'end'
-
-
class Adler32(Checksum32):
"""Codec add a adler32 checksum to the buffer.

@@ -168,3 +152,19 @@ def decode(self, buf, out=None):
out.view("uint8")[:] = b[:-4]
return out
return memoryview(b[:-4])
+
+
+if _crc32c:
+
+ class CRC32C(Checksum32):
+ """Codec add a crc32c checksum to the buffer.
+
+ Parameters
+ ----------
+ location : 'start' or 'end'
+ Where to place the checksum in the buffer.
+ """
+
+ codec_id = 'crc32c'
+ checksum = _crc32c.crc32c # type: ignore[union-attr]
+ location = 'end'
diff --git a/numcodecs/tests/test_checksum32.py b/numcodecs/tests/test_checksum32.py
index 8acec3fc..9bdc25cb 100644
--- a/numcodecs/tests/test_checksum32.py
+++ b/numcodecs/tests/test_checksum32.py
@@ -1,13 +1,10 @@
import itertools
+from contextlib import suppress

import numpy as np
import pytest

-try:
- from numcodecs.checksum32 import CRC32, CRC32C, Adler32
-except ImportError: # pragma: no cover
- pytest.skip("numcodecs.checksum32 not available", allow_module_level=True)
-
+from numcodecs.checksum32 import CRC32, Adler32
from numcodecs.tests.common import (
check_backwards_compatibility,
check_config,
@@ -17,6 +14,12 @@
check_repr,
)

+has_crc32c = False
+with suppress(ImportError):
+ from numcodecs.checksum32 import CRC32C
+
+ has_crc32c = True
+
# mix of dtypes: integer, float, bool, string
# mix of shapes: 1D, 2D, 3D
# mix of orders: C, F
@@ -39,11 +42,16 @@
codecs = [
CRC32(),
CRC32(location="end"),
- CRC32C(location="start"),
- CRC32C(),
Adler32(),
Adler32(location="end"),
]
+if has_crc32c:
+ codecs.extend(
+ [
+ CRC32C(location="start"),
+ CRC32C(),
+ ]
+ )


@pytest.mark.parametrize(("codec", "arr"), itertools.product(codecs, arrays))
@@ -88,25 +96,28 @@ def test_err_encode_list(codec):
def test_err_location():
with pytest.raises(ValueError):
CRC32(location="foo")
- with pytest.raises(ValueError):
- CRC32C(location="foo")
with pytest.raises(ValueError):
Adler32(location="foo")
+ if has_crc32c:
+ with pytest.raises(ValueError):
+ CRC32C(location="foo")


def test_repr():
check_repr("CRC32(location='start')")
- check_repr("CRC32C(location='start')")
- check_repr("Adler32(location='start')")
check_repr("CRC32(location='end')")
- check_repr("CRC32C(location='end')")
+ check_repr("Adler32(location='start')")
check_repr("Adler32(location='end')")
+ if has_crc32c:
+ check_repr("CRC32C(location='start')")
+ check_repr("CRC32C(location='end')")


def test_backwards_compatibility():
check_backwards_compatibility(CRC32.codec_id, arrays, [CRC32()])
check_backwards_compatibility(Adler32.codec_id, arrays, [Adler32()])
- check_backwards_compatibility(CRC32C.codec_id, arrays, [CRC32C()])
+ if has_crc32c:
+ check_backwards_compatibility(CRC32C.codec_id, arrays, [CRC32C()])


@pytest.mark.parametrize("codec", codecs)
@@ -127,6 +138,7 @@ def test_err_out_too_small(codec):
codec.decode(codec.encode(arr), out)


+@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed")
def test_crc32c_checksum():
arr = np.arange(0, 64, dtype="uint8")
buf = CRC32C(location="end").encode(arr)

0 comments on commit f2f587c

Please sign in to comment.