diff --git a/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-6c167ff7220c4033b9eb3299d4627564.json b/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-6c167ff7220c4033b9eb3299d4627564.json new file mode 100644 index 000000000..d7a00e7d1 --- /dev/null +++ b/packages/smithy-checksums/.changes/next-release/smithy-checksums-feature-6c167ff7220c4033b9eb3299d4627564.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Added CRC64NVME checksums." +} \ No newline at end of file diff --git a/packages/smithy-checksums/README.md b/packages/smithy-checksums/README.md index 227cf26c2..1e579b6d1 100644 --- a/packages/smithy-checksums/README.md +++ b/packages/smithy-checksums/README.md @@ -1,7 +1,7 @@ # smithy-checksums -This package provides the checksum algorithms used by Smithy clients: CRC32 and -CRC32C, and, through `new()`, SHA-1, SHA-256, SHA-512, and MD5. +This package provides the checksum algorithms used by Smithy clients: CRC32, +CRC32C, CRC64NVME, and, through `new()`, SHA-1, SHA-256, SHA-512, and MD5. ```python import smithy_checksums diff --git a/packages/smithy-checksums/build_backend/smithy_checksums_build.py b/packages/smithy-checksums/build_backend/smithy_checksums_build.py index c41cdc8ad..0eb5ca38f 100644 --- a/packages/smithy-checksums/build_backend/smithy_checksums_build.py +++ b/packages/smithy-checksums/build_backend/smithy_checksums_build.py @@ -36,7 +36,7 @@ _CARGO_TOML = Path(__file__).parent.parent / "Cargo.toml" _PURE_PYTHON_HINT = ( f"To install the pure-Python implementation instead, set {_ENV_VAR}=never. It " - "produces identical checksums, but CRC32C is much slower." + "produces identical checksums, but CRC32C and CRC64NVME are much slower." ) diff --git a/packages/smithy-checksums/pyproject.toml b/packages/smithy-checksums/pyproject.toml index 91ed1fd03..3d3561542 100644 --- a/packages/smithy-checksums/pyproject.toml +++ b/packages/smithy-checksums/pyproject.toml @@ -9,7 +9,7 @@ authors = [ description = "Checksum algorithms for Smithy tooling, with native acceleration." readme = "README.md" license = {text = "Apache License 2.0"} -keywords = ["smithy", "sdk", "checksum", "crc32c"] +keywords = ["smithy", "sdk", "checksum", "crc32c", "crc64nvme"] classifiers = [ "Development Status :: 2 - Pre-Alpha", "Intended Audience :: Developers", diff --git a/packages/smithy-checksums/rust/lib.rs b/packages/smithy-checksums/rust/lib.rs index 4ed6086c1..0d170bcc9 100644 --- a/packages/smithy-checksums/rust/lib.rs +++ b/packages/smithy-checksums/rust/lib.rs @@ -48,6 +48,7 @@ macro_rules! crc_state { crc_state!(Crc32State, Crc32IsoHdlc, 4); crc_state!(Crc32cState, Crc32Iscsi, 4); +crc_state!(Crc64NvmeState, Crc64Nvme, 8); fn lock(state: &Mutex) -> MutexGuard<'_, S> { // A panic while holding the lock can't leave a checksum half-updated in a @@ -127,9 +128,10 @@ macro_rules! checksum_class { checksum_class!(Crc32, Crc32State); checksum_class!(Crc32C, Crc32cState); +checksum_class!(Crc64Nvme, Crc64NvmeState); #[pymodule] mod _native { #[pymodule_export] - use super::{Crc32, Crc32C}; + use super::{Crc32, Crc32C, Crc64Nvme}; } diff --git a/packages/smithy-checksums/src/smithy_checksums/__init__.py b/packages/smithy-checksums/src/smithy_checksums/__init__.py index 88eb88d5d..74a7c6e9c 100644 --- a/packages/smithy-checksums/src/smithy_checksums/__init__.py +++ b/packages/smithy-checksums/src/smithy_checksums/__init__.py @@ -2,10 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 """Checksum algorithms for Smithy clients. -CRC32 and CRC32C use a native extension when one is available for the platform -and fall back to pure Python otherwise. The fallback produces identical results -but is orders of magnitude slower, except for CRC32, which uses :mod:`zlib`. SHA-1, SHA-256, SHA-512, and MD5 are available -through :func:`new` and always use :mod:`hashlib`. +CRC32, CRC32C, and CRC64NVME use a native extension when one is available for the +platform and fall back to pure Python otherwise. The fallback produces identical +results but is orders of magnitude slower, except for CRC32, which uses :mod:`zlib`. +SHA-1, SHA-256, SHA-512, and MD5 are available through :func:`new` and always use +:mod:`hashlib`. """ import hashlib @@ -22,6 +23,7 @@ __all__ = ( "CRC32", "CRC32C", + "CRC64NVME", "HAS_NATIVE_EXTENSION", "Checksum", "new", @@ -30,7 +32,7 @@ HAS_NATIVE_EXTENSION: Final[bool] = _NATIVE """Whether the native extension is in use. -When it isn't, CRC32C is computed in pure Python. +When it isn't, CRC32C and CRC64NVME are computed in pure Python. """ @@ -121,7 +123,16 @@ class CRC32C(_BackendChecksum): digest_size = 4 -_CHECKSUMS: Final = {cls.name: cls for cls in (CRC32, CRC32C)} +class CRC64NVME(_BackendChecksum): + """CRC-64/NVME.""" + + __slots__ = () + + name = "crc64nvme" + digest_size = 8 + + +_CHECKSUMS: Final = {cls.name: cls for cls in (CRC32, CRC32C, CRC64NVME)} _HASHLIB_ALGORITHMS: Final = frozenset(("sha1", "sha256", "sha512")) @@ -129,10 +140,10 @@ def new(name: str) -> Checksum: """Creates a checksum from the name of its algorithm. Names are case-insensitive, so the values of a checksum algorithm enum such as - ``CRC32C`` or ``SHA256`` can be passed directly. + ``CRC64NVME`` or ``SHA256`` can be passed directly. - :param name: One of ``crc32``, ``crc32c``, ``sha1``, ``sha256``, ``sha512``, or - ``md5``. + :param name: One of ``crc32``, ``crc32c``, ``crc64nvme``, ``sha1``, ``sha256``, + ``sha512``, or ``md5``. :raises ValueError: If the algorithm isn't supported. """ key = name.lower() diff --git a/packages/smithy-checksums/src/smithy_checksums/_native.pyi b/packages/smithy-checksums/src/smithy_checksums/_native.pyi index 1163b9bb9..48214f330 100644 --- a/packages/smithy-checksums/src/smithy_checksums/_native.pyi +++ b/packages/smithy-checksums/src/smithy_checksums/_native.pyi @@ -15,3 +15,8 @@ class Crc32C: def update(self, data: _Bytes, /) -> None: ... def digest(self) -> bytes: ... def copy(self) -> Self: ... + +class Crc64Nvme: + def update(self, data: _Bytes, /) -> None: ... + def digest(self) -> bytes: ... + def copy(self) -> Self: ... diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/backend.py b/packages/smithy-checksums/src/smithy_checksums/_private/backend.py index c542afd87..82b5d85a3 100644 --- a/packages/smithy-checksums/src/smithy_checksums/_private/backend.py +++ b/packages/smithy-checksums/src/smithy_checksums/_private/backend.py @@ -31,11 +31,13 @@ def _load() -> tuple[bool, Mapping[str, ImplFactory]]: return False, { "crc32": crc.Crc32, "crc32c": crc.Crc32C, + "crc64nvme": crc.Crc64Nvme, } return True, { "crc32": _native.Crc32, "crc32c": _native.Crc32C, + "crc64nvme": _native.Crc64Nvme, } diff --git a/packages/smithy-checksums/src/smithy_checksums/_private/crc.py b/packages/smithy-checksums/src/smithy_checksums/_private/crc.py index 92e5f3682..806856e38 100644 --- a/packages/smithy-checksums/src/smithy_checksums/_private/crc.py +++ b/packages/smithy-checksums/src/smithy_checksums/_private/crc.py @@ -76,3 +76,13 @@ class Crc32C(_ReflectedCrc): _table = _reflected_table(0x82F63B78) _mask = 0xFFFFFFFF _size = 4 + + +class Crc64Nvme(_ReflectedCrc): + """CRC-64/NVME.""" + + __slots__ = () + + _table = _reflected_table(0x9A6C9329AC4BC9B5) + _mask = 0xFFFFFFFFFFFFFFFF + _size = 8 diff --git a/packages/smithy-checksums/tests/unit/test_checksums.py b/packages/smithy-checksums/tests/unit/test_checksums.py index ba17d1a4d..f29d5bab0 100644 --- a/packages/smithy-checksums/tests/unit/test_checksums.py +++ b/packages/smithy-checksums/tests/unit/test_checksums.py @@ -10,6 +10,7 @@ from smithy_checksums import ( CRC32, CRC32C, + CRC64NVME, Checksum, new, ) @@ -19,6 +20,7 @@ CLASSES = { "crc32": CRC32, "crc32c": CRC32C, + "crc64nvme": CRC64NVME, } @@ -90,6 +92,7 @@ def compute(_: int) -> str: [ ("crc32", CRC32), ("CRC32C", CRC32C), + ("CRC64NVME", CRC64NVME), ], ) def test_new_returns_checksum_classes(name: str, expected: type[Checksum]) -> None: diff --git a/packages/smithy-checksums/tests/unit/test_implementations.py b/packages/smithy-checksums/tests/unit/test_implementations.py index a50b39184..063047753 100644 --- a/packages/smithy-checksums/tests/unit/test_implementations.py +++ b/packages/smithy-checksums/tests/unit/test_implementations.py @@ -23,6 +23,7 @@ PURE: dict[str, Factory] = { "crc32": crc.Crc32, "crc32c": crc.Crc32C, + "crc64nvme": crc.Crc64Nvme, } @@ -41,6 +42,7 @@ def _load_native() -> ModuleType | None: else { "crc32": _NATIVE_MODULE.Crc32, "crc32c": _NATIVE_MODULE.Crc32C, + "crc64nvme": _NATIVE_MODULE.Crc64Nvme, } ) requires_native = pytest.mark.skipif( @@ -173,6 +175,7 @@ def test_matches_crt(impls: dict[str, Factory], algorithm: str) -> None: compute: dict[str, Callable[[bytes], bytes]] = { "crc32": lambda data: crt.crc32(data).to_bytes(4, "big"), "crc32c": lambda data: crt.crc32c(data).to_bytes(4, "big"), + "crc64nvme": lambda data: crt.crc64nvme(data).to_bytes(8, "big"), } data = os.urandom(3000) for length in [*range(0, 300, 7), 1023, 1024, 1025, 3000]: diff --git a/packages/smithy-checksums/tests/unit/vectors.py b/packages/smithy-checksums/tests/unit/vectors.py index 7697f57ce..a02f018aa 100644 --- a/packages/smithy-checksums/tests/unit/vectors.py +++ b/packages/smithy-checksums/tests/unit/vectors.py @@ -2,11 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 """Expected digests shared by the checksum tests.""" -ALGORITHMS = ("crc32", "crc32c") +ALGORITHMS = ("crc32", "crc32c", "crc64nvme") # Digests from independent sources: the standard check value of each CRC (the -# digest of "123456789"), the CRC32C test vectors from RFC 3720, and "Hello world" -# digests that match the ones the CRT produces. +# digest of "123456789"), the CRC32C test vectors from RFC 3720, the CRC64NVME test +# cases from the NVMe NVM Command Set specification, and "Hello world" digests that +# match the ones the CRT produces. KNOWN_ANSWERS: dict[str, dict[bytes, str]] = { "crc32": {b"123456789": "cbf43926", b"Hello world": "8bd69e52"}, "crc32c": { @@ -19,6 +20,15 @@ bytes(range(32)): "46dd794e", bytes(range(31, -1, -1)): "113fdb5c", }, + "crc64nvme": { + b"123456789": "ae8b14860a799888", + b"Hello world": "38e259d03f312adb", + # The 64b CRC test cases in the NVMe NVM Command Set specification. + bytes(4096): "6482d367eb22b64e", + b"\xff" * 4096: "c0ddba7302eca3ac", + bytes(i % 256 for i in range(4096)): "3e729f5f6750449c", + bytes(255 - i % 256 for i in range(4096)): "9a2df64b8e9e517e", + }, } @@ -64,4 +74,21 @@ def pattern(length: int) -> bytes: 1025: "c8d03add", 16385: "896b2de8", }, + "crc64nvme": { + 0: "0000000000000000", + 1: "d5da5047efec8728", + 3: "3f3990291e6454d7", + 4: "29513a0b0028dea8", + 8: "7703cc98ac9209d7", + 9: "0eb6fdbb89d04d0b", + 16: "cf6454c75df95620", + 17: "5c34355fb173cefd", + 128: "a5296136625de4f7", + 129: "c3d556fcf23d6a24", + 240: "717e94e4fd87236d", + 241: "2737bbceef8467bc", + 1024: "fee95dc7adac063f", + 1025: "034aba81017791d2", + 16385: "9c0531959bc1eacf", + }, } diff --git a/scripts/api-snapshot/api.json b/scripts/api-snapshot/api.json index 915410341..37809ed55 100644 --- a/scripts/api-snapshot/api.json +++ b/scripts/api-snapshot/api.json @@ -608,6 +608,7 @@ "smithy_checksums": { "CRC32": "class", "CRC32C": "class", + "CRC64NVME": "class", "Checksum": "protocol", "HAS_NATIVE_EXTENSION": "bool", "new": "function"