zlib: add crc32c · nodejs/node@df2ea49 · GitHub
Skip to content

Commit df2ea49

Browse files
authored
zlib: add crc32c
Add zlib.crc32c(data[, value]), which computes the CRC-32C checksum (Castagnoli polynomial, RFC 3720 section 12.1). CRC-32C has better error detection properties than the CRC-32 of zlib.crc32() and is the checksum required by many current protocols, formats and services, among them: - iSCSI header and data digests (RFC 7143) - SCTP packet checksums (RFC 9260) - iWARP MPA framing (RFC 5044) - NVMe/TCP header and data digests - ext4, Btrfs and XFS metadata checksums - the Snappy framing format - Apache Kafka record batches - LevelDB and RocksDB block checksums - TensorFlow TFRecord files - Google Cloud Storage and Amazon S3 object checksums Implementing any of these in JavaScript currently needs a userland CRC-32C, which is far slower than native code. The function mirrors zlib.crc32(): it accepts a string (encoded as UTF-8) or an ArrayBufferView, takes an optional 32-bit unsigned starting value so a checksum can be continued across chunks, and has a V8 fast API path. The checksum is computed by absl::ExtendCrc32c() from the bundled Abseil, which is already built and linked into node, so this adds no new dependency and no new checksum implementation. absl_crc32c is added to the libraries linked with --shared-abseil. Abseil selects its hardware-accelerated CRC-32C path (SSE4.2 and PCLMULQDQ on x64, the CRC32 and PMULL extensions on arm64) at compile time, so with node's default compiler flags the portable table-driven implementation is used. It runs at about the same speed as zlib.crc32() (about 6.5 GB/s on a recent x64 machine), while the hardware path reaches about 38 GB/s. Builds whose target CPU baseline already includes those instructions get the hardware path without changes. Enabling it for generic builds is left out on purpose: on x64, Abseil uses the accelerated code for unrecognized CPUs without a runtime feature check, so compiling it in unconditionally would fault on CPUs without SSE4.2. Signed-off-by: Philipp Dunkel <pip@pipobscure.com> PR-URL: #66828 Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Paolo Insogna <paolo@cowtech.it> Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com>
1 parent 754325c commit df2ea49

8 files changed

Lines changed: 258 additions & 1 deletion

File tree

‎benchmark/zlib/crc32c.js‎

Lines changed: 47 additions & 0 deletions

‎configure.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2923,6 +2923,7 @@ def make_bin_override():
29232923
'absl_cleanup',
29242924
'absl_config',
29252925
'absl_cord',
2926+
'absl_crc32c',
29262927
'absl_core_headers',
29272928
'absl_die_if_null',
29282929
'absl_dynamic_annotations',

‎doc/api/zlib.md‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2375,6 +2375,49 @@ crc = zlib.crc32(Buffer.from('hello', 'utf16le')); // 1427272415
23752375
crc = zlib.crc32(Buffer.from('world', 'utf16le'), crc); // 4150509955
23762376
```
23772377

2378+
## `zlib.crc32c(data[, value])`
2379+
2380+
<!-- YAML
2381+
added: REPLACEME
2382+
-->
2383+
2384+
* `data` {string|Buffer|TypedArray|DataView} When `data` is a string,
2385+
it will be encoded as UTF-8 before being used for computation.
2386+
* `value` {integer} An optional starting value. It must be a 32-bit unsigned
2387+
integer. **Default:** `0`
2388+
* Returns: {integer} A 32-bit unsigned integer containing the checksum.
2389+
2390+
Computes a 32-bit [Cyclic Redundancy Check][] checksum of `data` using the
2391+
Castagnoli polynomial (CRC-32C, [RFC 3720][] section 12.1). If `value` is
2392+
specified, it is used as the starting value of the checksum, otherwise, 0 is
2393+
used as the starting value. Passing the result of a previous call as `value`
2394+
continues the checksum, as with [`zlib.crc32()`][].
2395+
2396+
CRC-32C is a different checksum from the CRC-32 computed by
2397+
[`zlib.crc32()`][]; the two produce different results for the same input.
2398+
CRC-32C is used by protocols and formats such as iSCSI, SCTP, ext4, Btrfs,
2399+
and the Snappy framing format, and has better error detection properties than
2400+
CRC-32.
2401+
2402+
The CRC algorithm is designed to compute checksums and to detect error
2403+
in data transmission. It's not suitable for cryptographic authentication.
2404+
2405+
Strings are encoded as UTF-8, in the same way as [`zlib.crc32()`][].
2406+
2407+
```mjs
2408+
import zlib from 'node:zlib';
2409+
2410+
let crc = zlib.crc32c('hello'); // 2591144780
2411+
crc = zlib.crc32c('world', crc); // 1456190592
2412+
```
2413+
2414+
```cjs
2415+
const zlib = require('node:zlib');
2416+
2417+
let crc = zlib.crc32c('hello'); // 2591144780
2418+
crc = zlib.crc32c('world', crc); // 1456190592
2419+
```
2420+
23782421
## `zlib.createBrotliCompress([options])`
23792422

23802423
<!-- YAML
@@ -3382,6 +3425,7 @@ Create a Zstandard decompression transform.
33823425
[Brotli parameters]: #brotli-constants
33833426
[Cyclic redundancy check]: https://en.wikipedia.org/wiki/Cyclic_redundancy_check
33843427
[Memory usage tuning]: #memory-usage-tuning
3428+
[RFC 3720]: https://www.rfc-editor.org/rfc/rfc3720.html
33853429
[RFC 7932]: https://www.rfc-editor.org/rfc/rfc7932.html
33863430
[Streams API]: stream.md
33873431
[Zstd compressor options]: #compressor-options-1
@@ -3451,6 +3495,7 @@ Create a Zstandard decompression transform.
34513495
[`zlib.ZipEntry.createSymlink()`]: #static-method-zlibzipentrycreatesymlinkfilename-target-options
34523496
[`zlib.ZipEntry.createSync()`]: #static-method-zlibzipentrycreatesyncfilename-data-options
34533497
[`zlib.ZipFile.open()`]: #static-method-zlibzipfileopenfilename-options
3498+
[`zlib.crc32()`]: #zlibcrc32data-value
34543499
[`zlib.createZipArchive()`]: #zlibcreateziparchiveentries-options
34553500
[`zlib.createZipArchiveSync()`]: #zlibcreateziparchivesyncentries-options
34563501
[`zlib.getMaxZipContentSize()`]: #zlibgetmaxzipcontentsize

‎lib/zlib.js‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ const {
5959
isUint8Array,
6060
} = require('internal/util/types');
6161
const binding = internalBinding('zlib');
62-
const { crc32: crc32Native } = binding;
62+
const { crc32: crc32Native, crc32c: crc32cNative } = binding;
6363
const assert = require('internal/assert');
6464
const {
6565
Buffer,
@@ -1076,6 +1076,16 @@ function crc32(data, value = 0) {
10761076
return crc32Native(data, value);
10771077
}
10781078

1079+
function crc32c(data, value = 0) {
1080+
if (typeof data !== 'string' && !isArrayBufferView(data)) {
1081+
throw new ERR_INVALID_ARG_TYPE('data', ['Buffer', 'TypedArray', 'DataView', 'string'], data);
1082+
}
1083+
validateUint32(value, 'value');
1084+
// Coerce -0 to +0.
1085+
value += 0;
1086+
return crc32cNative(data, value);
1087+
}
1088+
10791089
// Legacy alias on the C++ wrapper object. This is not public API, so we may
10801090
// want to runtime-deprecate it at some point. There's no hurry, though.
10811091
ObjectDefineProperty(binding.Zlib.prototype, 'jsref', {
@@ -1086,6 +1096,7 @@ ObjectDefineProperty(binding.Zlib.prototype, 'jsref', {
10861096

10871097
module.exports = {
10881098
crc32,
1099+
crc32c,
10891100
Deflate,
10901101
Inflate,
10911102
Gzip,

‎src/node_zlib.cc‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
#include "threadpoolwork-inl.h"
3535
#include "util-inl.h"
3636

37+
#include "absl/crc/crc32c.h"
3738
#include "node_debug.h"
3839
#include "v8-fast-api-calls.h"
3940
#include "v8.h"
@@ -2127,6 +2128,36 @@ static uint32_t FastCRC32(v8::Local<v8::Value> receiver,
21272128

21282129
static CFunction fast_crc32_(CFunction::Make(FastCRC32));
21292130

2131+
static inline uint32_t CRC32CImpl(Isolate* isolate,
2132+
Local<Value> data,
2133+
uint32_t value) {
2134+
return CallOnSequence<uint32_t>(
2135+
isolate, data, [&](const char* ptr, size_t size) -> uint32_t {
2136+
return static_cast<uint32_t>(absl::ExtendCrc32c(
2137+
absl::crc32c_t{value}, std::string_view(ptr, size)));
2138+
});
2139+
}
2140+
2141+
static void CRC32C(const FunctionCallbackInfo<Value>& args) {
2142+
CHECK(args[0]->IsArrayBufferView() || args[0]->IsString());
2143+
CHECK(args[1]->IsUint32());
2144+
uint32_t value = args[1].As<v8::Uint32>()->Value();
2145+
args.GetReturnValue().Set(CRC32CImpl(args.GetIsolate(), args[0], value));
2146+
}
2147+
2148+
static uint32_t FastCRC32C(v8::Local<v8::Value> receiver,
2149+
v8::Local<v8::Value> data,
2150+
uint32_t value,
2151+
// NOLINTNEXTLINE(runtime/references)
2152+
v8::FastApiCallbackOptions& options) {
2153+
TRACK_V8_FAST_API_CALL("zlib.crc32c");
2154+
v8::HandleScope handle_scope(options.isolate);
2155+
CHECK(data->IsArrayBufferView() || data->IsString());
2156+
return CRC32CImpl(options.isolate, data, value);
2157+
}
2158+
2159+
static CFunction fast_crc32c_(CFunction::Make(FastCRC32C));
2160+
21302161
void Initialize(Local<Object> target,
21312162
Local<Value> unused,
21322163
Local<Context> context,
@@ -2140,6 +2171,7 @@ void Initialize(Local<Object> target,
21402171
MakeClass<ZstdDecompressStream>::Make(env, target, "ZstdDecompress");
21412172

21422173
SetFastMethodNoSideEffect(context, target, "crc32", CRC32, &fast_crc32_);
2174+
SetFastMethodNoSideEffect(context, target, "crc32c", CRC32C, &fast_crc32c_);
21432175
target->Set(env->context(),
21442176
FIXED_ONE_BYTE_STRING(env->isolate(), "ZLIB_VERSION"),
21452177
FIXED_ONE_BYTE_STRING(env->isolate(), ZLIB_VERSION)).Check();
@@ -2153,6 +2185,8 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
21532185
MakeClass<ZstdDecompressStream>::Make(registry);
21542186
registry->Register(CRC32);
21552187
registry->Register(fast_crc32_);
2188+
registry->Register(CRC32C);
2189+
registry->Register(fast_crc32c_);
21562190
}
21572191

21582192
} // anonymous namespace

‎test/parallel/test-zlib-crc32c.js‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
'use strict';
2+
3+
require('../common');
4+
const zlib = require('zlib');
5+
const assert = require('assert');
6+
const { Buffer } = require('buffer');
7+
8+
// Test vectors from RFC 3720, Appendix B.4.
9+
{
10+
const ascending = Buffer.alloc(32);
11+
const descending = Buffer.alloc(32);
12+
for (let i = 0; i < 32; i++) {
13+
ascending[i] = i;
14+
descending[i] = 31 - i;
15+
}
16+
const iscsiReadPdu = Buffer.from(
17+
'01c00000000000000000000000000000' +
18+
'14000000000004000000001400000018' +
19+
'28000000000000000200000000000000', 'hex');
20+
21+
const tests = [
22+
[Buffer.alloc(32, 0x00), 0x8a9136aa],
23+
[Buffer.alloc(32, 0xff), 0x62a8ab43],
24+
[ascending, 0x46dd794e],
25+
[descending, 0x113fdb5c],
26+
[iscsiReadPdu, 0xd9963a56],
27+
];
28+
for (const [data, expected] of tests) {
29+
assert.strictEqual(zlib.crc32c(data), expected);
30+
assert.strictEqual(zlib.crc32c(data, 0), expected);
31+
}
32+
}
33+
34+
assert.strictEqual(zlib.crc32c('123456789'), 0xe3069283);
35+
assert.strictEqual(zlib.crc32c(Buffer.from('123456789')), 0xe3069283);
36+
37+
assert.strictEqual(zlib.crc32c(''), 0);
38+
assert.strictEqual(zlib.crc32c(Buffer.alloc(0)), 0);
39+
assert.strictEqual(zlib.crc32c('', 0xffffffff), 0xffffffff);
40+
assert.strictEqual(zlib.crc32c(Buffer.alloc(0), 0x12345678), 0x12345678);
41+
assert.strictEqual(zlib.crc32c('123456789', -0), 0xe3069283);
42+
43+
// Strings are encoded as UTF-8.
44+
{
45+
const str = 'héllo wörld ✓ 🚀';
46+
assert.strictEqual(zlib.crc32c(str), zlib.crc32c(Buffer.from(str, 'utf8')));
47+
}
48+
49+
// Only the bytes covered by the view are used.
50+
{
51+
const backing = Buffer.from('xx123456789yy');
52+
const view = backing.subarray(2, 11);
53+
assert.strictEqual(zlib.crc32c(view), 0xe3069283);
54+
assert.strictEqual(
55+
zlib.crc32c(new Uint8Array(backing.buffer, backing.byteOffset + 2, 9)),
56+
0xe3069283);
57+
assert.strictEqual(
58+
zlib.crc32c(new DataView(backing.buffer, backing.byteOffset + 2, 9)),
59+
0xe3069283);
60+
}
61+
62+
// Chaining with `value` matches a single pass, across sizes and split points
63+
// large enough to reach every code path of the implementation.
64+
{
65+
const data = Buffer.allocUnsafe(1 << 20);
66+
for (let i = 0; i < data.length; i++) data[i] = (i * 1103515245 + 12345) & 0xff;
67+
for (const len of [1, 7, 15, 16, 17, 63, 64, 65, 255, 256, 4095, 4096,
68+
65537, data.length]) {
69+
const chunk = data.subarray(0, len);
70+
const whole = zlib.crc32c(chunk);
71+
for (const split of new Set([0, 1, len >> 1, len - 1, len])) {
72+
const crc = zlib.crc32c(chunk.subarray(split),
73+
zlib.crc32c(chunk.subarray(0, split)));
74+
assert.strictEqual(crc, whole, `len=${len} split=${split}`);
75+
}
76+
}
77+
}
78+
79+
// Different from CRC-32.
80+
assert.notStrictEqual(zlib.crc32c('123456789'), zlib.crc32('123456789'));
81+
82+
[undefined, null, true, 1, () => {}, {}].forEach((invalid) => {
83+
assert.throws(() => { zlib.crc32c(invalid); }, { code: 'ERR_INVALID_ARG_TYPE' });
84+
});
85+
86+
[null, true, () => {}, {}].forEach((invalid) => {
87+
assert.throws(() => { zlib.crc32c('test', invalid); }, { code: 'ERR_INVALID_ARG_TYPE' });
88+
});
89+
90+
[-1, 2 ** 32, 1.5, NaN].forEach((invalid) => {
91+
assert.throws(() => { zlib.crc32c('test', invalid); }, { code: 'ERR_OUT_OF_RANGE' });
92+
});
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// Flags: --expose-internals --no-warnings --allow-natives-syntax
2+
'use strict';
3+
4+
const common = require('../common');
5+
const assert = require('assert');
6+
const zlib = require('zlib');
7+
8+
{
9+
function testFastPath() {
10+
const expected = 0x86a072c0; // zlib.crc32c('test', 0)
11+
assert.strictEqual(zlib.crc32c('test', 0), expected);
12+
return expected;
13+
}
14+
15+
eval('%PrepareFunctionForOptimization(zlib.crc32c)');
16+
testFastPath();
17+
eval('%OptimizeFunctionOnNextCall(zlib.crc32c)');
18+
testFastPath();
19+
testFastPath();
20+
21+
if (common.isDebug) {
22+
const { internalBinding } = require('internal/test/binding');
23+
const { getV8FastApiCallCount } = internalBinding('debug');
24+
assert.strictEqual(getV8FastApiCallCount('zlib.crc32c'), 2);
25+
}
26+
}

‎typings/internalBinding/zlib.d.ts‎

Lines changed: 1 addition & 0 deletions

0 commit comments

Comments
 (0)