buffer: add Buffer.stringLength() · nodejs/node@2777803 · GitHub
Skip to content

Commit 2777803

Browse files
mcollinaaduh95
authored andcommitted
buffer: add Buffer.stringLength()
Add `Buffer.stringLength(input[, encoding])`, the counterpart of `Buffer.byteLength()`: it returns the number of UTF-16 code units that `buf.toString(encoding)` would produce, without decoding. For UTF-8 the count is computed with simdutf. Invalid input is counted with the same maximal-subpart replacement that the decoder applies, so the result always matches `toString().length`. The other encodings are computed from `byteLength` alone. This lets code that accumulates streamed input check the result against `buffer.constants.MAX_STRING_LENGTH` and size its memory budget before decoding. Refs: #66062 Signed-off-by: Matteo Collina <hello@matteocollina.com> PR-URL: #66064 Reviewed-By: Paolo Insogna <paolo@cowtech.it> Reviewed-By: Gürgün Dayıoğlu <hey@gurgun.day>
1 parent d3f55c7 commit 2777803

5 files changed

Lines changed: 340 additions & 0 deletions

File tree

Lines changed: 28 additions & 0 deletions

‎doc/api/buffer.md‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1013,6 +1013,59 @@ console.log(`${str}: ${str.length} characters, ` +
10131013
When `string` is a {Buffer|DataView|TypedArray|ArrayBuffer|SharedArrayBuffer},
10141014
the byte length as reported by `.byteLength` is returned.
10151015

1016+
### Static method: `Buffer.stringLength(input[, encoding])`
1017+
1018+
<!-- YAML
1019+
added: REPLACEME
1020+
-->
1021+
1022+
* `input` {Buffer | ArrayBuffer | TypedArray} The bytes that would be decoded.
1023+
* `encoding` {string} The character encoding `input` would be decoded with.
1024+
**Default:** `'utf8'`.
1025+
* Returns: {integer}
1026+
1027+
Returns the length, in UTF-16 code units, of the string that
1028+
`buf.toString(encoding)` would produce for the same bytes, without decoding
1029+
them. This is the counterpart of [`Buffer.byteLength()`][], which returns the
1030+
number of bytes a string would encode to.
1031+
1032+
For `'utf8'`, invalid byte sequences are counted as they would be decoded:
1033+
each maximal invalid subsequence becomes one `U+FFFD` replacement character.
1034+
For every other encoding the result is computed from `input.byteLength` alone.
1035+
1036+
A detached `ArrayBuffer`, or a `TypedArray` backed by one, is treated as empty.
1037+
1038+
The result is not capped: compare it with
1039+
[`buffer.constants.MAX_STRING_LENGTH`][] before decoding to know whether the
1040+
decode can succeed at all. A string of `n` code units occupies between `n` and
1041+
`2 * n` bytes of memory.
1042+
1043+
```mjs
1044+
import { Buffer, constants } from 'node:buffer';
1045+
1046+
const buf = Buffer.from('€ 100', 'utf8');
1047+
1048+
console.log(Buffer.stringLength(buf));
1049+
// Prints: 5
1050+
console.log(Buffer.stringLength(buf, 'hex'));
1051+
// Prints: 14
1052+
console.log(Buffer.stringLength(buf) <= constants.MAX_STRING_LENGTH);
1053+
// Prints: true
1054+
```
1055+
1056+
```cjs
1057+
const { Buffer, constants } = require('node:buffer');
1058+
1059+
const buf = Buffer.from('€ 100', 'utf8');
1060+
1061+
console.log(Buffer.stringLength(buf));
1062+
// Prints: 5
1063+
console.log(Buffer.stringLength(buf, 'hex'));
1064+
// Prints: 14
1065+
console.log(Buffer.stringLength(buf) <= constants.MAX_STRING_LENGTH);
1066+
// Prints: true
1067+
```
1068+
10161069
### Static method: `Buffer.compare(buf1, buf2)`
10171070

10181071
<!-- YAML
@@ -5712,6 +5765,7 @@ or after startup, if the alignment has to hold at run time.
57125765
[`Buffer.alloc()`]: #static-method-bufferallocsize-fill-encoding
57135766
[`Buffer.allocUnsafe()`]: #static-method-bufferallocunsafesize-alignment
57145767
[`Buffer.allocUnsafeSlow()`]: #static-method-bufferallocunsafeslowsize-alignment
5768+
[`Buffer.byteLength()`]: #static-method-bufferbytelengthstring-encoding
57155769
[`Buffer.concat()`]: #static-method-bufferconcatlist-totallength
57165770
[`Buffer.copyBytesFrom()`]: #static-method-buffercopybytesfromview-offset-length
57175771
[`Buffer.from(array)`]: #static-method-bufferfromarray

‎lib/buffer.js‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ const {
2626
ArrayBufferIsView,
2727
ArrayIsArray,
2828
ArrayPrototypeForEach,
29+
MathCeil,
2930
MathFloor,
3031
MathMin,
3132
MathTrunc,
@@ -62,6 +63,7 @@ const {
6263
fill: bindingFill,
6364
isAscii: bindingIsAscii,
6465
isUtf8: bindingIsUtf8,
66+
stringLengthUtf8: bindingStringLengthUtf8,
6567
indexOfBuffer,
6668
indexOfNumber,
6769
indexOfString,
@@ -938,6 +940,7 @@ function byteLength(string, encoding) {
938940
}
939941

940942
Buffer.byteLength = byteLength;
943+
Buffer.stringLength = stringLength;
941944

942945
// For backwards compatibility.
943946
ObjectDefineProperty(Buffer.prototype, 'parent', {
@@ -1495,6 +1498,32 @@ function isAscii(input) {
14951498
throw new ERR_INVALID_ARG_TYPE('input', ['ArrayBuffer', 'Buffer', 'TypedArray'], input);
14961499
}
14971500

1501+
function stringLength(input, encoding = 'utf8') {
1502+
if (!isTypedArray(input) && !isAnyArrayBuffer(input)) {
1503+
throw new ERR_INVALID_ARG_TYPE('input', ['ArrayBuffer', 'Buffer', 'TypedArray'], input);
1504+
}
1505+
validateString(encoding, 'encoding');
1506+
const ops = getEncodingOps(encoding);
1507+
if (ops === undefined) {
1508+
throw new ERR_UNKNOWN_ENCODING(encoding);
1509+
}
1510+
const length = input.byteLength;
1511+
switch (ops.encodingVal) {
1512+
case encodingsMap.utf8:
1513+
return length === 0 ? 0 : bindingStringLengthUtf8(input);
1514+
case encodingsMap.utf16le:
1515+
return MathFloor(length / 2);
1516+
case encodingsMap.hex:
1517+
return length * 2;
1518+
case encodingsMap.base64:
1519+
return MathCeil(length / 3) * 4;
1520+
case encodingsMap.base64url:
1521+
return MathCeil(length * 4 / 3);
1522+
default: // latin1, ascii
1523+
return length;
1524+
}
1525+
}
1526+
14981527
module.exports = {
14991528
Buffer,
15001529
transcode,

‎src/node_buffer.cc‎

Lines changed: 95 additions & 0 deletions

0 commit comments

Comments
 (0)