src: implement Windows-1252 encoding support and update related tests · nodejs/node@7f0ffdd · GitHub
Skip to content

Commit 7f0ffdd

Browse files
mertcanaltinaduh95
mertcanaltin
authored and
aduh95
committed
src: implement Windows-1252 encoding support and update related tests
PR-URL: #60893 Fixes: #60888 Fixes: #59515 Fixes: #56542 Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Rafael Gonzaga <rafael.nunu@hotmail.com>
1 parent d4a2d8a commit 7f0ffdd

6 files changed

Lines changed: 121 additions & 44 deletions

File tree

‎lib/internal/encoding.js‎

Lines changed: 7 additions & 7 deletions

‎src/encoding_binding.cc‎

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,8 @@ void BindingData::CreatePerIsolateProperties(IsolateData* isolate_data,
228228
SetMethodNoSideEffect(isolate, target, "decodeUTF8", DecodeUTF8);
229229
SetMethodNoSideEffect(isolate, target, "toASCII", ToASCII);
230230
SetMethodNoSideEffect(isolate, target, "toUnicode", ToUnicode);
231-
SetMethodNoSideEffect(isolate, target, "decodeLatin1", DecodeLatin1);
231+
SetMethodNoSideEffect(
232+
isolate, target, "decodeWindows1252", DecodeWindows1252);
232233
}
233234

234235
void BindingData::CreatePerContextProperties(Local<Object> target,
@@ -246,10 +247,10 @@ void BindingData::RegisterTimerExternalReferences(
246247
registry->Register(DecodeUTF8);
247248
registry->Register(ToASCII);
248249
registry->Register(ToUnicode);
249-
registry->Register(DecodeLatin1);
250+
registry->Register(DecodeWindows1252);
250251
}
251252

252-
void BindingData::DecodeLatin1(const FunctionCallbackInfo<Value>& args) {
253+
void BindingData::DecodeWindows1252(const FunctionCallbackInfo<Value>& args) {
253254
Environment* env = Environment::GetCurrent(args);
254255

255256
CHECK_GE(args.Length(), 1);
@@ -262,7 +263,6 @@ void BindingData::DecodeLatin1(const FunctionCallbackInfo<Value>& args) {
262263
}
263264

264265
bool ignore_bom = args[1]->IsTrue();
265-
bool has_fatal = args[2]->IsTrue();
266266

267267
ArrayBufferViewContents<uint8_t> buffer(args[0]);
268268
const uint8_t* data = buffer.data();
@@ -277,20 +277,45 @@ void BindingData::DecodeLatin1(const FunctionCallbackInfo<Value>& args) {
277277
return args.GetReturnValue().SetEmptyString();
278278
}
279279

280-
std::string result(length * 2, '\0');
281-
282-
size_t written = simdutf::convert_latin1_to_utf8(
283-
reinterpret_cast<const char*>(data), length, result.data());
280+
// Windows-1252 specific mapping for bytes 128-159
281+
// These differ from Latin-1/ISO-8859-1
282+
static const uint16_t windows1252_mapping[32] = {
283+
0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, // 80-87
284+
0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F, // 88-8F
285+
0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, // 90-97
286+
0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178 // 98-9F
287+
};
288+
289+
std::string result;
290+
result.reserve(length * 3); // Reserve space for UTF-8 output
291+
292+
for (size_t i = 0; i < length; i++) {
293+
uint8_t byte = data[i];
294+
uint32_t codepoint;
295+
296+
// Check if byte is in the special Windows-1252 range (128-159)
297+
if (byte >= 0x80 && byte <= 0x9F) {
298+
codepoint = windows1252_mapping[byte - 0x80];
299+
} else {
300+
// For all other bytes, Windows-1252 is identical to Latin-1
301+
codepoint = byte;
302+
}
284303

285-
if (has_fatal && written == 0) {
286-
return node::THROW_ERR_ENCODING_INVALID_ENCODED_DATA(
287-
env->isolate(), "The encoded data was not valid for encoding latin1");
304+
// Convert codepoint to UTF-8
305+
if (codepoint < 0x80) {
306+
result.push_back(static_cast<char>(codepoint));
307+
} else if (codepoint < 0x800) {
308+
result.push_back(static_cast<char>(0xC0 | (codepoint >> 6)));
309+
result.push_back(static_cast<char>(0x80 | (codepoint & 0x3F)));
310+
} else {
311+
result.push_back(static_cast<char>(0xE0 | (codepoint >> 12)));
312+
result.push_back(static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F)));
313+
result.push_back(static_cast<char>(0x80 | (codepoint & 0x3F)));
314+
}
288315
}
289316

290-
std::string_view view(result.c_str(), written);
291-
292317
Local<Value> ret;
293-
if (ToV8Value(env->context(), view, env->isolate()).ToLocal(&ret)) {
318+
if (ToV8Value(env->context(), result, env->isolate()).ToLocal(&ret)) {
294319
args.GetReturnValue().Set(ret);
295320
}
296321
}

‎src/encoding_binding.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ class BindingData : public SnapshotableObject {
3131
static void EncodeInto(const v8::FunctionCallbackInfo<v8::Value>& args);
3232
static void EncodeUtf8String(const v8::FunctionCallbackInfo<v8::Value>& args);
3333
static void DecodeUTF8(const v8::FunctionCallbackInfo<v8::Value>& args);
34-
static void DecodeLatin1(const v8::FunctionCallbackInfo<v8::Value>& args);
34+
static void DecodeWindows1252(
35+
const v8::FunctionCallbackInfo<v8::Value>& args);
3536

3637
static void ToASCII(const v8::FunctionCallbackInfo<v8::Value>& args);
3738
static void ToUnicode(const v8::FunctionCallbackInfo<v8::Value>& args);

‎test/parallel/test-internal-encoding-binding.js‎

Lines changed: 26 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -8,41 +8,46 @@ const assert = require('node:assert');
88
const { internalBinding } = require('internal/test/binding');
99
const binding = internalBinding('encoding_binding');
1010

11+
// Windows-1252 specific tests
1112
{
12-
// Valid input
13-
const buf = Uint8Array.from([0xC1, 0xE9, 0xF3]);
14-
assert.strictEqual(binding.decodeLatin1(buf, false, false), 'Áéó');
13+
// Test Windows-1252 special characters in 128-159 range
14+
// These differ from Latin-1
15+
assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x80), false, false), '€');
16+
assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x82), false, false), '‚');
17+
assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x83), false, false), 'ƒ');
18+
assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x9F), false, false), 'Ÿ');
1519
}
1620

1721
{
18-
// Empty input
19-
const buf = Uint8Array.from([]);
20-
assert.strictEqual(binding.decodeLatin1(buf, false, false), '');
22+
// Test Windows-1252 characters outside 128-159 range (same as Latin-1)
23+
const buf = Uint8Array.from([0xC1, 0xE9, 0xF3]);
24+
assert.strictEqual(binding.decodeWindows1252(buf, false, false), 'Áéó');
2125
}
2226

2327
{
24-
// Invalid input, but Latin1 has no invalid chars and should never throw.
25-
const buf = new TextEncoder().encode('Invalid Latin1 🧑‍🧑‍🧒‍🧒');
26-
assert.strictEqual(
27-
binding.decodeLatin1(buf, false, false),
28-
'Invalid Latin1 ð\x9F§\x91â\x80\x8Dð\x9F§\x91â\x80\x8Dð\x9F§\x92â\x80\x8Dð\x9F§\x92'
29-
);
28+
// Empty input
29+
const buf = Uint8Array.from([]);
30+
assert.strictEqual(binding.decodeWindows1252(buf, false, false), '');
3031
}
3132

33+
// Windows-1252 specific tests
3234
{
33-
// IgnoreBOM with BOM
34-
const buf = Uint8Array.from([0xFE, 0xFF, 0xC1, 0xE9, 0xF3]);
35-
assert.strictEqual(binding.decodeLatin1(buf, true, false), 'þÿÁéó');
35+
// Test Windows-1252 special characters in 128-159 range
36+
// These differ from Latin-1
37+
assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x80), false, false), '€');
38+
assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x82), false, false), '‚');
39+
assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x83), false, false), 'ƒ');
40+
assert.strictEqual(binding.decodeWindows1252(Uint8Array.of(0x9F), false, false), 'Ÿ');
3641
}
3742

3843
{
39-
// Fatal and InvalidInput, but Latin1 has no invalid chars and should never throw.
40-
const buf = Uint8Array.from([0xFF, 0xFF, 0xFF]);
41-
assert.strictEqual(binding.decodeLatin1(buf, false, true), 'ÿÿÿ');
44+
// Test Windows-1252 characters outside 128-159 range (same as Latin-1)
45+
const buf = Uint8Array.from([0xC1, 0xE9, 0xF3]);
46+
assert.strictEqual(binding.decodeWindows1252(buf, false, false), 'Áéó');
4247
}
4348

4449
{
45-
// IgnoreBOM and Fatal, but Latin1 has no invalid chars and should never throw.
46-
const buf = Uint8Array.from([0xFE, 0xFF, 0xC1, 0xE9, 0xF3]);
47-
assert.strictEqual(binding.decodeLatin1(buf, true, true), 'þÿÁéó');
50+
// Empty input
51+
const buf = Uint8Array.from([]);
52+
assert.strictEqual(binding.decodeWindows1252(buf, false, false), '');
4853
}

‎test/parallel/test-util-text-decoder.js‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,49 @@ test('TextDecoder correctly decodes windows-1252 encoded data', { skip: !common.
1515

1616
assert.strictEqual(decodedString, expectedString);
1717
});
18+
19+
// Test for the difference between Latin1 and Windows-1252 in the 128-159
20+
// range
21+
// Ref: https://github.com/nodejs/node/issues/60888
22+
test('TextDecoder correctly decodes windows-1252 special characters in ' +
23+
'128-159 range', { skip: !common.hasIntl }, () => {
24+
const decoder = new TextDecoder('windows-1252');
25+
26+
// Test specific characters that differ between Latin1 and Windows-1252.
27+
// € Euro sign
28+
assert.strictEqual(decoder.decode(Uint8Array.of(128)).codePointAt(0),
29+
8364);
30+
// ‚ Single low-9 quotation mark
31+
assert.strictEqual(decoder.decode(Uint8Array.of(130)).codePointAt(0),
32+
8218);
33+
// Latin small letter f with hook (ƒ)
34+
assert.strictEqual(decoder.decode(Uint8Array.of(131)).codePointAt(0),
35+
402);
36+
// Ÿ Latin capital letter Y with diaeresis
37+
assert.strictEqual(decoder.decode(Uint8Array.of(159)).codePointAt(0),
38+
376);
39+
40+
// Test the full range to ensure no character is treated as Latin1
41+
// Directly.
42+
const expectedMappings = [
43+
[128, 8364], [129, 129], [130, 8218], [131, 402], [132, 8222],
44+
[133, 8230], [134, 8224], [135, 8225], [136, 710], [137, 8240],
45+
[138, 352], [139, 8249], [140, 338], [141, 141], [142, 381],
46+
[143, 143], [144, 144], [145, 8216], [146, 8217], [147, 8220],
47+
[148, 8221], [149, 8226], [150, 8211], [151, 8212], [152, 732],
48+
[153, 8482], [154, 353], [155, 8250], [156, 339], [157, 157],
49+
[158, 382], [159, 376],
50+
];
51+
52+
for (const [byte, expectedCodePoint] of expectedMappings) {
53+
const result = decoder.decode(Uint8Array.of(byte));
54+
const actualCodePoint = result.codePointAt(0);
55+
assert.strictEqual(
56+
actualCodePoint,
57+
expectedCodePoint,
58+
`Byte 0x${byte.toString(16)} should decode to ` +
59+
`U+${expectedCodePoint.toString(16)} but got ` +
60+
`U+${actualCodePoint.toString(16)}`
61+
);
62+
}
63+
});

‎typings/internalBinding/encoding_binding.d.ts‎

Lines changed: 1 addition & 1 deletion

0 commit comments

Comments
 (0)