8000 buffer: add buffer.isUtf8 for utf8 validation · nodejs/node@fbe399c · GitHub
[go: up one dir, main page]
Skip to content

Commit fbe399c

Browse files
anonrigRafaelGSS
anonrig
authored and
RafaelGSS
committed
buffer: add buffer.isUtf8 for utf8 validation
PR-URL: #45947 Reviewed-By: Robert Nagy <ronagy@icloud.com> Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Luigi Pinca <luigipinca@gmail.com> Reviewed-By: Rafael Gonzaga <rafael.nunu@hotmail.com> Reviewed-By: Ben Noordhuis <info@bnoordhuis.nl> Reviewed-By: Anna Henningsen <anna@addaleax.net>
1 parent 05fee67 commit fbe399c

File tree

7 files changed

+131
-1
lines changed

7 files changed

+131
-1
lines changed

‎doc/api/buffer.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5130,6 +5130,17 @@ For code running using Node.js APIs, converting between base64-encoded strings
51305130
and binary data should be performed using `Buffer.from(str, 'base64')` and
51315131
`buf.toString('base64')`.**
51325132

5133+
### `buffer.isUtf8(input)`
5134+
5135+
<!-- YAML
5136+
added: REPLACEME
5137+
-->
5138+
5139+
* input {Buffer | ArrayBuffer | TypedArray} The input to validate.
5140+
* Returns: {boolean} Returns `true` if and only if the input is valid UTF-8.
5141+
5142+
This function is used to check if input contains UTF-8 code points (characters).
5143+
51335144
### `buffer.INSPECT_MAX_BYTES`
51345145

51355146
<!-- YAML

‎lib/buffer.js‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ const {
5757
compareOffset,
5858
createFromString,
5959
fill: bindingFill,
60+
isUtf8: bindingIsUtf8,
6061
indexOfBuffer,
6162
indexOfNumber,
6263
indexOfString,
@@ -84,7 +85,8 @@ const {
8485
const {
8586
isAnyArrayBuffer,
8687
isArrayBufferView,
87-
isUint8Array
88+
isUint8Array,
89+
isTypedArray,
8890
} = require('internal/util/types');
8991
const {
9092
inspect: utilInspect
@@ -1314,10 +1316,19 @@ function atob(input) {
13141316
return Buffer.from(input, 'base64').toString('latin1');
13151317
}
13161318

1319+
function isUtf8(input) {
1320+
if (isTypedArray(input) || isAnyArrayBuffer(input)) {
1321+
return bindingIsUtf8(input);
1322+
}
1323+
1324+
throw new ERR_INVALID_ARG_TYPE('input', ['TypedArray', 'Buffer'], input);
1325+
}
1326+
13171327
module.exports = {
13181328
Buffer,
13191329
SlowBuffer,
13201330
transcode,
1331+
isUtf8,
13211332

13221333
// Legacy
13231334
kMaxLength,

‎src/node_buffer.cc‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1223,6 +1223,20 @@ static void EncodeInto(const FunctionCallbackInfo<Value>& args) {
12231223
results[1] = written;
12241224
}
12251225

1226+
static void IsUtf8(const FunctionCallbackInfo<Value>& args) {
1227+
Environment* env = Environment::GetCurrent(args);
1228+
CHECK_EQ(args.Length(), 1);
1229+
CHECK(args[0]->IsTypedArray() || args[0]->IsArrayBuffer() ||
1230+
args[0]->IsSharedArrayBuffer());
1231+
ArrayBufferViewContents<char> abv(args[0]);
1232+
1233+
if (abv.WasDetached()) {
1234+
return node::THROW_ERR_INVALID_STATE(
1235+
env, "Cannot validate on a detached buffer");
1236+
}
1237+
1238+
args.GetReturnValue().Set(simdutf::validate_utf8(abv.data(), abv.length()));
1239+
}
12261240

12271241
void SetBufferPrototype(const FunctionCallbackInfo<Value>& args) {
12281242
Environment* env = Environment::GetCurrent(args);
@@ -1358,6 +1372,8 @@ void Initialize(Local<Object> target,
13581372
SetMethod(context, target, "encodeInto", EncodeInto);
13591373
SetMethodNoSideEffect(context, target, "encodeUtf8String", EncodeUtf8String);
13601374

1375+
SetMethodNoSideEffect(context, target, "isUtf8", IsUtf8);
1376+
13611377
target
13621378
->Set(context,
13631379
FIXED_ONE_BYTE_STRING(isolate, "kMaxLength"),
@@ -1413,6 +1429,8 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
14131429
registry->Register(EncodeInto);
14141430
registry->Register(EncodeUtf8String);
14151431

1432+
registry->Register(IsUtf8);
1433+
14161434
registry->Register(StringSlice<ASCII>);
14171435
registry->Register(StringSlice<BASE64>);
14181436
registry->Register(StringSlice<BASE64URL>);

‎src/node_errors.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ void OOMErrorHandler(const char* location, const v8::OOMDetails& details);
6868
V(ERR_INVALID_ARG_TYPE, TypeError) \
6969
V(ERR_INVALID_OBJECT_DEFINE_PROPERTY, TypeError) \
7070
V(ERR_INVALID_MODULE, Error) \
71+
V(ERR_INVALID_STATE, Error) \
7172
V(ERR_INVALID_THIS, TypeError) \
7273
V(ERR_INVALID_TRANSFER_OBJECT, TypeError) \
7374
V(ERR_MEMORY_ALLOCATION_FAILED, Error) \

‎src/util-inl.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -555,6 +555,7 @@ void ArrayBufferViewContents<T, S>::ReadValue(v8::Local<v8::Value> buf) {
555555
auto ab = buf.As<v8::ArrayBuffer>();
556556
length_ = ab->ByteLength();
557557
data_ = static_cast<T*>(ab->Data());
558+
was_detached_ = ab->WasDetached();
558559
} else {
559560
CHECK(buf->IsSharedArrayBuffer());
560561
auto sab = buf.As<v8::SharedArrayBuffer>();

‎src/util.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -511,6 +511,7 @@ class ArrayBufferViewContents {
511511
inline void Read(v8::Local<v8::ArrayBufferView> abv);
512512
inline void ReadValue(v8::Local<v8::Value> buf);
513513

514+
inline bool WasDetached() const { return was_detached_; }
514515
inline const T* data() const { return data_; }
515516
inline size_t length() const { return length_; }
516517

@@ -525,6 +526,7 @@ class ArrayBufferViewContents {
525526
T stack_storage_[kStackStorageSize];
526527
T* data_ = nullptr;
527528
size_t length_ = 0;
529+
bool was_detached_ = false;
528530
};
529531

530532
class Utf8Value : public MaybeStackBuffer<char> {

‎test/parallel/test-buffer-isutf8.js‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
'use strict';
2+
3+
require('../common');
4+
const assert = require('assert');
5+
const { isUtf8, Buffer } = require('buffer');
6+
const { TextEncoder } = require('util');
7+
8+
const encoder = new TextEncoder();
9+
10+
assert.strictEqual(isUtf8(encoder.encode('hello')), true);
11+
assert.strictEqual(isUtf8(encoder.encode('ğ')), true);
12+
assert.strictEqual(isUtf8(Buffer.from([])), true);
13+
14+
// Taken from test/fixtures/wpt/encoding/textdecoder-fatal.any.js
15+
[
16+
[0xFF], // 'invalid code'
17+
[0xC0], // 'ends early'
18+
[0xE0], // 'ends early 2'
19+
[0xC0, 0x00], // 'invalid trail'
20+
[0xC0, 0xC0], // 'invalid trail 2'
21+
[0xE0, 0x00], // 'invalid trail 3'
22+
[0xE0, 0xC0], // 'invalid trail 4'
23+
[0xE0, 0x80, 0x00], // 'invalid trail 5'
24+
[0xE0, 0x80, 0xC0], // 'invalid trail 6'
25+
[0xFC, 0x80, 0x80, 0x80, 0x80, 0x80], // '> 0x10FFFF'
26+
[0xFE, 0x80, 0x80, 0x80, 0x80, 0x80], // 'obsolete lead byte'
27+
28+
// Overlong encodings
29+
[0xC0, 0x80], // 'overlong U+0000 - 2 bytes'
30+
[0xE0, 0x80, 0x80], // 'overlong U+0000 - 3 bytes'
31+
[0xF0, 0x80, 0x80, 0x80], // 'overlong U+0000 - 4 bytes'
32+
[0xF8, 0x80, 0x80, 0x80, 0x80], // 'overlong U+0000 - 5 bytes'
33+
[0xFC, 0x80, 0x80, 0x80, 0x80, 0x80], // 'overlong U+0000 - 6 bytes'
34+
35+
[0xC1, 0xBF], // 'overlong U+007F - 2 bytes'
36+
[0xE0, 0x81, 0xBF], // 'overlong U+007F - 3 bytes'
37+
[0xF0, 0x80, 0x81, 0xBF], // 'overlong U+007F - 4 bytes'
38+
[0xF8, 0x80, 0x80, 0x81, 0xBF], // 'overlong U+007F - 5 bytes'
39+
[0xFC, 0x80, 0x80, 0x80, 0x81, 0xBF], // 'overlong U+007F - 6 bytes'
40+
41+
[0xE0, 0x9F, 0xBF], // 'overlong U+07FF - 3 bytes'
42+
[0xF0, 0x80, 0x9F, 0xBF], // 'overlong U+07FF - 4 bytes'
43+
[0xF8, 0x80, 0x80, 0x9F, 0xBF], // 'overlong U+07FF - 5 bytes'
44+
[0xFC, 0x80, 0x80, 0x80, 0x9F, 0xBF], // 'overlong U+07FF - 6 bytes'
45+
46+
[0xF0, 0x8F, 0xBF, 0xBF], // 'overlong U+FFFF - 4 bytes'
47+
[0xF8, 0x80, 0x8F, 0xBF, 0xBF], // 'overlong U+FFFF - 5 bytes'
48+
[0xFC, 0x80, 0x80, 0x8F, 0xBF, 0xBF], // 'overlong U+FFFF - 6 bytes'
49+
50+
[0xF8, 0x84, 0x8F, 0xBF, 0xBF], // 'overlong U+10FFFF - 5 bytes'
51+
[0xFC, 0x80, 0x84, 0x8F, 0xBF, 0xBF], // 'overlong U+10FFFF - 6 bytes'
52+
53+
// UTF-16 surrogates encoded as code points in UTF-8
54+
[0xED, 0xA0, 0x80], // 'lead surrogate'
55+
[0xED, 0xB0, 0x80], // 'trail surrogate'
56+
[0xED, 0xA0, 0x80, 0xED, 0xB0, 0x80], // 'surrogate pair'
57+
].forEach((input) => {
58+
assert.strictEqual(isUtf8(Buffer.from(input)), false);
59+
});
60+
61+
[
62+
null,
63+
undefined,
64+
'hello',
65+
true,
66+
false,
67+
].forEach((input) => {
68+
assert.throws(
69+
() => { isUtf8(input); },
70+
{
71+
code: 'ERR_INVALID_ARG_TYPE',
72+
},
73+
);
74+
});
75+
76+
{
77+
// Test with detached array buffers
78+
const arrayBuffer = new ArrayBuffer(1024);
79+
structuredClone(arrayBuffer, { transfer: [arrayBuffer] });
80+
assert.throws(
81+
() => { is 38BA Utf8(arrayBuffer); },
82+
{
83+
code: 'ERR_INVALID_STATE'
84+
}
85+
);
86+
}

0 commit comments

Comments
 (0)
0