8000 Use utfx for writing strings on node, which is about 2.4 times faster… · protobufjs/bytebuffer.js@919da88 · GitHub
[go: up one dir, main page]
Skip to content

Commit 919da88

Browse files
dcodeIOdcodeIO
committed
Use utfx for writing strings on node, which is about 2.4 times faster than using intermediate buffers
1 parent d2c3f08 commit 919da88

File tree

13 files changed

+141
-138
lines changed

13 files changed

+141
-138
lines changed

‎bower.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "bytebuffer",
3-
"version": "3.2.0",
3+
"version": "3.2.1",
44
"author": "Daniel Wirtz <dcode@dcode.io>",
55
"description": "A full-featured ByteBuffer implementation using typed arrays.",
66
"main": "dist/ByteBufferAB.js",

‎dist/ByteBufferAB.js

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@
118118
* @const
119119
* @expose
120120
*/
121-
ByteBuffer.VERSION = "3.2.0";
121+
ByteBuffer.VERSION = "3.2.1";
122122

123123
/**
124124
* Little endian constant that can be used instead of its boolean value. Evaluates to `true`.
@@ -1399,7 +1399,6 @@
13991399
throw new RangeError("Illegal offset: 0 <= "+offset+" (+"+0+") <= "+this.buffer.byteLength);
14001400
}
14011401
var start = offset;
1402-
// UTF8 strings do not contain zero bytes in between except for the zero character, so:
14031402
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str))[1];
14041403
offset += k+1;
14051404
var capacity12 = this.buffer.byteLength;
@@ -1481,8 +1480,7 @@
14811480
throw new RangeError("Illegal offset: 0 <= "+offset+" (+"+0+") <= "+this.buffer.byteLength);
14821481
}
14831482
var start = offset,
1484-
k;
1485-
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str), this.noAssert)[1];
1483+
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str))[1];
14861484
offset += 4+k;
14871485
var capacity13 = this.buffer.byteLength;
14881486
if (offset > capacity13)
@@ -1530,7 +1528,7 @@
15301528
sd;
15311529
utfx.decodeUTF8toUTF16(function() {
15321530
return offset < k ? this.view.getUint8(offset++) : null;
1533-
}.bind(this), sd = utfx.stringDestination(), this.noAssert);
1531+
}.bind(this), sd = utfx.stringDestination());
15341532
str = sd();
15351533
if (relative) {
15361534
this.offset = offset;
@@ -1578,9 +1576,8 @@
15781576
if (offset < 0 || offset + 0 > this.buffer.byteLength)
15791577
throw new RangeError("Illegal offset: 0 <= "+offset+" (+"+0+") <= "+this.buffer.byteLength);
15801578
}
1581-
var k;
1582-
var start = offset;
1583-
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str))[1];
1579+
var start = offset,
1580+
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str))[1];
15841581
offset += k;
15851582
var capacity14 = this.buffer.byteLength;
15861583
if (offset > capacity14)
@@ -1690,7 +1687,7 @@
16901687
var k = offset + length;
16911688
utfx.decodeUTF8toUTF16(function() {
16921689
return offset < k ? this.view.getUint8(offset++) : null;
1693-
}.bind(this), sd = utfx.stringDestination(), this.noAssert);
1690+
}.bind(this), sd = utfx.stringDestination());
16941691
if (offset !== k)
16951692
throw new RangeError("Illegal range: Truncated data, "+offset+" == "+k);
16961693
if (relative) {
@@ -1744,9 +1741,8 @@
17441741
throw new RangeError("Illegal offset: 0 <= "+offset+" (+"+0+") <= "+this.buffer.byteLength);
17451742
}
17461743
var start = offset,
1747-
k, l;
1748-
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str), this.noAssert)[1];
1749-
l = ByteBuffer.calculateVarint32(k);
1744+
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str))[1],
1745+
l = ByteBuffer.calculateVarint32(k);
17501746
offset += l+k;
17511747
var capacity15 = this.buffer.byteLength;
17521748
if (offset > capacity15)
@@ -1793,7 +1789,7 @@
17931789
sd = utfx.stringDestination();
17941790
utfx.decodeUTF8toUTF16(function() {
17951791
return offset < k ? this.view.getUint8(offset++) : null;
1796-
}.bind(this), sd, this.noAssert);
1792+
}.bind(this), sd);
17971793
str = sd();
17981794
if (relative) {
17991795
this.offset = offset;

‎dist/ByteBufferAB.min.js

Lines changed: 44 additions & 44 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎dist/ByteBufferAB.min.js.gz

-14 Bytes
Binary file not shown.

‎dist/ByteBufferAB.min.map

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎dist/ByteBufferNB.js

Lines changed: 26 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ module.exports = (function() {
110110
* @const
111111
* @expose
112112
*/
113-
ByteBuffer.VERSION = "3.2.0";
113+
ByteBuffer.VERSION = "3.2.1";
114114

115115
/**
116116
* Little endian constant that can be used instead of its boolean value. Evaluates to `true`.
@@ -1552,18 +1552,16 @@ module.exports = (function() {
15521552
throw new RangeError("Illegal offset: 0 <= "+offset+" (+"+0+") <= "+this.buffer.length);
15531553
}
15541554
var start = offset;
1555-
// UTF8 strings do not contain zero bytes in between except for the zero character, so:
1556-
var buffer = new Buffer(str, 'utf8');
1557-
k = buffer.length;
1555+
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str))[1];
15581556
offset += k+1;
15591557
var capacity12 = this.buffer.length;
15601558
if (offset > capacity12)
15611559
this.resize((capacity12 *= 2) > offset ? capacity12 : offset);
15621560
offset -= k+1;
1563-
buffer.copy(this.buffer, offset);
1564-
offset += k;
1561+
utfx.encodeUTF16toUTF8(utfx.stringSource(str), function(b) {
1562+
this.buffer[offset++] = b;
1563+
}.bind(this));
15651564
this.buffer[offset++] = 0;
1566-
buffer = null;
15671565
if (relative) {
15681566
this.offset = offset - start;
15691567
return this;
@@ -1634,9 +1632,7 @@ module.exports = (function() {
16341632
throw new RangeError("Illegal offset: 0 <= "+offset+" (+"+0+") <= "+this.buffer.length);
16351633
}
16361634
var start = offset,
1637-
k;
1638-
var buffer = new Buffer(str, "utf8");
1639-
k = buffer.length;
1635+
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str))[1];
16401636
offset += 4+k;
16411637
var capacity13 = this.buffer.length;
16421638
if (offset > capacity13)
@@ -1654,8 +1650,11 @@ module.exports = (function() {
16541650
this.buffer[offset+3] = k & 0xFF;
16551651
}
16561652
offset += 4;
1657-
buffer.copy(this.buffer, offset);
1658-
offset += k;
1653+
utfx.encodeUTF16toUTF8(utfx.stringSource(str), function(b) {
1654+
this.buffer[offset++] = b;
1655+
}.bind(this));
1656+
if (offset !== start + 4 + k)
1657+
throw new RangeError("Illegal range: Truncated data, "+offset+" == "+(offset+4+k));
16591658
if (relative) {
16601659
this.offset = offset;
16611660
return this;
@@ -1747,20 +1746,21 @@ module.exports = (function() {
17471746
if (offset < 0 || offset + 0 > this.buffer.length)
17481747
throw new RangeError("Illegal offset: 0 <= "+offset+" (+"+0+") <= "+this.buffer.length);
17491748
}
1750-
var k;
1751-
var buffer = new Buffer(str, 'utf8');
1752-
k = buffer.length;
1749+
var start = offset,
1750+
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str))[1];
17531751
offset += k;
17541752
var capacity14 = this.buffer.length;
17551753
if (offset > capacity14)
17561754
this.resize((capacity14 *= 2) > offset ? capacity14 : offset);
17571755
offset -= k;
1758-
buffer.copy(this.buffer, offset);
1756+
utfx.encodeUTF16toUTF8(utfx.stringSource(str), function(b) {
1757+
this.buffer[offset++] = b;
1758+
}.bind(this));
17591759
if (relative) {
1760-
this.offset += k;
1760+
this.offset = offset;
17611761
return this;
17621762
}
1763-
return k;
1763+
return offset - start;
17641764
};
17651765

17661766
/**
@@ -1909,18 +1909,19 @@ module.exports = (function() {
19091909
throw new RangeError("Illegal offset: 0 <= "+offset+" (+"+0+") <= "+this.buffer.length);
19101910
}
19111911
var start = offset,
1912-
k, l;
1913-
var buffer = new Buffer(str, "utf8");
1914-
k = buffer.length;
1915-
l = ByteBuffer.calculateVarint32(k);
1912+
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str))[1],
1913+
l = ByteBuffer.calculateVarint32(k);
19161914
offset += l+k;
19171915
var capacity15 = this.buffer.length;
19181916
if (offset > capacity15)
19191917
this.resize((capacity15 *= 2) > offset ? capacity15 : offset);
19201918
offset -= l+k;
1921-
offset += this.writeVarint32(buffer.length, offset);
1922-
buffer.copy(this.buffer, offset);
1923-
offset += buffer.length;
1919+
offset += this.writeVarint32(k, offset);
1920+
utfx.encodeUTF16toUTF8(utfx.stringSource(str), function(b) {
1921+
this.buffer[offset++] = b;
1922+
}.bind(this));
1923+
if (offset !== start+k+l)
1924+
throw new RangeError("Illegal range: Truncated data, "+offset+" == "+(offset+k+l));
19241925
if (relative) {
19251926
this.offset = offset;
19261927
return this;

‎donate.png

1.52 KB
Loading

‎package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "bytebuffer",
3-
"version": "3.2.0",
3+
"version": "3.2.1",
44
"author": "Daniel Wirtz <dcode@dcode.io>",
55
"description": "The swiss army knife for binary data in JavaScript.",
66
"main": "index.js",

‎src/types/strings/cstring.js

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -24,23 +24,18 @@ ByteBuffer.prototype.writeCString = function(str, offset) {
2424
//? ASSERT_OFFSET();
2525
}
2626
var start = offset;
27-
// UTF8 strings do not contain zero bytes in between except for the zero character, so:
28-
//? if (NODE) {
29-
var buffer = new Buffer(str, 'utf8');
30-
k = buffer.length;
31-
//? ENSURE_CAPACITY('k+1');
32-
buffer.copy(this.buffer, offset);
33-
offset += k;
34-
this.buffer[offset++] = 0;
35-
buffer = null;
36-
//? } else {
3727
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str))[1];
3828
//? ENSURE_CAPACITY('k+1');
3929
utfx.encodeUTF16toUTF8(utfx.stringSource(str), function(b) {
30+
//? if (NODE)
31+
this.buffer[offset++] = b;
32+
//? else
4033
this.view.setUint8(offset++, b);
4134
}.bind(this));
35+
//? if (NODE)
36+
this.buffer[offset++] = 0;
37+
//? else
4238
this.view.setUint8(offset++, 0);
43-
//? }
4439
if (relative) {
4540
this.offset = offset - start;
4641
return this;

‎src/types/strings/istring.js

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,26 +18,21 @@ ByteBuffer.prototype.writeIString = function(str, offset) {
1818
//? ASSERT_OFFSET();
1919
}
2020
var start = offset,
21-
k;
22-
//? if (NODE) {
23-
var buffer = new Buffer(str, "utf8");
24-
k = buffer.length;
21+
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str))[1];
2522
//? ENSURE_CAPACITY('4+k');
23+
//? if (NODE)
2624
//? WRITE_UINT32_ARRAY('k');
27-
offset += 4;
28-
buffer.copy(this.buffer, offset);
29-
offset += k;
30-
//? } else {
31-
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str), this.noAssert)[1];
32-
//? ENSURE_CAPACITY('4+k');
25+
//? else
3326
this.view.setUint32(offset, k, this.littleEndian);
3427
offset += 4;
3528
utfx.encodeUTF16toUTF8(utfx.stringSource(str), function(b) {
29+
//? if (NODE)
30+
this.buffer[offset++] = b;
31+
//? else
3632
this.view.setUint8(offset++, b);
3733
}.bind(this));
3834
if (offset !== start + 4 + k)
3935
throw new RangeError("Illegal range: Truncated data, "+offset+" == "+(offset+4+k));
40-
//? }
4136
if (relative) {
4237
this.offset = offset;
4338
return this;
@@ -76,7 +71,7 @@ ByteBuffer.prototype.readIString = function(offset) {
7671
sd;
7772
utfx.decodeUTF8toUTF16(function() {
7873
return offset < k ? this.view.getUint8(offset++) : null;
79-
}.bind(this), sd = utfx.stringDestination(), this.noAssert);
74+
}.bind(this), sd = utfx.stringDestination());
8075
str = sd();
8176
//? }
8277
if (relative) {

‎src/types/strings/utf8string.js

Lines changed: 6 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -29,30 +29,20 @@ ByteBuffer.prototype.writeUTF8String = function(str, offset) {
2929
if (!this.noAssert) {
3030
//? ASSERT_OFFSET();
3131
}
32-
var k;
33-
//? if (NODE) {
34-
var buffer = new Buffer(str, 'utf8');
35-
k = buffer.length;
36-
//? ENSURE_CAPACITY('k');
37-
buffer.copy(this.buffer, offset);
38-
if (relative) {
39-
this.offset += k;
40-
return this;
41-
}
42-
return k;
43-
//? } else {
44-
var start = offset;
45-
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str))[1];
32+
var start = offset,
33+
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str))[1];
4634
//? ENSURE_CAPACITY('k');
4735
utfx.encodeUTF16toUTF8(utfx.stringSource(str), function(b) {
36+
//? if (NODE)
37+
this.buffer[offset++] = b;
38+
//? else
4839
this.view.setUint8(offset++, b);
4940
}.bind(this));
5041
if (relative) {
5142
this.offset = offset;
5243
return this;
5344
}
5445
return offset - start;
55-
//? }
5646
};
5747
//? if (ALIASES) {
5848

@@ -162,7 +152,7 @@ ByteBuffer.prototype.readUTF8String = function(length, metrics, offset) {
162152
var k = offset + length;
163153
utfx.decodeUTF8toUTF16(function() {
164154
return offset < k ? this.view.getUint8(offset++) : null;
165-
}.bind(this), sd = utfx.stringDestination(), this.noAssert);
155+
}.bind(this), sd = utfx.stringDestination());
166156
if (offset !== k)
167157
throw new RangeError("Illegal range: Truncated data, "+offset+" == "+k);
168158
if (relative) {

‎src/types/strings/vstring.js

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,26 +18,18 @@ ByteBuffer.prototype.writeVString = function(str, offset) {
1818
//? ASSERT_OFFSET();
1919
}
2020
var start = offset,
21-
k, l;
22-
//? if (NODE) {
23-
var buffer = new Buffer(str, "utf8");
24-
k = buffer.length;
25-
l = ByteBuffer.calculateVarint32(k);
26-
//? ENSURE_CAPACITY('l+k');
27-
offset += this.writeVarint32(buffer.length, offset);
28-
buffer.copy(this.buffer, offset);
29-
offset += buffer.length;
30-
//? } else {
31-
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str), this.noAssert)[1];
32-
l = ByteBuffer.calculateVarint32(k);
21+
k = utfx.calculateUTF16asUTF8(utfx.stringSource(str))[1],
22+
l = ByteBuffer.calculateVarint32(k);
3323
//? ENSURE_CAPACITY('l+k');
3424
offset += this.writeVarint32(k, offset);
3525
utfx.encodeUTF16toUTF8(utfx.stringSource(str), function(b) {
26+
//? if (NODE)
27+
this.buffer[offset++] = b;
28+
//? else
3629
this.view.setUint8(offset++, b);
3730
}.bind(this));
3831
if (offset !== start+k+l)
3932
throw new RangeError("Illegal range: Truncated data, "+offset+" == "+(offset+k+l));
40-
//? }
4133
if (relative) {
4234
this.offset = offset;
4335
return this;
@@ -74,7 +66,7 @@ ByteBuffer.prototype.readVString = function(offset) {
7466
sd = utfx.stringDestination();
7567
utfx.decodeUTF8toUTF16(function() {
7668
return offset < k ? this.view.getUint8(offset++) : null;
77-
}.bind(this), sd, this.noAssert);
69+
}.bind(this), sd);
7870
str = sd();
7971
//? }
8072
if (relative) {

‎tests/buffer_utfx.js

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// This file compares encoding speed of an intermediate node Buffer with using utfx directly.
2+
// Turns out that utfx is about 2.4 times faster when writing while buffers are about 2.4 times faster when reading.
3+
4+
var utfx = require("utfx");
5+
6+
var sample = "hello world";
7+
8+
(function() {
9+
console.time("Buffer write");
10+
for (var i=0; i<100000; ++i)
11+
new Buffer(sample, "utf8");
12+
console.timeEnd("Buffer write");
13+
var buf = new Buffer(sample, "utf8");
14+
console.time("Buffer read");
15+
for (i=0; i<100000; ++i)
16+
buf.toString("utf8");
17+
console.timeEnd("Buffer read");
18+
})();
19+
20+
(function() {
21+
console.time("utfx write");
22+
for (var i=0; i<100000; ++i)
23+
utfx.encodeUTF16toUTF8(utfx.stringSource(sample), function(b) {});
24+
console.timeEnd("utfx write");
25+
var arr = new Buffer(sample, "utf8");
26+
console.time("utfx read");
27+
for (i=0; i<100000; ++i) {
28+
var j=0;
29+
utfx.decodeUTF8toUTF16(function() {
30+
return j < arr.length ? arr[j++] : null;
31+
}, function(c) {});
32+
}
33+
console.timeEnd("utfx read");
34+
})();

0 commit comments

Comments
 (0)
0