8000 Feature/allow foxx reply compression (#14939) · open-bigdata/arangodb@011d354 · GitHub
[go: up one dir, main page]
Skip to content

Commit 011d354

Browse files
dothebartjsteemann
dothebart
and
jsteemann
authored
Feature/allow foxx reply compression (arangodb#14939)
Co-authored-by: Jan <jsteemann@users.noreply.github.com>
1 parent d376ef9 commit 011d354

File tree

21 files changed

+675
-334
lines changed

21 files changed

+675
-334
lines changed

‎CHANGELOG

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
devel
22
-----
33

4+
* Add option to content-transfer encode gzip Foxx replies.
5+
6+
* Simplify internal request compression/decompression handling code.
7+
48
* In the shards overview the list of servers to move the leader shard to, now
59
also contains the current followers. This means that from now on also active
610
follower servers can be nominated as the leading server for that specific

‎arangod/GeneralServer/CommTask.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -827,14 +827,14 @@ bool CommTask::handleContentEncoding(GeneralRequest& req) {
827827
size_t len = raw.size();
828828
if (encoding == "gzip") {
829829
VPackBuffer<uint8_t> dst;
830-
if (!arangodb::encoding::gzipUncompress(src, len, dst)) {
830+
if (arangodb::encoding::gzipUncompress(src, len, dst) != TRI_ERROR_NO_ERROR) {
831831
return false;
832832
}
833833
req.setPayload(std::move(dst));
834834
return true;
835835
} else if (encoding == "deflate") {
836836
VPackBuffer<uint8_t> dst;
837-
if (!arangodb::encoding::gzipDeflate(src, len, dst)) {
837+
if (arangodb::encoding::gzipInflate(src, len, dst) != TRI_ERROR_NO_ERROR) {
838838
return false;
839839
}
840840
req.setPayload(std::move(dst));

‎arangod/RestHandler/RestStatusHandler.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,7 @@ RestStatus RestStatusHandler::executeOverview() {
272272
}
273273
}
274274

275-
auto const res = TRI_DeflateStringBuffer(buffer.stringBuffer(), buffer.size());
275+
auto const res = buffer.deflate();
276276

277277
if (res != TRI_ERROR_NO_ERROR) {
278278
result.add("hash", VPackValue(buffer.c_str()));

‎arangod/V8Server/v8-actions.cpp

Lines changed: 37 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -742,6 +742,7 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g,
742742

743743
bool bodySet = false;
744744
TRI_GET_GLOBAL_STRING(BodyKey);
745+
745746
if (TRI_HasProperty(context, isolate, res, BodyKey)) {
746747
// check if we should apply result transformations
747748
// transformations turn the result from one type into another
@@ -750,17 +751,17 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g,
750751
// array, e.g. res.transformations = [ "base64encode" ]
751752
TRI_GET_GLOBAL_STRING(TransformationsKey);
752753
v8::Handle<v8::Value> transformArray = res->Get(context, TransformationsKey).FromMaybe(v8::Local<v8::Value>());
753-
754+
v8::Handle<v8::Value> bodyVal = res->Get(context, BodyKey).FromMaybe(v8::Local<v8::Value>());
754755
switch (response->transportType()) {
755756
case Endpoint::TransportType::HTTP: {
756757
// OBI FIXME - vpack
757758
// HTTP SHOULD USE vpack interface
758759

759760
HttpResponse* httpResponse = dynamic_cast<HttpResponse*>(response);
760-
if (transformArray->IsArray()) {
761-
TRI_GET_GLOBAL_STRING(BodyKey);
762-
std::string out(TRI_ObjectToString(isolate, res->Get(context, BodyKey).FromMaybe(v8::Local<v8::Value>())));
763-
v8::Handle<v8::Array> transformations = transformArray.As<v8::Array>();
761+
v8::Handle<v8::Array> transformations = transformArray.As<v8::Array>();
762+
bool setRegularBody = !transformArray->IsArray();
763+
if (!setRegularBody) {
764+
std::string out(TRI_ObjectToString(isolate, bodyVal));
764765

765766
for (uint32_t i = 0; i < transformations->Length(); i++) {
766767
v8::Handle<v8::Value> transformator =
@@ -778,38 +779,44 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g,
778779
out = StringUtils::decodeBase64(out);
779780
// set the correct content-encoding header
780781
response->setHeaderNC(StaticStrings::ContentEncoding, StaticStrings::Binary);
782+
} else if (name == "gzip") {
783+
response->setAllowCompression(true);
784+
setRegularBody = true;
785+
} else if (name == "deflate") {
786+
response->setAllowCompression(true);
787+
setRegularBody = true;
781788
}
782789
}
783-
784-
// what type is out? always json?
785-
httpResponse->body().appendText(out);
786-
httpResponse->sealBody();
787-
} else {
788-
TRI_GET_GLOBAL_STRING(BodyKey);
789-
v8::Handle<v8::Value> b = res->Get(context, BodyKey).FromMaybe(v8::Local<v8::Value>());
790-
if (V8Buffer::hasInstance(isolate, b)) {
790+
if (!setRegularBody) {
791+
// what type is out? always json?
792+
httpResponse->body().appendText(out);
793+
httpResponse->sealBody();
794+
}
795+
}
796+
if (setRegularBody) {
797+
if (V8Buffer::hasInstance(isolate, bodyVal)) {
791798
// body is a Buffer
792-
auto obj = b.As<v8::Object>();
799+
auto obj = bodyVal.As<v8::Object>();
793800
httpResponse->body().appendText(V8Buffer::data(isolate, obj),
794801
V8Buffer::length(isolate, obj));
795802
httpResponse->sealBody();
796803
} else if (autoContent && request->contentTypeResponse() == rest::ContentType::VPACK) {
797804
// use velocypack
798805
try {
799-
std::string json = TRI_ObjectToString(isolate, res->Get(context, BodyKey).FromMaybe(v8::Local<v8::Value>()));
806+
std::string json = TRI_ObjectToString(isolate, bodyVal);
800807
VPackBuffer<uint8_t> buffer;
801808
VPackBuilder builder(buffer);
802809
VPackParser parser(builder);
803810
parser.parse(json);
804811
httpResponse->setContentType(rest::ContentType::VPACK);
805812
httpResponse->setPayload(std::move(buffer));
806813
} catch (...) {
807-
httpResponse->body().appendText(TRI_ObjectToString(isolate, res->Get(context, BodyKey).FromMaybe(v8::Local<v8::Value>())));
814+
httpResponse->body().appendText(TRI_ObjectToString(isolate, bodyVal));
808815
httpResponse->sealBody();
809816
}
810817
} else {
811818
// treat body as a string
812-
httpResponse->body().appendText(TRI_ObjectToString(isolate, res->Get(context, BodyKey).FromMaybe(v8::Local<v8::Value>())));
819+
httpResponse->body().appendText(TRI_ObjectToString(isolate, bodyVal));
813820
httpResponse->sealBody();
814821
}
815822
}
@@ -818,14 +825,11 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g,
818825
case Endpoint::TransportType::VST: {
819826
VPackBuffer<uint8_t> buffer;
820827
VPackBuilder builder(buffer);
821-
822-
v8::Handle<v8::Value> v8Body = res->Get(context, BodyKey).FromMaybe(v8::Local<v8::Value>());
823828
std::string out;
824829

825830
// decode and set out
826831
if (transformArray->IsArray()) {
827-
TRI_GET_GLOBAL_STRING(BodyKey);
828-
out = TRI_ObjectToString(isolate, res->Get(context, BodyKey).FromMaybe(v8::Local<v8::Value>())); // there is one case where
832+
out = TRI_ObjectToString(isolate, bodyVal); // there is one case where
829833
// we do not need a string
830834
v8::Handle<v8::Array> transformations = transformArray.As<v8::Array>();
831835

@@ -838,27 +842,31 @@ static void ResponseV8ToCpp(v8::Isolate* isolate, TRI_v8_global_t const* v8g,
838842
// check available transformations
839843
if (name == "base64decode") {
840844
out = StringUtils::decodeBase64(out);
845+
} else if (name == "gzip") {
846+
response->setAllowCompression(true);
847+
} else if (name == "deflate") {
848+
response->setAllowCompression(true);
841849
}
842850
}
843851
}
844852

845853
// out is not set
846854
if (out.empty()) {
847-
if (autoContent && !V8Buffer::hasInstance(isolate, v8Body)) {
848-
if (v8Body->IsString()) {
849-
out = TRI_ObjectToString(isolate, res->Get(context, BodyKey).FromMaybe(v8::Local<v8::Value>())); // should get moved
855+
if (autoContent && !V8Buffer::hasInstance(isolate, bodyVal)) {
856+
if (bodyVal->IsString()) {
857+
out = TRI_ObjectToString(isolate, bodyVal); // should get moved
850858
} else {
851-
TRI_V8ToVPack(isolate, builder, v8Body, false);
859+
TRI_V8ToVPack(isolate, builder, bodyVal, false);
852860
response->setContentType(rest::ContentType::VPACK);
853861
}
854862
} else if (V8Buffer::hasInstance(isolate,
855-
v8Body)) { // body form buffer - could
856-
// contain json or not
863+
bodyVal)) { // body form buffer - could
864+
// contain json or not
857865
// REVIEW (fc) - is this correct?
858-
auto obj = v8Body.As<v8::Object>();
866+
auto obj = bodyVal.As<v8::Object>();
859867
out = std::string(V8Buffer::data(isolate, obj), V8Buffer::length(isolate, obj));
860868
} else { // body is text - does not contain json
861-
out = TRI_ObjectToString(isolate, res->Get(context, BodyKey).FromMaybe(v8::Local<v8::Value>())); // should get moved
869+
out = TRI_ObjectToString(isolate, bodyVal); // should get moved
862870
}
863871
}
864872

‎arangosh/Shell/V8ClientConnection.cpp

Lines changed: 87 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333
#include "ApplicationFeatures/V8SecurityFeature.h"
3434
#include "Basics/FileUtils.h"
3535
#include "Basics/StringUtils.h"
36+
#include "Basics/EncodingUtils.h"
37+
#include "Basics/StringBuffer.h"
3638
#include "Basics/Utf8Helper.h"
3739
#include "Basics/VelocyPackHelper.h"
3840
#include "Import/ImportHelper.h"
@@ -1855,27 +1857,72 @@ bool setPostBody(fu::Request& request,
18551857

18561858
bool canParseResponse(fu::Response const& response) {
18571859
return (response.isContentTypeVPack() || response.isContentTypeJSON()) &&
1858-
response.contentEncoding() == fuerte::ContentEncoding::Identity &&
1860+
(response.contentEncoding() == fuerte::ContentEncoding::Identity ||
1861+
response.contentEncoding() == fuerte::ContentEncoding::Gzip ||
1862+
response.contentEncoding() == fuerte::ContentEncoding::Deflate) &&
18591863
response.payload().size() > 0;
18601864
}
18611865

1866+
1867+
18621868
v8::Local<v8::Value> parseReplyBodyToV8(fu::Response const& response,
18631869
v8::Isolate* isolate) {
1864-
if (response.contentType() == fu::ContentType::VPack) {
1865-
std::vector<VPackSlice> const& slices = response.slices();
1866-
return TRI_VPackToV8(isolate, slices[0]);
1867-
} else if (response.contentType() == fu::ContentType::Json) {
1870+
if ((response.contentType() != fu::ContentType::VPack) &&
1871+
(response.contentType() != fu::ContentType::Json) ) {
1872+
return v8::Undefined(isolate);
1873+
}
1874+
1875+
if (response.contentEncoding() == fuerte::ContentEncoding::Deflate ||
1876+
response.contentEncoding() == fuerte::ContentEncoding::Gzip) {
1877+
// TODO: working with the stringbuffer adds another alloc / copy.
1878+
// translateResultBodyToV8 will probably decode once more.
1879+
// this uses more resources than neccessary; a better solution
1880+
// would implement this inside fuerte.
18681881
auto responseBody = response.payload();
1869-
try {
1870-
auto parsedBody = VPackParser::fromJson(
1871-
reinterpret_cast<char const*>(
1872-
responseBody.data()),
1873-
responseBody.size());
1874-
return TRI_VPackToV8(isolate, parsedBody->slice());
1875-
} catch (std::exception const& ex) {
1876-
std::string err("Error parsing the server JSON reply: ");
1877-
err += ex.what();
1878-
TRI_CreateErrorObject(isolate, TRI_ERROR_HTTP_CORRUPTED_JSON, err, true);
1882+
VPackBuffer<uint8_t> inflateBuf;
1883+
ErrorCode code = TRI_ERROR_NO_ERROR;
1884+
if (response.contentEncoding() == fuerte::ContentEncoding::Deflate) {
1885+
code = arangodb::encoding::gzipInflate(
1886+
reinterpret_cast<uint8_t const*>(responseBody.data()), responseBody.size(), inflateBuf);
1887+
} else {
1888+
code = arangodb::encoding::gzipUncompress(
1889+
reinterpret_cast<uint8_t const*>(responseBody.data()), responseBody.size(), inflateBuf);
1890+
}
1891+
if (code != TRI_ERROR_NO_ERROR) {
1892+
std::string err("Error inflating compressed response body");
1893+
TRI_CreateErrorObject(isolate, code, err, true);
1894+
return v8::Undefined(isolate);
1895+
}
1896+
if (response.contentType() == fu::ContentType::VPack) {
1897+
auto const& slices = VPackSlice(inflateBuf.data());
1898+
return TRI_VPackToV8(isolate, slices);
1899+
} else {
1900+
try {
1901+
auto parsedBody = VPackParser::fromJson(inflateBuf.data(), inflateBuf.size());
1902+
return TRI_VPackToV8(isolate, parsedBody->slice());
1903+
} catch (std::exception const& ex) {
1904+
std::string err("Error parsing the server JSON reply: ");
1905+
err += ex.what();
1906+
TRI_CreateErrorObject(isolate, TRI_ERROR_HTTP_CORRUPTED_JSON, err, true);
1907+
}
1908+
}
1909+
} else {
1910+
if (response.contentType() == fu::ContentType::VPack) {
1911+
std::vector<VPackSlice> const& slices = response.slices();
1912+
return TRI_VPackToV8(isolate, slices[0]);
1913+
} else {
1914+
auto responseBody = response.payload();
1915+
try {
1916+
auto parsedBody = VPackParser::fromJson(
1917+
reinterpret_cast<char const*>(
1918+
responseBody.data()),
1919+
responseBody.size());
1920+
return TRI_VPackToV8(isolate, parsedBody->slice());
1921+
} catch (std::exception const& ex) {
1922+
std::string err("Error parsing the server JSON reply: ");
1923+
err += ex.what();
1924+
TRI_CreateErrorObject(isolate, TRI_ERROR_HTTP_CORRUPTED_JSON, err, true);
1925+
}
18791926
}
18801927
}
18811928
return v8::Undefined(isolate);
@@ -1884,15 +1931,37 @@ v8::Local<v8::Value> parseReplyBodyToV8(fu::Response const& response,
18841931
v8::Local<v8::Value> translateResultBodyToV8(fu::Response const& response,
18851932
v8::Isolate* isolate) {
18861933
auto responseBody = response.payload();
1887-
if ((response.contentEncoding() == fuerte::ContentEncoding::Identity) &&
1934+
if (((response.contentEncoding() == fuerte::ContentEncoding::Identity) ||
1935+
(response.contentEncoding() == fuerte::ContentEncoding::Gzip) ||
1936+
(response.contentEncoding() == fuerte::ContentEncoding::Deflate) ) &&
18881937
(
18891938
response.isContentTypeJSON() ||
18901939
response.isContentTypeText() ||
18911940
response.isContentTypeHtml()
18921941
)
18931942
) {
1894-
const char* bodyStr = reinterpret_cast<const char*>(responseBody.data());
1895-
return TRI_V8_PAIR_STRING(isolate, bodyStr, responseBody.size());
1943+
if (response.contentEncoding() == fuerte::ContentEncoding::Deflate ||
1944+
response.contentEncoding() == fuerte::ContentEncoding::Gzip) {
1945+
auto responseBody = response.payload();
1946+
VPackBuffer<uint8_t> inflateBuf;
1947+
ErrorCode code = TRI_ERROR_NO_ERROR;
1948+
if (response.contentEncoding() == fuerte::ContentEncoding::Deflate) {
1949+
code = arangodb::encoding::gzipInflate(
1950+
reinterpret_cast<uint8_t const*>(responseBody.data()), responseBody.size(), inflateBuf);
1951+
} else {
1952+
code = arangodb::encoding::gzipUncompress(
1953+
reinterpret_cast<uint8_t const*>(responseBody.data()), responseBody.size(), inflateBuf);
1954+
}
1955+
if (code != TRI_ERROR_NO_ERROR) {
1956+
std::string err("Error inflating compressed response body");
1957+
TRI_CreateErrorObject(isolate, code, err, true);
1958+
return v8::Undefined(isolate);
1959+
}
1960+
return TRI_V8_PAIR_STRING(isolate, inflateBuf.data(), inflateBuf.size());
1961+
} else {
1962+
const char* bodyStr = reinterpret_cast<const char*>(responseBody.data());
1963+
return TRI_V8_PAIR_STRING(isolate, bodyStr, responseBody.size());
1964+
}
18961965
} else {
18971966
V8Buffer* buffer = V8Buffer::New
18981967
(isolate,

‎js/server/modules/@arangodb/foxx/router/response.js

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,13 @@ module.exports =
100100
}
101101
}
102102

103+
transformations(which) {
104+
if (!this._raw.hasOwnProperty('transformations')) {
105+
this._raw.transformations = [];
106+
}
107+
this._raw.transformations.push(which);
108+
}
109+
103110
getHeader (name) {
104111
name = name.toLowerCase();
105112
if (name === 'content-type') {

0 commit comments

Comments
 (0)
0