8000 Feature 3.8/add write pipe func (#14765) · arangodb/arangodb@f5ae228 · GitHub
[go: up one dir, main page]
Skip to content

Commit f5ae228

Browse files
dothebartjsteemannmpoeterKVS85
authored
Feature 3.8/add write pipe func (#14765)
* Feature/add write pipe func (#14736) * attempt to first uninstall the old service on silent upgrade as well, so the new one can be installed afterwards without problems. * fix syntax * add write pipe function * remove functions * fix return type * add V8 wrappers for write pipe and close pipe * Update js/common/bootstrap/modules/fs.js Co-authored-by: Jan <jsteemann@users.noreply.github.com> * Update lib/Basics/FileUtils.cpp Co-authored-by: Jan <jsteemann@users.noreply.github.com> * Update lib/Basics/process-utils.cpp Co-authored-by: Jan <jsteemann@users.noreply.github.com> * Update lib/Basics/process-utils.cpp Co-authored-by: Jan <jsteemann@users.noreply.github.com> * Update lib/V8/v8-utils.cpp Co-authored-by: Jan <jsteemann@users.noreply.github.com> * Update lib/V8/v8-utils.cpp Co-authored-by: Jan <jsteemann@users.noreply.github.com> * Update lib/V8/v8-utils.cpp Co-authored-by: Jan <jsteemann@users.noreply.github.com> * Update lib/V8/v8-utils.cpp Co-authored-by: Jan <jsteemann@users.noreply.github.com> * add pipe through test * add pipe through test * add pipe through test * Fix stdin/out forwarding on windows * add nullpointer check * revert accidantial commit * add error handling * wintendo linebreaks Co-authored-by: Jan <jsteemann@users.noreply.github.com> Co-authored-by: mpoeter <manuel@arangodb.com> * Update lib/V8/v8-utils.cpp Co-authored-by: Jan <jsteemann@users.noreply.github.com> Co-authored-by: Jan <jsteemann@users.noreply.github.com> Co-authored-by: mpoeter <manuel@arangodb.com> Co-authored-by: Vadim Kondratyev <vadim@arangodb.com>
1 parent 61d4f05 commit f5ae228

File tree

9 files changed

+276
-93
lines changed

9 files changed

+276
-93
lines changed

‎js/client/modules/@arangodb/testsuites/arangosh.js‎

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,14 @@ const yaml = require('js-yaml');
3232

3333
const pu = require('@arangodb/testutils/process-utils');
3434
const tu = require('@arangodb/testutils/test-utils');
35+
const internal = require('internal');
36+
const toArgv = internal.toArgv;
37+
const executeScript = internal.executeScript;
38+
const statusExternal = internal.statusExternal;
39+
const executeExternal = internal.executeExternal;
40+
const executeExternalAndWait = internal.executeExternalAndWait;
3541

36-
const toArgv = require('internal').toArgv;
37-
const executeScript = require('internal').executeScript;
38-
const executeExternalAndWait = require('internal').executeExternalAndWait;
39-
40-
const platform = require('internal').platform;
42+
const platform = internal.platform;
4143

4244
// const BLUE = require('internal').COLORS.COLOR_BLUE;
4345
const CYAN = require('internal').COLORS.COLOR_CYAN;
@@ -257,6 +259,54 @@ function arangosh (options) {
257259
'q = `FOR i\nIN [1,2,3]\nRETURN i`;\nq += "abc"\n', 0, {'server.endpoint': 'none'});
258260
print();
259261

262+
print('\n--------------------------------------------------------------------------------');
263+
print('pipe through external arangosh');
264+
print('--------------------------------------------------------------------------------');
265+
let section = "testArangoshPipeThrough";
266+
let args = pu.makeArgs.arangosh(options);
267+
args['javascript.execute-string'] = "print(require('internal').pollStdin())";
268+
269+
const startTime = time();
270+
let res = executeExternal(pu.ARANGOSH_BIN, toArgv(args), true, 0 /*, coverageEnvironment() */);
271+
const deltaTime = time() - startTime;
272+
273+
fs.writePipe(res.pid, "bla\n");
274+
fs.closePipe(res.pid, false);
275+
let output = fs.readPipe(res.pid);
276+
// Arangosh will output a \n on its own, so we will get back 2:
277+
let searchstring = "bla\n\n";
278+
if (platform.substr(0, 3) === 'win') {
279+
searchstring = "bla\r\n\r\n";
280+
}
281+
let success = output === searchstring;
282+
283+
let rc = statusExternal(res.pid, true);
284+
let failSuccess = (rc.hasOwnProperty('exit') && rc.exit === 0);
285+
failSuccess = failSuccess && success;
286+
if (options.extremeVerbosity) {
287+
print(toArgv(args));
288+
print(rc);
289+
print('pipe output: ' + output);
290+
}
291+
if (!failSuccess) {
292+
ret.failed += 1;
293+
ret[section] = {
294+
'failed': 1,
295+
'message': 'piping through "bla\\n" didn\'t work out, got: "' +
296+
output + '"',
297+
'total': 1
298+
};
299+
} else {
300+
ret[section] = {
301+
'failed': 0,
302+
'total': 1,
303+
'status': failSuccess
304+
};
305+
}
306+
307+
ret[section]['duration'] = time() - startTime;
308+
print((failSuccess ? GREEN : RED) + 'Status: ' + (failSuccess ? 'SUCCESS' : 'FAIL') + RESET);
309+
260310
if (platform.substr(0, 3) !== 'win') {
261311
var echoSuccess = true;
262312
var deltaTime2 = 0;

‎js/common/bootstrap/modules/fs.js‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -457,14 +457,32 @@ global.DEFINE_MODULE('fs', (function () {
457457
}
458458

459459
// //////////////////////////////////////////////////////////////////////////////
460-
// / @brief read
460+
// / @brief read from pipe
461461
// //////////////////////////////////////////////////////////////////////////////
462462

463463
if (global.SYS_READPIPE) {
464464
exports.readPipe = global.SYS_READPIPE;
465465
delete global.SYS_READPIPE;
466466
}
467467

468+
// //////////////////////////////////////////////////////////////////////////////
469+
// / @brief write to pipe
470+
// //////////////////////////////////////////////////////////////////////////////
471+
472+
if (global.SYS_WRITEPIPE) {
473+
exports.writePipe = global.SYS_WRITEPIPE;
474+
delete global.SYS_WRITEPIPE;
475+
}
476+
477+
// //////////////////////////////////////////////////////////////////////////////
478+
// / @brief close read (true) or write (false) pipe
479+
// //////////////////////////////////////////////////////////////////////////////
480+
481+
if (global.SYS_CLOSEPIPE) {
482+
exports.closePipe = global.SYS_CLOSEPIPE;
483+
delete global.SYS_CLOSEPIPE;
484+
}
485+
468486
// //////////////////////////////////////////////////////////////////////////////
469487
// / @brief readBuffer and readFileSync
470488
// //////////////////////////////////////////////////////////////////////////////

‎lib/Basics/FileUtils.cpp‎

Lines changed: 39 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
#include "FileUtils.h"
3535

3636
#include "Basics/operating-system.h"
37+
#include "Basics/process-utils.h"
3738

3839
#ifdef TRI_HAVE_DIRENT_H
3940
#include <dirent.h>
@@ -739,80 +740,51 @@ void makePathAbsolute(std::string& path) {
739740
}
740741
}
741742

742-
static void throwProgramError(std::string const& filename) {
743-
auto res = TRI_set_errno(TRI_ERROR_SYS_ERROR);
744-
745-
LOG_TOPIC("a557b", TRACE, arangodb::Logger::FIXME)
746-
<< StringUtils::concatT("open failed for file '", filename, "': ", TRI_last_error());
747-
748-
THROW_ARANGO_EXCEPTION(res);
749-
}
750-
751743
std::string slurpProgram(std::string const& program) {
752-
#ifdef _WIN32
753-
icu::UnicodeString uprog(program.c_str(), static_cast<int32_t>(program.length()));
754-
FILE* fp = _wpopen(reinterpret_cast<wchar_t const*>(uprog.getTerminatedBuffer()), L"r");
755-
#else
756-
FILE* fp = popen(program.c_str(), "r");
757-
#endif
758-
759-
constexpr size_t chunkSize = 8192;
760-
StringBuffer buffer(chunkSize, false);
761-
762-
if (fp) {
763-
int c;
764-
765-
while ((c = getc(fp)) != EOF) {
766-
buffer.appendChar((char)c);
767-
}
768-
769-
#ifdef _WIN32
770-
int res = _pclose(fp);
771-
#else
772-
int res = pclose(fp);
773-
#endif
744+
ExternalProcess const* process;
745+
ExternalId external;
746+
ExternalProcessStatus res;
747+
std::string output;
748+
std::vector<std::string> moreArgs;
749+
std::vector<std::string> additionalEnv;
750+
char buf[1024];
751+
752+
moreArgs.push_back(std::string("version"));
753+
754+
TRI_CreateExternalProcess(program.c_str(),
755+
moreArgs,
756+
additionalEnv,
757+
true,
758+
&external);
759+
if (external._pid == TRI_INVALID_PROCESS_ID) {
760+
auto res = TRI_set_errno(TRI_ERROR_SYS_ERROR);
774761

775-
if (res != 0) {
776-
throwProgramError(program);
777-
}
778-
} else {
779-
throwProgramError(program);
762+
LOG_TOPIC("a557b", TRACE, arangodb::Logger::FIXME)
763+
<< StringUtils::concatT("open failed for file '",
764+
program, "': ",
765+
TRI_last_error());
766+
THROW_ARANGO_EXCEPTION(res);
780767
}
768+
process = TRI_LookupSpawnedProcess(external._pid);
769+
if (process == nullptr) {
770+
auto res = TRI_set_errno(TRI_ERROR_SYS_ERROR);
781771

782-
return std::string(buffer.data(), buffer.length());
783-
}
784-
785-
int slurpProgramWithExitcode(std::string const& program, std::string& output) {
786-
#ifdef _WIN32
787-
icu::UnicodeString uprog(program.c_str(), static_cast<int32_t>(program.length()));
788-
FILE* fp = _wpopen(reinterpret_cast<wchar_t const*>(uprog.getTerminatedBuffer()), L"r");
789-
#else
790-
FILE* fp = popen(program.c_str(), "r");
791-
#endif
792-
793-
constexpr size_t chunkSize = 8192;
794-
StringBuffer buffer(chunkSize, false);
795-
796-
if (fp) {
797-
int c;
798-
799-
while ((c = getc(fp)) != EOF) {
800-
buffer.appendChar((char)c);
772+
LOG_TOPIC("a557c", TRACE, arangodb::Logger::FIXME)
773+
<< StringUtils::concatT("process gone? '",
774+
program, "': ",
775+
TRI_last_error());
776+
THROW_ARANGO_EXCEPTION(res);
777+
}
778+
while (res = TRI_CheckExternalProcess(external, false, 0),
779+
(res._status == TRI_EXT_RUNNING)) {
780+
auto nRead = TRI_ReadPipe(process, buf, sizeof(buf) - 1);
781+
if (nRead > 0) {
782+
output.append(buf, nRead);
801783
}
802-
803-
#ifdef _WIN32
804-
int res = _pclose(fp);
805-
#else
806-
int res = pclose(fp);
807-
#endif
808-
809-
output = std::string(buffer.data(), buffer.length());
810-
return res;
811784
}
785+
return output;
786+
}
812787

813-
throwProgramError(program);
814-
return 1; // Just to please the compiler.
815-
};
816788

817789
} // namespace FileUtils
818790
} // namespace basics

‎lib/Basics/FileUtils.h‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -146,9 +146,6 @@ std::string dirname(std::string const&);
146146
// returns the output of a program
147147
std::string slurpProgram(std::string const& program);
148148

149-
// returns the output of a program
150-
int slurpProgramWithExitcode(std::string const& program, std::string& output);
151-
152149
} // namespace FileUtils
153150
} // namespace basics
154151
} // namespace arangodb

‎lib/Basics/process-utils.cpp‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -966,6 +966,38 @@ void TRI_CreateExternalProcess(char const* executable,
966966
/// @brief Reads from the pipe of processes
967967
////////////////////////////////////////////////////////////////////////////////
968968

969+
void TRI_ClosePipe(ExternalProcess* process,
970+
bool read) {
971+
if (process == nullptr ||
972+
(read && TRI_IS_INVALID_PIPE(process->_readPipe)) ||
973+
(!read && TRI_IS_INVALID_PIPE(process->_writePipe))
974+
) {
975+
return;
976+
}
977+
978+
auto pipe = (read) ? &process->_readPipe : &process->_writePipe;
979+
980+
#ifndef _WIN32
981+
if (*pipe != -1) {
982+
FILE* stream = fdopen(*pipe, "w");
983+
if (stream != nullptr) {
984+
fflush(stream);
985+
}
986+
close(*pipe);
987+
*pipe = -1;
988+
}
989+
#else
990+
if (*pipe != INVALID_HANDLE_VALUE) {
991+
CloseHandle(*pipe);
992+
*pipe = INVALID_HANDLE_VALUE;
993+
}
994+
#endif
995+
}
996+
997+
////////////////////////////////////////////////////////////////////////////////
998+
/// @brief Reads from the pipe of processes
999+
////////////////////////////////////////////////////////////////////////////////
1000+
9691001
TRI_read_return_t TRI_ReadPipe(ExternalProcess const* process,
9701002
char* buffer,
9711003
size_t bufferSize) {
@@ -983,6 +1015,25 @@ TRI_read_return_t TRI_ReadPipe(ExternalProcess const* process,
9831015
}
9841016

9851017

1018+
////////////////////////////////////////////////////////////////////////////////
1019+
/// @brief writes from the pipe of processes
1020+
////////////////////////////////////////////////////////////////////////////////
1021+
1022+
bool TRI_WritePipe(ExternalProcess const* process,
1023+
char const* buffer,
1024+
size_t bufferSize) {
1025+
if (process == nullptr || TRI_IS_INVALID_PIPE(process->_writePipe)) {
1026+
return false;
1027+
}
1028+
1029+
#ifndef _WIN32
1030+
return TRI_WritePointer(process->_writePipe, buffer, bufferSize);
1031+
#else
1032+
return TRI_WRITE_POINTER(process->_writePipe, buffer, bufferSize);
1033+
#endif
1034+
}
1035+
1036+
9861037
////////////////////////////////////////////////////////////////////////////////
9871038
/// @brief returns the status of an external process
9881039
////////////////////////////////////////////////////////////////////////////////

‎lib/Basics/process-utils.h‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,10 +178,24 @@ void TRI_CreateExternalProcess(char const* executable,
178178
/// @brief Reads from the pipe of processes
179179
////////////////////////////////////////////////////////////////////////////////
180180

181+
void TRI_ClosePipe(ExternalProcess* process,
182+
bool read);
183+
////////////////////////////////////////////////////////////////////////////////
184+
/// @brief Reads from the pipe of processes
185+
////////////////////////////////////////////////////////////////////////////////
186+
181187
TRI_read_return_t TRI_ReadPipe(ExternalProcess const* process,
182188
char* buffer,
183189
size_t bufferSize);
184190

191+
////////////////////////////////////////////////////////////////////////////////
192+
/// @brief Reads from the pipe of processes
193+
////////////////////////////////////////////////////////////////////////////////
194+
195+
bool TRI_WritePipe(ExternalProcess const* process,
196+
char const* buffer,
197+
size_t bufferSize);
198+
185199
////////////////////////////////////////////////////////////////////////////////
186200
/// @brief returns the status of an external process
187201
////////////////////////////////////////////////////////////////////////////////

‎lib/Basics/win-utils.cpp‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,22 @@ TRI_read_return_t TRI_READ_POINTER(HANDLE fd, void* Buffer, size_t length) {
258258
return static_cast<TRI_read_return_t>(length);
259259
}
260260

261+
bool TRI_WRITE_POINTER(HANDLE fd, void const* buffer, size_t length) {
262+
char const* ptr = static_cast<char const*>(buffer);
263+
while (0 < length) {
264+
DWORD len = static_cast<DWORD>(length);
265+
DWORD written = 0;
266+
if (!WriteFile(fd, ptr, len, &written, nullptr)) {
267+
TRI_set_errno(TRI_ERROR_SYS_ERROR);
268+
LOG_TOPIC("420b2", ERR, arangodb::Logger::FIXME) << "cannot write: " << TRI_LAST_ERROR_STR;
269+
return false;
270+
}
271+
ptr += written;
272+
length -= written;
273+
}
274+
275+
return true;
276+
}
261277

262278
FILE* TRI_FOPEN(char const* filename, char const* mode) {
263279
icu::UnicodeString fn(filename);

‎lib/Basics/win-utils.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ bool terminalKnowsANSIColors();
130130
////////////////////////////////////////////////////////////////////////////////
131131

132132
TRI_read_return_t TRI_READ_POINTER(HANDLE fd, void* Buffer, size_t length);
133+
bool TRI_WRITE_POINTER(HANDLE fd, void const* buffer, size_t length);
133134

134135
std::string getFileNameFromHandle(HANDLE fileHandle);
135136
#endif

0 commit comments

Comments
 (0)
0