From a152b83161ceb1201cad1174165d52745e7cd876 Mon Sep 17 00:00:00 2001 From: Manuel B Date: Mon, 28 Aug 2017 14:16:10 +0200 Subject: [PATCH 1/6] cpp current_user() definition --- arangod/Aql/AqlFunctionFeature.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/Aql/AqlFunctionFeature.cpp b/arangod/Aql/AqlFunctionFeature.cpp index 60c836d3e8a0..d5ff267764e7 100644 --- a/arangod/Aql/AqlFunctionFeature.cpp +++ b/arangod/Aql/AqlFunctionFeature.cpp @@ -491,7 +491,7 @@ void AqlFunctionFeature::addMiscFunctions() { add({"IS_SAME_COLLECTION", "AQL_IS_SAME_COLLECTION", ".h,.h", true, true, false, true, true, &Functions::IsSameCollection}); add({"CURRENT_USER", "AQL_CURRENT_USER", "", false, false, false, false, - true}); + true, &Functions::CurrentUser}); add({"CURRENT_DATABASE", "AQL_CURRENT_DATABASE", "", false, false, false, false, true, &Functions::CurrentDatabase}); add({"COLLECTION_COUNT", "AQL_COLLECTION_COUNT", ".h", false, false, true, From 28e67d8fb38dc3b2c8a2d2424263b8e215961893 Mon Sep 17 00:00:00 2001 From: Manuel B Date: Mon, 28 Aug 2017 14:19:41 +0200 Subject: [PATCH 2/6] cpp currentuser() function header --- arangod/Aql/Functions.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/arangod/Aql/Functions.h b/arangod/Aql/Functions.h index ddf895aa073c..6d5c040f9ef0 100644 --- a/arangod/Aql/Functions.h +++ b/arangod/Aql/Functions.h @@ -284,9 +284,6 @@ struct Functions { VPackFunctionParameters const&); static AqlValue NotNull(arangodb::aql::Query*, transaction::Methods*, VPackFunctionParameters const&); - static AqlValue CurrentDatabase(arangodb::aql::Query*, - transaction::Methods*, - VPackFunctionParameters const&); static AqlValue CollectionCount(arangodb::aql::Query*, transaction::Methods*, VPackFunctionParameters const&); @@ -313,6 +310,12 @@ struct Functions { static AqlValue IsSameCollection(arangodb::aql::Query*, transaction::Methods*, VPackFunctionParameters const&); + static AqlValue CurrentUser(arangodb::aql::Query*, + transaction::Methods*, + VPackFunctionParameters const&); + static AqlValue CurrentDatabase(arangodb::aql::Query*, + transaction::Methods*, + VPackFunctionParameters const&); }; } } From aa9820621bd5ffb9259943e6a78429eeee424868 Mon Sep 17 00:00:00 2001 From: Manuel B Date: Mon, 28 Aug 2017 14:22:03 +0200 Subject: [PATCH 3/6] cpp aql current_user implementation --- arangod/Aql/Functions.cpp | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/arangod/Aql/Functions.cpp b/arangod/Aql/Functions.cpp index 9dd78d55caf3..b2256730801a 100644 --- a/arangod/Aql/Functions.cpp +++ b/arangod/Aql/Functions.cpp @@ -3328,17 +3328,6 @@ AqlValue Functions::NotNull(arangodb::aql::Query* query, return AqlValue(arangodb::basics::VelocyPackHelper::NullValue()); } -/// @brief function CURRENT_DATABASE -AqlValue Functions::CurrentDatabase( - arangodb::aql::Query* query, transaction::Methods* trx, - VPackFunctionParameters const& parameters) { - ValidateParameters(parameters, "CURRENT_DATABASE", 0, 0); - - transaction::BuilderLeaser builder(trx); - builder->add(VPackValue(query->vocbase()->name())); - return AqlValue(builder.get()); -} - /// @brief function COLLECTION_COUNT AqlValue Functions::CollectionCount( arangodb::aql::Query* query, transaction::Methods* trx, @@ -3724,6 +3713,32 @@ AqlValue Functions::IsSameCollection( return AqlValue(arangodb::basics::VelocyPackHelper::NullValue()); } +#include "Utils/ExecContext.h" + +/// @brief function CURRENT_USER +AqlValue Functions::CurrentUser( + arangodb::aql::Query* query, transaction::Methods* trx, + VPackFunctionParameters const& parameters) { + + if (ExecContext::CURRENT == nullptr) { + return AqlValue(arangodb::basics::VelocyPackHelper::NullValue()); + } + + std::string const& username = ExecContext::CURRENT->user(); + return AqlValue(username.c_str(), username.length()); +} + +/// @brief function CURRENT_DATABASE +AqlValue Functions::CurrentDatabase( + arangodb::aql::Query* query, transaction::Methods* trx, + VPackFunctionParameters const& parameters) { + ValidateParameters(parameters, "CURRENT_DATABASE", 0, 0); + + transaction::BuilderLeaser builder(trx); + builder->add(VPackValue(query->vocbase()->name())); + return AqlValue(builder.get()); +} + #include "Pregel/PregelFeature.h" #include "Pregel/Worker.h" From 9fe3efefa689e94f1097f1dffdd288dcce5cce65 Mon Sep 17 00:00:00 2001 From: Manuel B Date: Mon, 28 Aug 2017 14:23:19 +0200 Subject: [PATCH 4/6] aql current_user test --- js/server/tests/aql/aql-functions-misc.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js/server/tests/aql/aql-functions-misc.js b/js/server/tests/aql/aql-functions-misc.js index 3274ebdb1372..71879d47031c 100644 --- a/js/server/tests/aql/aql-functions-misc.js +++ b/js/server/tests/aql/aql-functions-misc.js @@ -520,9 +520,9 @@ function ahuacatlMiscFunctionsTestSuite () { } } - var actual = getQueryResults("RETURN CURRENT_USER()"); // there is no current user in the non-request context - assertEqual([ expected ], actual); + assertEqual([ expected ], getQueryResults("RETURN NOOPT(CURRENT_USER())")); + assertEqual([ expected ], getQueryResults("RETURN NOOPT(V8(CURRENT_USER()))")); }, //////////////////////////////////////////////////////////////////////////////// From 600db7f6887b2478830dfceb888a7361ddfd5c48 Mon Sep 17 00:00:00 2001 From: Manuel B Date: Wed, 30 Aug 2017 11:01:03 +0200 Subject: [PATCH 5/6] Update Functions.cpp --- arangod/Aql/Functions.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arangod/Aql/Functions.cpp b/arangod/Aql/Functions.cpp index b2256730801a..f46bf053ff05 100644 --- a/arangod/Aql/Functions.cpp +++ b/arangod/Aql/Functions.cpp @@ -46,6 +46,7 @@ #include "Transaction/Helpers.h" #include "Transaction/Methods.h" #include "Transaction/Context.h" +#include "Utils/ExecContext.h" #include "VocBase/LogicalCollection.h" #include "VocBase/ManagedDocumentResult.h" @@ -3713,8 +3714,6 @@ AqlValue Functions::IsSameCollection( return AqlValue(arangodb::basics::VelocyPackHelper::NullValue()); } -#include "Utils/ExecContext.h" - /// @brief function CURRENT_USER AqlValue Functions::CurrentUser( arangodb::aql::Query* query, transaction::Methods* trx, From f6da82a1bc0bb814102454caa702190b671879ac Mon Sep 17 00:00:00 2001 From: Manuel B Date: Wed, 30 Aug 2017 11:02:12 +0200 Subject: [PATCH 6/6] Create Jenkinsfile.groovy --- Installation/Pipeline/Jenkinsfile.groovy | 861 +++++++++++++++++++++++ 1 file changed, 861 insertions(+) create mode 100644 Installation/Pipeline/Jenkinsfile.groovy diff --git a/Installation/Pipeline/Jenkinsfile.groovy b/Installation/Pipeline/Jenkinsfile.groovy new file mode 100644 index 000000000000..83ba5ae30f0d --- /dev/null +++ b/Installation/Pipeline/Jenkinsfile.groovy @@ -0,0 +1,861 @@ +// -*- mode: groovy-mode + +properties( + [[ + $class: 'BuildDiscarderProperty', + strategy: [$class: 'LogRotator', + artifactDaysToKeepStr: '3', + artifactNumToKeepStr: '5', + daysToKeepStr: '3', + numToKeepStr: '5'] + ]] +) + +def defaultLinux = true +def defaultMac = false +def defaultWindows = false +def defaultBuild = true +def defaultCleanBuild = false +def defaultCommunity = true +def defaultEnterprise = true +// def defaultRunResilience = false +def defaultRunTests = true + +properties([ + parameters([ + booleanParam( + defaultValue: defaultLinux, + description: 'build and run tests on Linux', + name: 'Linux' + ), + booleanParam( + defaultValue: defaultMac, + description: 'build and run tests on Mac', + name: 'Mac' + ), + booleanParam( + defaultValue: defaultWindows, + description: 'build and run tests in Windows', + name: 'Windows' + ), + booleanParam( + defaultValue: defaultCleanBuild, + description: 'clean build directories', + name: 'cleanBuild' + ), + booleanParam( + defaultValue: defaultCommunity, + description: 'build and run tests for community', + name: 'Community' + ), + booleanParam( + defaultValue: defaultEnterprise, + description: 'build and run tests for enterprise', + name: 'Enterprise' + ), + // booleanParam( + // defaultValue: defaultRunResilience, + // description: 'run resilience tests', + // name: 'runResilience' + // ), + booleanParam( + defaultValue: defaultRunTests, + description: 'run tests', + name: 'runTests' + ) + ]) +]) + +// start with empty build directory +cleanBuild = params.cleanBuild + +// build community +useCommunity = params.Community + +// build enterprise +useEnterprise = params.Enterprise + +// build linux +useLinux = params.Linux + +// build mac +useMac = params.Mac + +// build windows +useWindows = params.Windows + +// run resilience tests +//runResilience = params.runResilience + +// run tests +runTests = params.runTests + +// restrict builds +restrictions = [:] + +// ----------------------------------------------------------------------------- +// --SECTION-- CONSTANTS AND HELPERS +// ----------------------------------------------------------------------------- + + +// github proxy repositiory +proxyRepo = 'http://c1:8088/github.com/arangodb/arangodb' + +// github repositiory for resilience tests +// resilienceRepo = 'http://c1:8088/github.com/arangodb/resilience-tests' + +// github repositiory for enterprise version +enterpriseRepo = 'http://c1:8088/github.com/arangodb/enterprise' + +// Jenkins credentials for enterprise repositiory +credentials = '8d893d23-6714-4f35-a239-c847c798e080' + +// source branch for pull requests +sourceBranchLabel = env.BRANCH_NAME + +if (env.BRANCH_NAME =~ /^PR-/) { + def prUrl = new URL("https://api.github.com/repos/arangodb/arangodb/pulls/${env.CHANGE_ID}") + sourceBranchLabel = new groovy.json.JsonSlurper().parseText(prUrl.text).head.label + + def reg = ~/^arangodb:/ + sourceBranchLabel = sourceBranchLabel - reg +} + +if (sourceBranchLabel == ~/devel$/) { + useWindows = true + useMac = true +} + + +// ----------------------------------------------------------------------------- +// --SECTION-- SCRIPTS SCM +// ----------------------------------------------------------------------------- + +def checkoutCommunity() { + if (cleanBuild) { + deleteDir() + } + retry(3) { + try { + checkout( + changelog: false, + poll: false, + scm: [ + $class: 'GitSCM', + branches: [[name: "*/feature/improve-jenkins"]], + doGenerateSubmoduleConfigurations: false, + extensions: [], + submoduleCfg: [], + userRemoteConfigs: [[url: proxyRepo]]]) + } + catch (exc) { + echo "GITHUB checkout failed, retrying in 1min" + sleep 60 + throw exc + } + } +} + +def checkoutEnterprise() { + try { + echo "Trying enterprise branch ${sourceBranchLabel}" + + checkout( + changelog: false, + poll: false, + scm: [ + $class: 'GitSCM', + branches: [[name: "*/${sourceBranchLabel}"]], + doGenerateSubmoduleConfigurations: false, + extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'enterprise']], + submoduleCfg: [], + userRemoteConfigs: [[credentialsId: credentials, url: enterpriseRepo]]]) + } + catch (exc) { + echo "Failed ${sourceBranchLabel}, trying enterprise branch devel" + + checkout( + changelog: false, + poll: false, + scm: [ + $class: 'GitSCM', + branches: [[name: "*/devel"]], + doGenerateSubmoduleConfigurations: false, + extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'enterprise']], + submoduleCfg: [], + userRemoteConfigs: [[credentialsId: credentials, url: enterpriseRepo]]]) + } + +} + +// def checkoutResilience() { +// checkout( +// changelog: false, +// poll: false, +// scm: [ +// $class: 'GitSCM', +// branches: [[name: "*/master"]], +// doGenerateSubmoduleConfigurations: false, +// extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'resilience']], +// submoduleCfg: [], +// userRemoteConfigs: [[credentialsId: credentials, url: resilienceRepo]]]) + +// } + +def checkCommitMessages() { + def causes = currentBuild.rawBuild.getCauses() + def causeDescription = causes[0].getShortDescription(); + def changeLogSets = currentBuild.changeSets + def seenCommit = false + def skip = false + + for (int i = 0; i < changeLogSets.size(); i++) { + def entries = changeLogSets[i].items + + for (int j = 0; j < entries.length; j++) { + seenCommit = true + + def entry = entries[j] + + def author = entry.author + def commitId = entry.commitId + def msg = entry.msg + def timestamp = new Date(entry.timestamp) + + echo msg + + if (msg ==~ /(?i).*\[ci:[^\]]*clean[ \]].*/) { + echo "using clean build because message contained 'clean'" + cleanBuild = true + } + + if (msg ==~ /(?i).*\[ci:[^\]]*skip[ \]].*/) { + echo "skipping everything because message contained 'skip'" + skip = true + } + + def files = new ArrayList(entry.affectedFiles) + + for (int k = 0; k < files.size(); k++) { + def file = files[k] + def editType = file.editType.name + def path = file.path + + echo "File " + file + ", path " + path + } + } + } + + if (causeDescription =~ /Started by user/) { + echo "build started by user" + } + else if (skip) { + useLinux = false + useMac = false + useWindows = false + useCommunity = false + useEnterprise = false + // runResilience = false + runTests = false + } + else { + useLinux = true + useMac = true + useWindows = true + useCommunity = true + useEnterprise = true + // runResilience = true + runTests = true + if (env.BRANCH_NAME == "devel" || env.BRANCH_NAME == "3.2") { + echo "build of main branch" + } + else if (env.BRANCH_NAME =~ /^PR-/) { + echo "build of PR" + + restrictions = [ + "build-community-linux" : true, + "build-community-mac" : true, + "build-community-windows" : true, + "build-enterprise-linux" : true, + "build-enterprise-mac" : true, + "build-enterprise-windows" : true, + "test-cluster-community-mmfiles-linux" : true, + "test-cluster-community-rocksdb-linux" : true, + "test-cluster-enterprise-mmfiles-linux" : true, + "test-cluster-enterprise-rocksdb-linux" : true, + "test-singleserver-community-mmfiles-linux" : true, + "test-singleserver-community-rocksdb-linux" : true, + "test-singleserver-enterprise-mmfiles-linux" : true, + "test-singleserver-enterprise-rocksdb-linux" : true + ] + } + else { + + restrictions = [ + "build-community-mac" : true, + // "build-community-windows" : true, + "build-enterprise-linux" : true, + "test-cluster-enterprise-rocksdb-linux" : true, + "test-singleserver-community-mmfiles-mac" : true + // "test-singleserver-community-rocksdb-windows" : true + ] + } + } + + echo """BRANCH_NAME: ${env.BRANCH_NAME} +SOURCE: ${sourceBranchLabel} +CHANGE_ID: ${env.CHANGE_ID} +CHANGE_TARGET: ${env.CHANGE_TARGET} +JOB_NAME: ${env.JOB_NAME} +CAUSE: ${causeDescription} + +Linux: ${useLinux} +Mac: ${useMac} +Windows: ${useWindows} +Clean Build: ${cleanBuild} +Building Community: ${useCommunity} +Building Enterprise: ${useEnterprise} +Running Tests: ${runTests} + +Restrictions: ${restrictions.keySet().join(", ")} +""" +} + +// ----------------------------------------------------------------------------- +// --SECTION-- SCRIPTS STASH +// ----------------------------------------------------------------------------- + +def stashBinaries(os, edition) { + stash name: "binaries-${os}-${edition}", includes: "build/bin/**, build/tests/**, build/etc/**, etc/**, Installation/Pipeline/**, js/**, scripts/**, UnitTests/**, utils/**, resilience/**, enterprise/js/**", excludes: "build/bin/*.exe, build/bin/*.pdb, build/bin/*.ilk, build/tests/*.exe, build/tests/*.pdb, build/tests/*.ilk, js/node/node_modules/eslint*" +} + +def unstashBinaries(os, edition) { + unstash name: "binaries-${os}-${edition}" +} + +// ----------------------------------------------------------------------------- +// --SECTION-- VARIABLES +// ----------------------------------------------------------------------------- + +buildJenkins = [ + "linux": "linux && build", + "mac" : "mac", + "windows": "windows" +] + +testJenkins = [ + "linux": "linux && tests", + "mac" : "mac", + "windows": "windows" +] + +// ----------------------------------------------------------------------------- +// --SECTION-- SCRIPTS JSLINT +// ----------------------------------------------------------------------------- + +def jslint() { + sh './Installation/Pipeline/test_jslint.sh' +} + +// ----------------------------------------------------------------------------- +// --SECTION-- SCRIPTS TESTS +// ----------------------------------------------------------------------------- + +def getStartPort(os) { + if (os == "windows") { + return powershell (returnStdout: true, script: "Installation/Pipeline/port.ps1") + } else { + return sh (returnStdout: true, script: "Installation/Pipeline/port.sh") + } +} + +def rspecify(os, test) { + if (os == "windows") { + return [test, test, "--rspec C:\\tools\\ruby23\\bin\\rspec.bat"] + } else { + return [test, test, ""] + } +} + +def getTests(os, edition, mode, engine) { + if (mode == "singleserver") { + return [ + ["agency","agency" ,""], + ["boost", "boost", "--skipCache false"], + ["arangobench","arangobench" ,""], + ["arangosh","arangosh" ,""], + ["authentication", "authentication",""], + ["authentication_parameters","authentication_parameters" ,""], + ["cluster_sync","cluster_sync" ,""], + ["config","config" ,""], + ["dfdb","dfdb" ,""], + //"dump", + //"dump_authentication", + ["endpoints","endpoints" ,""], + rspecify(os, "http_replication"), + rspecify(os, "http_server"), + ["replication_sync", "replication_sync",""], + ["replication_static", "replication_static",""], + ["replication_ongoing","replication_ongoing" ,""], + ["server_http","server_http" ,""], + ["shell_client", "shell_client",""], + ["shell_replication","shell_replication" ,""], + ["shell_server","shell_server" ,""], + ["shell_server_aql_1", "shell_server_aql","--testBuckets 4/0", ,""], + ["shell_server_aql_2", "shell_server_aql","--testBuckets 4/1", ,""], + ["shell_server_aql_3", "shell_server_aql","--testBuckets 4/2", ,""], + ["shell_server_aql_4", "shell_server_aql","--testBuckets 4/3", ,""], + rspecify(os, "ssl_server"), + ["upgrade","upgrade" , ""] + ] + } else { + return [ + ["arangobench","arangobench" , ""], + ["arangosh","arangosh" , ""], + ["authentication","authentication" , ""], + ["authentication_parameters","authentication_parameters" , ""], + ["config","config" , ""], + ["dump","dump" , ""], + ["dump_authentication","dump_authentication" , ""], + ["endpoints","endpoints" , ""], + rspecify(os, "http_server"), + ["server_http", "server_http", ""], + ["shell_client", "shell_client", ""], + ["shell_server", "shell_server", ""], + ["shell_server_aql_1", "shell_server_aql","--testBuckets 4/0"], + ["shell_server_aql_2", "shell_server_aql","--testBuckets 4/1"], + ["shell_server_aql_3", "shell_server_aql","--testBuckets 4/2"], + ["shell_server_aql_4", "shell_server_aql","--testBuckets 4/3"], + rspecify(os, "ssl_server"), + ["upgrade","upgrade" , ""] + ] + } +} + +def testEdition(os, edition, mode, engine, port) { + def arch = "LOG_test_${os}_${edition}_${mode}_${engine}" + + if (os == 'linux' || os == 'mac') { + sh "rm -rf ${arch}" + sh "mkdir -p ${arch}" + } + else if (os == 'windows') { + bat "del /F /Q ${arch}" + powershell "New-Item -ItemType Directory -Force -Path ${arch}" + } + + def parallelity = 4 + def testIndex = 0 + def tests = getTests(os, edition, mode, engine) + + // this is an `Array.reduce()` in groovy :S + def testSteps = tests.inject([:]) { testMap, testStruct -> + def lockIndex = testIndex % parallelity + testIndex++ + + def name = testStruct[0] + def test = testStruct[1] + def testArgs = "--prefix ${os}-${edition}-${mode}-${engine} --configDir etc/jenkins --skipLogAnalysis true --skipTimeCritical true --skipNonDeterministic true --storageEngine ${engine} " + testStruct[2] + + def portInterval = 10 + if (mode == "cluster") { + portInterval = 40 + testArgs += " --cluster true" + } + + testMap["test-${os}-${edition}-${mode}-${engine}-${name}"] = { + // copy in groovy + testArgs += " --minPort " + port + testArgs += " --maxPort " + (port + portInterval - 1) + def command = "build/bin/arangosh --log.level warning --javascript.execute UnitTests/unittest.js ${test} -- " + command += testArgs + lock("test-${env.NODE_NAME}-${env.JOB_NAME}-${env.BUILD_ID}-${edition}-${engine}-${lockIndex}") { + timeout(15) { + if (os == "windows") { + powershell command + } else { + sh command + } + } + } + } + port += portInterval + testMap + } + + parallel testSteps + + if (os == 'windows') { + if (findFiles(glob: '*.dmp').length > 0) { + error("found dmp file") + } + } else { + if (findFiles(glob: 'core*').length > 0) { + error("found core file") + } + } +} + +def testCheck(os, edition, mode, engine) { + if (! runTests) { + echo "Not testing ${os} ${mode} because testing is not enabled" + return false + } + + if (os == 'linux' && ! useLinux) { + echo "Not testing ${os} ${mode} because testing on ${os} is not enabled" + return false + } + + if (os == 'mac' && ! useMac) { + echo "Not testing ${os} ${mode} because testing on ${os} is not enabled" + return false + } + + if (os == 'windows' && ! useWindows) { + echo "Not testing ${os} ${mode} because testing on ${os} is not enabled" + return false + } + + if (edition == 'enterprise' && ! useEnterprise) { + echo "Not testing ${os} ${mode} ${edition} because testing ${edition} is not enabled" + return false + } + + if (edition == 'community' && ! useCommunity) { + echo "Not testing ${os} ${mode} ${edition} because testing ${edition} is not enabled" + return false + } + + if (restrictions && !restrictions["test-${mode}-${edition}-${engine}-${os}"]) { + return false + } + + return true +} + +def testStep(os, edition, mode, engine) { + return { + node(testJenkins[os]) { + def buildName = "${os}-${edition}" + def name = "${os}-${edition}-${mode}-${engine}" + stage("test-${name}") { + // seriously...60 minutes is the super absolute max max max. + // even in the worst situations ArangoDB MUST be able to finish within 60 minutes + // even if the features are green this is completely broken performance wise.. + // DO NOT INCREASE!! + def port = 0 + unstashBinaries(os, edition) + port = getStartPort(os) as Integer + echo "Using start port: ${port}" + if (os == "windows") { + powershell "copy build\\bin\\RelWithDebInfo\\* build\\bin" + powershell "Installation/Pipeline/include/test_setup_tmp.ps1" + } else { + sh "chmod +x Installation/Pipeline/include/test_setup_tmp.inc && sh Installation/Pipeline/include/test_setup_tmp.inc" + } + timeout(60) { + try { + testEdition(os, edition, mode, engine, port) + } + finally { + def arch = "LOG_test_${os}_${edition}_${mode}_${engine}" + // step([$class: 'XUnitBuilder', + // thresholds: [[$class: 'FailedThreshold', unstableThreshold: '1']], + // tools: [[$class: 'JUnitType', failIfNotNew: false, pattern: 'out/*.xml']]]) + + if (os == 'linux' || os == 'mac') { + sh "find log-output -name 'FAILED_*' -exec cp '{}' . ';'" + sh "for i in logs log-output; do test -e \"\$i\" && mv \"\$i\" ${arch} || true; done" + sh "for i in core* tmp; do test -e \"\$i\" && mv \"\$i\" ${arch} || true; done" + sh "cp -a build/bin/* ${arch}" + if (port > 0) { + sh "Installation/Pipeline/port.sh --clean ${port}" + } + } else if (os == 'windows') { + powershell "move-item -Force -ErrorAction Ignore logs ${arch}" + powershell "move-item -Force -ErrorAction Ignore log-output ${arch}" + powershell "move-item -Force -ErrorAction Ignore .\\build\\bin\\*.dmp ${arch}" + powershell "move-item -Force -ErrorAction Ignore .\\build\\tests\\*.dmp ${arch}" + powershell "Copy-Item .\\build\\bin\\* -Include *.exe,*.pdb,*.ilk ${arch}" + } + + archiveArtifacts allowEmptyArchive: true, + artifacts: "${arch}/**, FAILED_*", + defaultExcludes: false + } + } + } + } + } +} + +def testStepParallel(osList, editionList, modeList) { + def branches = [:] + + for (os in osList) { + for (edition in editionList) { + for (mode in modeList) { + for (engine in ['mmfiles', 'rocksdb']) { + if (testCheck(os, edition, mode, engine)) { + def name = "test-${os}-${edition}-${mode}-${engine}"; + + branches[name] = testStep(os, edition, mode, engine) + } + } + } + } + } + parallel branches +} + +// ----------------------------------------------------------------------------- +// --SECTION-- SCRIPTS RESILIENCE +// ----------------------------------------------------------------------------- + +// def testResilience(os, engine, foxx) { +// withEnv(['LOG_COMMUNICATION=debug', 'LOG_REQUESTS=trace', 'LOG_AGENCY=trace']) { +// if (os == 'linux') { +// sh "./Installation/Pipeline/linux/test_resilience_${foxx}_${engine}_${os}.sh" +// } +// else if (os == 'mac') { +// sh "./Installation/Pipeline/mac/test_resilience_${foxx}_${engine}_${os}.sh" +// } +// else if (os == 'windows') { +// powershell ".\\Installation\\Pipeline\\test_resilience_${foxx}_${engine}_${os}.ps1" +// } +// } +// } + +// def testResilienceCheck(os, engine, foxx) { +// if (! runResilience) { +// return false +// } + +// if (os == 'linux' && ! useLinux) { +// return false +// } + +// if (os == 'mac' && ! useMac) { +// return false +// } + +// if (os == 'windows' && ! useWindows) { +// return false +// } + +// if (! useCommunity) { +// return false +// } + +// if (restrictions && !restrictions["test-resilience-${foxx}-${engine}-${os}"]) { +// return false +// } + +// return true +// } + +// def testResilienceStep(os, engine, foxx) { +// return { +// node(testJenkins[os]) { +// def edition = "community" +// def buildName = "${edition}-${os}" + +// def name = "${os}-${engine}-${foxx}" +// def arch = "LOG_resilience_${foxx}_${engine}_${os}" + +// stage("resilience-${name}") { +// if (os == 'linux' || os == 'mac') { +// sh "rm -rf ${arch}" +// sh "mkdir -p ${arch}" +// } +// else if (os == 'windows') { +// bat "del /F /Q ${arch}" +// powershell "New-Item -ItemType Directory -Force -Path ${arch}" +// } + +// try { +// try { +// timeout(120) { +// unstashBinaries(edition, os) +// testResilience(os, engine, foxx) +// } + +// if (findFiles(glob: 'resilience/core*').length > 0) { +// error("found core file") +// } +// } +// catch (exc) { +// if (os == 'linux' || os == 'mac') { +// sh "for i in build resilience/core* tmp; do test -e \"\$i\" && mv \"\$i\" ${arch} || true; done" +// } +// throw exc +// } +// finally { +// if (os == 'linux' || os == 'mac') { +// sh "for i in log-output; do test -e \"\$i\" && mv \"\$i\" ${arch}; done" +// } +// else if (os == 'windows') { +// bat "move log-output ${arch}" +// } +// } +// } +// finally { +// archiveArtifacts allowEmptyArchive: true, +// artifacts: "${arch}/**", +// defaultExcludes: false +// } +// } +// } +// } +// } + +// def testResilienceParallel(osList) { +// def branches = [:] + +// for (foxx in ['foxx', 'nofoxx']) { +// for (os in osList) { +// for (engine in ['mmfiles', 'rocksdb']) { +// if (testResilienceCheck(os, engine, foxx)) { +// def name = "test-resilience-${foxx}-${engine}-${os}" + +// branches[name] = testResilienceStep(os, engine, foxx) +// } +// } +// } +// } + +// if (branches.size() > 1) { +// parallel branches +// } +// else if (branches.size() == 1) { +// branches.values()[0]() +// } +// } + +// ----------------------------------------------------------------------------- +// --SECTION-- SCRIPTS BUILD +// ----------------------------------------------------------------------------- + +def buildEdition(os, edition) { + def arch = "LOG_build_${os}_${edition}" + + if (os == 'linux' || os == 'mac') { + sh "rm -rf ${arch}" + sh "mkdir -p ${arch}" + } + else if (os == 'windows') { + bat "del /F /Q ${arch}" + powershell "New-Item -ItemType Directory -Force -Path ${arch}" + } + + try { + if (os == 'linux') { + sh "./Installation/Pipeline/linux/build_${os}_${edition}.sh 64" + } + else if (os == 'mac') { + sh "./Installation/Pipeline/mac/build_${os}_${edition}.sh 16" + } + else if (os == 'windows') { + // i concede...we need a lock for windows...I could not get it to run concurrently... + // v8 would not build multiple times at the same time on the same machine: + // PDB API call failed, error code '24': ' etc etc + // in theory it should be possible to parallelize it by setting an environment variable (see the build script) but for v8 it won't work :( + // feel free to recheck if there is time somewhen...this thing here really should not be possible but + // ensure that there are 2 concurrent builds on the SAME node building v8 at the same time to properly test it + // I just don't want any more "yeah that might randomly fail. just restart" sentences any more + def hostname = powershell(returnStdout: true, script: "hostname") + lock('build-${hostname}') { + powershell ". .\\Installation\\Pipeline\\windows\\build_${os}_${edition}.ps1" + } + } + } + finally { + if (os == 'linux' || os == 'mac') { + sh "for i in log-output; do test -e \"\$i\" && mv \"\$i\" ${arch} || true; done" + } + else if (os == 'windows') { + powershell "Move-Item -ErrorAction Ignore -Path log-output/* -Destination ${arch}" + } + } +} + +def buildStepCheck(os, edition) { + if (os == 'linux' && ! useLinux) { + return false + } + + if (os == 'mac' && ! useMac) { + return false + } + + if (os == 'windows' && ! useWindows) { + return false + } + + if (edition == 'enterprise' && ! useEnterprise) { + return false + } + + if (edition == 'community' && ! useCommunity) { + return false + } + + if (restrictions && !restrictions["build-${edition}-${os}"]) { + return false + } + + return true +} + +def runEdition(os, edition) { + return { + node(buildJenkins[os]) { + def name = "${os}-${edition}" + + stage("build-${name}") { + timeout(30) { + checkoutCommunity() + checkCommitMessages() + if (edition == "enterprise") { + checkoutEnterprise() + } + // checkoutResilience() + } + + timeout(90) { + buildEdition(os, edition) + stashBinaries(os, edition) + } + + // we only need one jslint test per edition + if (os == "linux") { + stage("jslint-${edition}") { + echo "Running jslint for ${edition}" + jslint() + } + } + } + } + testStepParallel([os], [edition], ['cluster', 'singleserver']) + } +} + +def runOperatingSystems(osList) { + def branches = [:] + + for (os in osList) { + for (edition in ['community', 'enterprise']) { + if (buildStepCheck(os, edition)) { + branches["build-${os}-${edition}"] = runEdition(os, edition) + } + } + } + parallel branches +} + +// ----------------------------------------------------------------------------- +// --SECTION-- PIPELINE +// ----------------------------------------------------------------------------- + +runOperatingSystems(['linux', 'mac', 'windows'])