8000 Feature/jenkins pipeline (#3319) · MohammedDeveloper/arangodb@c30fdd5 · GitHub
[go: up one dir, main page]
Skip to content

Commit c30fdd5

Browse files
fcellerfceller
authored
Feature/jenkins pipeline (arangodb#3319)
1 parent f32329d commit c30fdd5

File tree

2 files changed

+141
-26
lines changed

2 files changed

+141
-26
lines changed

‎Installation/Pipeline/Jenkinsfile.groovy

Lines changed: 135 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,7 @@
11
// -*- mode: groovy-mode
22

3-
properties(
4-
[[
5-
$class: 'BuildDiscarderProperty',
6-
strategy: [$class: 'LogRotator',
7-
artifactDaysToKeepStr: '3',
8-
artifactNumToKeepStr: '5',
9-
daysToKeepStr: '3',
10-
numToKeepStr: '5']
11-
]]
12-
)
3+
properties([buildDiscarder(logRotator(artifactDaysToKeepStr: '3', artifactNumToKeepStr: '5 10000 ', daysToKeepStr: '3', numToKeepStr: '5'))])
4+
135

146
// -----------------------------------------------------------------------------
157
// --SECTION-- SELECTABLE PARAMETERS
@@ -121,6 +113,12 @@ restrictions = [:]
121113
// overview of configured builds and tests
122114
overview = ""
123115

116+
// results
117+
resultsKeys = []
118+
resultsStart = [:]
119+
resultsStop = [:]
120+
resultsStatus = [:]
121+
124122
// -----------------------------------------------------------------------------
125123
// --SECTION-- CONSTANTS AND HELPERS
126124
// -----------------------------------------------------------------------------
@@ -226,7 +224,24 @@ def checkEnabledMaintainer(maintainer, os, text) {
226224
return true
227225
}
228226

229-
def checkCoresAndSave(os, runDir, name, archRun) {
227+
def checkCores(os, runDir) {
228+
if (os == 'windows') {
229+
def files = findFiles(glob: "${runDir}/*.dmp")
230+
231+
if (files.length > 0) {
232+
error("found windows core file")
233+
}
234+
}
235+
else {
236+
def files = findFiles(glob: "${runDir}/core*")
237+
238+
if (files.length > 0) {
239+
error("found linux core file")
240+
}
241+
}
242+
}
243+
244+
def saveCores(os, runDir, name, archRun) {
230245
if (os == 'windows') {
231246
powershell "move-item -Force -ErrorAction Ignore ${runDir}/logs ${archRun}/${name}.logs"
232247
powershell "move-item -Force -ErrorAction Ignore ${runDir}/out ${archRun}/${name}.logs"
@@ -241,7 +256,7 @@ def checkCoresAndSave(os, runDir, name, archRun) {
241256

242257
powershell "copy-item .\\build\\bin\\* -Include *.exe,*.pdb,*.ilk ${archRun}"
243258

244-
error("found dmp file")
259+
return true
245260
}
246261
}
247262
else {
@@ -256,9 +271,11 @@ def checkCoresAndSave(os, runDir, name, archRun) {
256271

257272
sh "cp -a build/bin/* ${archRun}"
258273

259-
error("found core file")
274+
return true
260275
}
261276
}
277+
278+
return false
262279
}
263280

264281
def getStartPort(os) {
@@ -290,6 +307,31 @@ def shellAndPipe(command, logfile) {
290307
sh "(echo 1 > \"${logfile}.result\" ; ${command} ; echo \$? > \"${logfile}.result\") 2>&1 | tee -a \"${logfile}\" ; exit `cat \"${logfile}.result\"`"
291308
}
292309

310+
def logStartStage(logFile) {
311+
resultsKeys << logFile
312+
313+
resultsStart[logFile] = new Date()
314+
resultsStatus[logFile] = "started"
315+
316+
echo "started ${logFile}: ${resultsStart[logFile]}"
317+
}
318+
319+
def logStopStage(logFile) {
320+
resultsStop[logFile] = new Date()
321+
resultsStatus[logFile] = "finished"
322+
323+
echo "finished ${logFile}: ${resultsStop[logFile]}"
324+
}
325+
326+
def logExceptionStage(logFile, exc) {
327+
def msg = exc.toString()
328+
329+
resultsStop[logFile] = new Date()
330+
resultsStatus[logFile] = "failed ${msg}"
331+
332+
echo "failed ${logFile}: ${resultsStop[logFile]} ${msg}"
333+
}
334+
293335
// -----------------------------------------------------------------------------
294336
// --SECTION-- SCRIPTS SCM
295337
// -----------------------------------------------------------------------------
@@ -594,10 +636,16 @@ def jslint(os, edition, maintainer) {
594636
def logFile = "${arch}/jslint.log"
595637

596638
try {
639+
logStartStage(logFile)
640+
597641
shellAndPipe("./Installation/Pipeline/test_jslint.sh",logFile)
598642
sh "if grep ERROR ${logFile}; then exit 1; fi"
643+
644+
logStopStage(logFile)
599645
}
600646
catch (exc) {
647+
logExceptionStage(logFile, exc)
648+
601649
renameFolder(arch, archFail)
602650
fileOperations([fileCreateOperation(fileContent: 'JSLINT FAILED', fileName: "${archDir}-FAIL.txt")])
603651
throw exc
@@ -696,7 +744,7 @@ def setupTestEnvironment(os, edition, maintainer, logFile, runDir) {
696744
}
697745

698746
def executeTests(os, edition, maintainer, mode, engine, portInit, archDir, arch, stageName) {
699-
def parallelity = 4
747+
def parallelity = (mode == "cluster") ? ((os == "linux") ? 5 : 2) : ((os == "linux") ? 10 : 4)
700748
def testIndex = 0
701749
def tests = getTests(os, edition, maintainer, mode, engine)
702750

@@ -725,6 +773,7 @@ def executeTests(os, edition, maintainer, mode, engine, portInit, archDir, arch,
725773

726774
testMap["${stageName}-${name}"] = {
727775
def logFile = pwd() + "/" + "${arch}/${name}.log"
776+
def logFileRel = "${arch}/${name}.log"
728777
def logFileFailed = pwd() + "/" + "${arch}-FAIL/${name}.log"
729778
def archRun = pwd() + "/" + "${arch}-RUN"
730779

@@ -745,6 +794,8 @@ def executeTests(os, edition, maintainer, mode, engine, portInit, archDir, arch,
745794
setupTestEnvironment(os, edition, maintainer, logFile, runDir)
746795

747796
try {
797+
logStartStage(logFileRel)
798+
748799
// seriously...30 minutes is the super absolute max max max.
749800
// even in the worst situations ArangoDB MUST be able to finish within 60 minutes
750801
// even if the features are green this is completely broken performance wise..
@@ -773,8 +824,13 @@ def executeTests(os, edition, maintainer, mode, engine, portInit, archDir, arch,
773824
}
774825
}
775826
}
827+
828+
checkCores(os, runDir)
829+
logStopStage(logFileRel)
776830
}
777831
catch (exc) {
832+
logExceptionStage(logFileRel, exc)
833+
778834
def msg = exc.toString()
779835

780836
echo "caught error, copying log to ${logFileFailed}: ${msg}"
@@ -794,10 +850,9 @@ def executeTests(os, edition, maintainer, mode, engine, portInit, archDir, arch,
794850
throw exc
795851
}
796852
finally {
797-
def logFileRel = "${arch}/${name}.log"
798853
def logFileFailedRel = "${arch}-FAIL/${name}.log"
799854

800-
checkCoresAndSave(os, runDir, name, archRun)
855+
saveCores(os, runDir, name, archRun)
801856

802857
archiveArtifacts allowEmptyArchive: true,
803858
artifacts: "${archDir}-FAIL.txt, ${logFileRel}, ${logFileFailedRel}",
@@ -902,7 +957,17 @@ def testStepParallel(os, edition, maintainer, modeList) {
902957
}
903958
}
904959

905-
parallel branches
960+
def name = "test-${os}-${edition}-${maintainer}"
961+
962+
try {
963+
logStartStage(name)
964+
parallel branches
965+
logStopStage(name)
966+
}
967+
catch (exc) {
968+
logExceptionStage(name, exc)
969+
throw exc
970+
}
906971
}
907972

908973
// -----------------------------------------------------------------------------
@@ -1048,6 +1113,8 @@ def buildEdition(os, edition, maintainer) {
10481113
def logFile = "${arch}/build.log"
10491114

10501115
try {
1116+
logStartStage(logFile)
1117+
10511118
if (os == 'linux' || os == 'mac') {
10521119
sh "echo \"Host: `hostname`\" | tee -a ${logFile}"
10531120
sh "echo \"PWD: `pwd`\" | tee -a ${logFile}"
@@ -1078,8 +1145,12 @@ def buildEdition(os, edition, maintainer) {
10781145

10791146
powershell ". .\\Installation\\Pipeline\\windows\\build_${os}_${edition}_${maintainer}.ps1"
10801147
}
1148+
1149+
logStopStage(logFile)
10811150
}
10821151
catch (exc) {
1152+
logExceptionStage(logFile, exc)
1153+
10831154
def msg = exc.toString()
10841155

10851156
fileOperations([
@@ -1162,11 +1233,17 @@ def createDockerImage(edition, maintainer, stageName) {
11621233

11631234
withEnv(["DOCKERTAG=${packageName}-${dockerTag}"]) {
11641235
try {
1236+
logStartStage(logFile)
1237+
11651238
shellAndPipe("./scripts/build-docker.sh", logFile)
11661239
shellAndPipe("docker tag arangodb:${packageName}-${dockerTag} c1.triagens-gmbh.zz:5000/arangodb/${packageName}:${dockerTag}", logFile)
11671240
shellAndPipe("docker push c1.triagens-gmbh.zz:5000/arangodb/${packageName}:${dockerTag}", logFile)
1241+
1242+
logStopStage(logFile)
11681243
}
11691244
catch (exc) {
1245+
logExceptionStage(logFile, exc)
1246+
11701247
renameFolder(arch, archFail)
11711248
fileOperations([fileCreateOperation(fileContent: 'DOCKER FAILED', fileName: "${archDir}-FAIL.txt")])
11721249
throw exc
@@ -1256,16 +1333,49 @@ def runOperatingSystems(osList) {
12561333
}
12571334

12581335
timestamps {
1259-
node("master") {
1260-
echo sh(returnStdout: true, script: 'env')
1261-
}
1336+
try {
1337+
node("master") {
1338+
echo sh(returnStdout: true, script: 'env')
1339+
}
1340+
1341+
checkCommitMessages()
12621342

1263-
checkCommitMessages()
1343+
node("master") {
1344+
fileOperations([fileCreateOperation(fileContent: overview, fileName: "overview.txt")])
1345+
archiveArtifacts(allowEmptyArchive: true, artifacts: "overview.txt")
1346+
}
12641347

1265-
node("master") {
1266-
fileOperations([fileCreateOperation(fileContent: overview, fileName: "overview.txt")])
1267-
archiveArtifacts(allowEmptyArchive: true, artifacts: "overview.txt")
1348+
runOperatingSystems(['linux', 'mac', 'windows'])
12681349
}
1350+
finally {
1351+
results = ""
1352+
html = "<html><body><table>\n"
1353+
html += "<tr><th>Name</th><th>Start</th><th>Stop</th><th>Duration</th><th>Message</th></tr>\n"
1354+
1355+
for (key in resultsKeys) {
1356+
def start = resultsStart[key] ?: ""
1357+
def stop = resultsStop[key] ?: ""
1358+
def msg = resultsStatus[key] ?: ""
1359+
def diff = (start != "" && stop != "") ? groovy.time.TimeCategory.minus(stop, start) : "-"
1360+
def startf = start.format('yyyy/MM/dd HH:mm:ss')
1361+
def stopf = stop.format('yyyy/MM/dd HH:mm:ss')
1362+
def color = 'bgcolor="#FF8080"'
1363+
1364+
if (msg == "finished") {
1365+
color = 'bgcolor="#80FF80"'
1366+
}
1367+
1368+
results += "${key}: ${startf} - ${stopf} (${diff}) ${msg}\n"
1369+
html += "<tr ${color}><td>${key}</td><td>${startf}</td><td>${stopf}</td><td align=\"right\">${diff}</td><td align=\"right\">${msg}</td></tr>\n"
1370+
}
1371+
1372+
html += "</table></body></html>\n"
12691373

1270-
runOperatingSystems(['linux', 'mac', 'windows'])
1374+
node("master") {
1375+
fileOperations([fileCreateOperation(fileContent: results, fileName: "results.txt")])
1376+
fileOperations([fileCreateOperation(fileContent: html, fileName: "results.html")])
1377+
archiveArtifacts(allowEmptyArchive: true, artifacts: "results.*")
1378+
}
1379+
}
12711380
}
1381+

‎Installation/Pipeline/build_OS_EDITION_MAINTAINER.sh

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,10 @@ else
3232
exit 1
3333
fi
3434

35+
PACKAGING=
36+
3537
if [ "$os" == linux ]; then
38+
PACKAGING="-DPACKAGING=DEB"
3639
type="${type}_linux"
3740
load=40
3841
elif [ "$os" == mac ]; then
@@ -65,10 +68,12 @@ echo "PWD: `pwd`" | tee -a $logdir/build.log
6568
CXXFLAGS=-fno-omit-frame-pointer \
6669
cmake \
6770
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
71+
-DCMAKE_INSTALL_PREFIX=/ \
6872
-DUSE_CATCH_TESTS=On \
6973
-DDEBUG_SYNC_REPLICATION=On \
70-
$MAINTAINER \
7174
$ENTERPRISE \
75+
$MAINTAINER \
76+
$PACKAGING \
7277
.. 2>&1 | tee -a ../$logdir/build.log
7378

7479
echo "`date +%T` building..."

0 commit comments

Comments
 (0)
0