8000 Merge pull request #4359 from adriaanm/2.10.x · scala/scala-dev@8f0819a · GitHub
[go: up one dir, main page]
Skip to content

Commit 8f0819a

Browse files
gkossakowskigkossakowski
committed
Merge pull request #4359 from adriaanm/2.10.x
Port old 2.10 release script to new CI.
2 parents b2f5101 + 132d99f commit 8f0819a

File tree

1 file changed

+23
-301
lines changed

1 file changed

+23
-301
lines changed

‎scripts/jobs/integrate/bootstrap

Lines changed: 23 additions & 301 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,11 @@
11
#!/bin/bash -e
2+
3+
4+
5+
# NOTE: this is a quick backport of the 2.11.x script to 2.10.x -- some comments may be out dated
6+
7+
8+
29
# TODO: different scripts for the different phases -- usually we don't need to bootstrap the modules,
310
# since we can use the previous version of scala for STARR as well as for compiling the modules (assuming it's binary compatible)
411
# We should move away from the complicated bootstrap and set up our release schedule so we always have a previous build that satisfies these criteria.
@@ -68,32 +75,17 @@
6875
# (publishing only done when $WORKSPACE checkout's HEAD is tagged / SCALA_VER_BASE is set.)
6976

7077

71-
# set to something besides the default to build nightly snapshots of the modules instead of some tagged version
72-
moduleVersioning=${moduleVersioning-"versions.properties"}
73-
74-
publishPrivateTask=${publishPrivateTask-"publish"}
75-
publishSonatypeTaskCore=${publishSonatypeTaskCore-"publish-signed"}
76-
publishSonatypeTaskModules=${publishSonatypeTaskModules-"publish-signed"}
77-
publishLockerPrivateTask=${publishLockerPrivateTask-$publishPrivateTask} # set to "init" to speed up testing of the script (if you already built locker before)
78-
7978
sbtCmd=${sbtCmd-sbt} # TESTING (this is a marker for defaults to change when testing locally: should be sbtx on my mac)
8079

8180
# 0.13.5 does not respect "set every scalaVersion", see
8281
# https://github.com/scala/scala-parser-combinators/pull/27
8382
sbtCmd="$sbtCmd -sbt-version 0.13.2"
8483

85-
forceRebuild=${forceRebuild-no}
86-
8784
# publishToSonatype
8885
# set to anything but "yes" to avoid publishing to sonatype
8986
# overridden to "no" when no SCALA_VER_BASE is passed and HEAD is not tagged with a valid version tag
9087
#
9188

92-
antBuildTask="${antBuildTask-nightly}" # TESTING leave empty to avoid the sanity check (don't set it to "init" because ant will croak)
93-
clean="clean" # TESTING leave empty to speed up testing
94-
95-
96-
9789
baseDir=${WORKSPACE-`pwd`}
9890
scriptsDir="$baseDir/scripts"
9991
. $scriptsDir/common
@@ -175,134 +167,11 @@ sbtResolve() {
175167
'show update' >> $baseDir/logs/resolution 2>&1
176168
}
177169

178-
# Oh boy... can't use scaladoc to document scala-xml/scala-parser-combinators
179-
# if scaladoc depends on the same version of scala-xml/scala-parser-combinators.
180-
# Even if that version is available through the project's resolvers, sbt won't look past this project.
181-
# SOOOOO, we set the version to a dummy (-DOC), generate documentation,
182-
# then set the version to the right one and publish (which won't re-gen the docs).
183-
# Also tried publish-local without docs using 'set publishArtifact in (Compile, packageDoc) := false' and republishing, no dice.
184-
185-
# Each buildModule() function is invoked twice: first to build against locker and publish to private-repo, then
186-
# to build against the release and publish to sonatype (or publish-local if publishToSonatype is not "yes").
187-
# In the second round, sbtResolve is always true: the module will be found in the private-repo!
188-
# Therefore, if MODULE_BUILT is "yes" (in the second round), we know that we need to build (and publish) the
189-
# module again.
190-
#
191-
# Note: we tried an alternative solution in which sbtResolve would not look at private-repo, but that fails. For example,
192-
# scala-xml depends on scala-library, so sbt tries to find the scala-library of the version that we are currently building,
193-
# which exists only in private-repo.
194-
195-
buildXML() {
196-
if [ "$XML_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-xml" $XML_VER )
197-
then echo "Found scala-xml $XML_VER; not building."
198-
else
199-
update scala scala-xml "$XML_REF" && gfxd
200-
sbtBuild 'set version := "'$XML_VER'-DOC"' $clean doc 'set version := "'$XML_VER'"' test "${buildTasks[@]}"
201-
XML_BUILT="yes" # ensure the module is built and published when buildXML is invoked for the second time, see comment above
202-
fi
203-
}
204-
205-
buildParsers() {
206-
if [ "$PARSERS_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-parser-combinators" $PARSERS_VER )
207-
then echo "Found scala-parser-combinators $PARSERS_VER; not building."
208-
else
209-
update scala scala-parser-combinators "$PARSERS_REF" && gfxd
210-
sbtBuild 'set version := "'$PARSERS_VER'-DOC"' $clean doc 'set version := "'$PARSERS_VER'"' test "${buildTasks[@]}"
211-
PARSERS_BUILT="yes"
212-
fi
213-
}
214-
215-
buildPartest() {
216-
if [ "$PARTEST_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-partest" $PARTEST_VER )
217-
then echo "Found scala-partest $PARTEST_VER; not building."
218-
else
219-
update scala scala-partest "$PARTEST_REF" && gfxd
220-
sbtBuild 'set version :="'$PARTEST_VER'"' 'set VersionKeys.scalaXmlVersion := "'$XML_VER'"' 'set VersionKeys.scalaCheckVersion := "'$SCALACHECK_VER'"' $clean test "${buildTasks[@]}"
221-
PARTEST_BUILT="yes"
222-
fi
223-
}
224-
225-
# buildPartestIface() {
226-
# if [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-partest-interface" $PARTEST_IFACE_VER )
227-
# then echo "Found scala-partest-interface $PARTEST_IFACE_VER; not building."
228-
# else
229-
# update scala scala-partest-interface "$PARTEST_IFACE_REF" && gfxd
230-
# sbtBuild 'set version :="'$PARTEST_IFACE_VER'"' $clean "${buildTasks[@]}"
231-
# fi
232-
# }
233-
234-
buildContinuations() {
235-
if [ "$CONT_PLUG_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.plugins" "scala-continuations-plugin" $CONTINUATIONS_VER full )
236-
then echo "Found scala-continuations-plugin $CONTINUATIONS_VER; not building."
237-
else
238-
update scala scala-continuations $CONTINUATIONS_REF && gfxd
239-
240-
$sbtCmd $sbtArgs 'project plugin' "${scalaVersionTasks[@]}" "${publishTasks[@]}" \
241-
'set version := "'$CONTINUATIONS_VER'"' $clean "compile:package" test "${buildTasks[@]}" # https://github.com/scala/scala-continuations/pull/4
242-
CONT_PLUG_BUILT="yes"
243-
fi
244-
245-
if [ "$CONT_LIB_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.plugins" "scala-continuations-library" $CONTINUATIONS_VER )
246-
then echo "Found scala-continuations-library $CONTINUATIONS_VER; not building."
247-
else
248-
update scala scala-continuations $CONTINUATIONS_REF && gfxd
249-
$sbtCmd $sbtArgs 'project library' "${scalaVersionTasks[@]}" "${publishTasks[@]}" \
250-
'set version := "'$CONTINUATIONS_VER'"' $clean test "${buildTasks[@]}"
251-
CONT_LIB_BUILT="yes"
252-
fi
253-
}
254-
255-
buildSwing() {
256-
if [ "$SWING_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang.modules" "scala-swing" $SWING_VER )
257-
then echo "Found scala-swing $SWING_VER; not building."
258-
else
259-
update scala scala-swing "$SWING_REF" && gfxd
260-
sbtBuild 'set version := "'$SWING_VER'"' $clean test "${buildTasks[@]}"
261-
SWING_BUILT="yes"
262-
fi
263-
}
264-
265-
buildActorsMigration(){
266-
if [ "$ACTORS_MIGRATION_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scala-lang" "scala-actors-migration" $ACTORS_MIGRATION_VER )
267-
then echo "Found scala-actors-migration $ACTORS_MIGRATION_VER; not building."
268-
else
269-
update scala actors-migration "$ACTORS_MIGRATION_REF" && gfxd
270-
# not running tests because
271-
# [error] Test scala.actors.migration.NestedReact.testNestedReactAkka failed: java.util.concurrent.TimeoutException: Futures timed out after [20 seconds]
272-
sbtBuild 'set version := "'$ACTORS_MIGRATION_VER'"' 'set VersionKeys.continuationsVersion := "'$CONTINUATIONS_VER'"' $clean "${buildTasks[@]}"
273-
ACTORS_MIGRATION_BUILT="yes"
274-
fi
275-
}
276-
277-
buildScalacheck(){
278-
if [ "$SCALACHECK_BUILT" != "yes" ] && [ "$forceRebuild" != "yes" ] && ( sbtResolve "org.scalacheck" "scalacheck" $SCALACHECK_VER )
279-
then echo "Found scalacheck $SCALACHECK_VER; not building."
280-
else
281-
update rickynils scalacheck $SCALACHECK_REF && gfxd
282-
sbtBuild 'set version := "'$SCALACHECK_VER'"' 'set VersionKeys.scalaParserCombinatorsVersion := "'$PARSERS_VER'"' $clean $publishPrivateTask # test times out NOTE: never published to sonatype
283-
SCALACHECK_BUILT="yes"
284-
fi
285-
}
286-
287-
# build modules, using ${buildTasks[@]} (except for Scalacheck, which is hard-coded to publish to private-repo)
288-
buildModules() {
289-
buildXML
290-
buildParsers
291-
buildContinuations
292-
buildSwing
293-
buildActorsMigration
294-
buildScalacheck
295-
buildPartest
296-
# buildPartestIface
297-
}
298-
299170

300171
## BUILD STEPS:
301172

302173
determineScalaVersion() {
303174
cd $WORKSPACE
304-
parseScalaProperties "versions.properties"
305-
306175
if [ -z "$SCALA_VER_BASE" ]; then
307176
echo "No SCALA_VER_BASE specified."
308177

@@ -345,12 +214,6 @@ determineScalaVersion() {
345214
echo "version=$SCALA_VER" >> $baseDir/jenkins.properties
346215
echo "sbtDistVersionOverride=-Dproject.version=$SCALA_VER" >> $baseDir/jenkins.properties
347216

348-
# We don't override the scala binary version: when running in -nightly + versions.properties versioning mode,
349-
# we intend to be a drop-in replacement -- all you need to do is change the Scala version
350-
# In order to override this, add 'set every scalaBinaryVersion := "'$SCALA_BINARY_VER'"',
351-
# which, when used with pre-release Scala version numbers, will require tweaking at the sbt usage site as well.
352-
scalaVersionTasks=('set every scalaVersion := "'$SCALA_VER'"')
353-
354217
echo "Building Scala $SCALA_VER."
355218
}
356219

@@ -364,152 +227,29 @@ deriveVersionAnyTag() {
364227
echo "$(git describe | cut -dv -f2)-nightly"
365228
}
366229

367-
# determineScalaVersion must have been called
368-
deriveModuleVersions() {
369-
if [ "$moduleVersioning" == "versions.properties" ]
370-
then
371-
# use versions.properties as defaults when no version specified on command line
372-
XML_VER=${XML_VER-$scala_xml_version_number}
373-
PARSERS_VER=${PARSERS_VER-$scala_parser_combinators_version_number}
374-
CONTINUATIONS_VER=${CONTINUATIONS_VER-$scala_continuations_plugin_version_number}
375-
SWING_VER=${SWING_VER-$scala_swing_version_number}
376-
ACTORS_MIGRATION_VER=${ACTORS_MIGRATION_VER-$actors_migration_version_number}
377-
PARTEST_VER=${PARTEST_VER-$partest_version_number}
378-
SCALACHECK_VER=${SCALACHECK_VER-$scalacheck_version_number}
379-
380-
# If a _VER was not specified, the corresponding _REF will be non-empty by now (as specified, or HEAD)
381-
XML_REF=${XML_REF-"v$XML_VER"}
382-
PARSERS_REF=${PARSERS_REF-"v$PARSERS_VER"}
383-
CONTINUATIONS_REF=${CONTINUATIONS_REF-"v$CONTINUATIONS_VER"}
384-
SWING_REF=${SWING_REF-"v$SWING_VER"}
385-
ACTORS_MIGRATION_REF=${ACTORS_MIGRATION_REF-"v$ACTORS_MIGRATION_VER"}
386-
PARTEST_REF=${PARTEST_REF-"v$PARTEST_VER"}
387-
# PARTEST_IFACE_REF=${PARTEST_IFACE_REF-"v$PARTEST_IFACE_VER"}
388-
SCALACHECK_REF=${SCALACHECK_REF-"$SCALACHECK_VER"}
389-
else
390-
XML_VER=${XML_VER-$(deriveVersion scala scala-xml "$XML_REF")}
391-
PARSERS_VER=${PARSERS_VER-$(deriveVersion scala scala-parser-combinators "$PARSERS_REF")}
392-
CONTINUATIONS_VER=${CONTINUATIONS_VER-$(deriveVersion scala scala-continuations "$CONTINUATIONS_REF")}
393-
SWING_VER=${SWING_VER-$(deriveVersion scala scala-swing "$SWING_REF")}
394-
ACTORS_MIGRATION_VER=${ACTORS_MIGRATION_VER-$(deriveVersion scala actors-migration "$ACTORS_MIGRATION_REF")}
395-
PARTEST_VER=${PARTEST_VER-$(deriveVersion scala scala-partest "$PARTEST_REF")}
396-
SCALACHECK_VER=${SCALACHECK_VER-$(deriveVersionAnyTag rickynils scalacheck "$SCALACHECK_REF")}
397-
398-
XML_REF=${XML_REF-"HEAD"}
399-
PARSERS_REF=${PARSERS_REF-"HEAD"}
400-
CONTINUATIONS_REF=${CONTINUATIONS_REF-"HEAD"}
401-
SWING_REF=${SWING_REF-"HEAD"}
402-
ACTORS_MIGRATION_REF=${ACTORS_MIGRATION_REF-"HEAD"}
403-
PARTEST_REF=${PARTEST_REF-"HEAD"}
404-
# PARTEST_IFACE_REF=${PARTEST_IFACE_REF-"HEAD"}
405-
SCALACHECK_REF=${SCALACHECK_REF-"HEAD"}
406-
fi
407-
408-
echo "Module versions (versioning strategy: $moduleVersioning):"
409-
echo "ACTORS_MIGRATION = $ACTORS_MIGRATION_VER at $ACTORS_MIGRATION_REF"
410-
echo "CONTINUATIONS = $CONTINUATIONS_VER at $CONTINUATIONS_REF"
411-
echo "PARSERS = $PARSERS_VER at $PARSERS_REF"
412-
echo "PARTEST = $PARTEST_VER at $PARTEST_REF"
413-
echo "SCALACHECK = $SCALACHECK_VER at $SCALACHECK_REF"
414-
echo "SWING = $SWING_VER at $SWING_REF"
415-
echo "XML = $XML_VER at $XML_REF"
416-
417-
# PARTEST_IFACE_VER=${PARTEST_IFACE_VER-$(deriveVersion scala scala-partest-interface "$PARTEST_IFACE_REF")}
418-
}
419-
420-
constructUpdatedModuleVersions() {
421-
updatedModuleVersions=()
422-
423-
# force the new module versions for building the core. these may be different from the values in versions.properties,
424-
# either because the variables (XML_VER) were provided, or because we're building the modules from HEAD.
425-
# in the common case, the values are the same as in versions.properties.
426-
updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dactors-migration.version.number=$ACTORS_MIGRATION_VER")
427-
updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-continuations-library.version.number=$CONTINUATIONS_VER")
428-
updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-continuations-plugin.version.number=$CONTINUATIONS_VER")
429-
updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-parser-combinators.version.number=$PARSERS_VER")
430-
updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-swing.version.number=$SWING_VER")
431-
updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala-xml.version.number=$XML_VER")
432-
433-
updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dpartest.version.number=$PARTEST_VER")
434-
updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscalacheck.version.number=$SCALACHECK_VER")
435-
436-
# allow overriding the akka-actors and jline version using a jenkins build parameter
437-
if [ ! -z "$AKKA_ACTOR_VER" ]; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dakka-actor.version.number=$AKKA_ACTOR_VER"); fi
438-
if [ ! -z "$JLINE_VER" ] ; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Djline.version=$JLINE_VER"); fi
439-
440-
if [ ! -z "$SCALA_BINARY_VER" ]; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala.binary.version=$SCALA_BINARY_VER"); fi
441-
if [ ! -z "$SCALA_FULL_VER" ] ; then updatedModuleVersions=("${updatedModuleVersions[@]}" "-Dscala.full.version=$SCALA_FULL_VER"); fi
442-
}
443-
444-
# build locker (scala + modules) and quick, publishing everything to private-repo
445-
bootstrap() {
446-
echo "### Bootstrapping"
447-
230+
distro() {
448231
cd $WORKSPACE
449232

450-
#### LOCKER
451-
452-
echo "### Building locker"
453-
454-
# for bootstrapping, publish core (or at least smallest subset we can get away with)
455-
# so that we can build modules with this version of Scala and publish them locally
456-
# must publish under $SCALA_VER so that the modules will depend on this (binary) version of Scala
457-
# publish more than just core: partest needs scalap
458-
# in sabbus lingo, the resulting Scala build will be used as starr to build the released Scala compiler
459-
ant -Dmaven.version.number=$SCALA_VER\
460-
-Dremote.snapshot.repository=NOPE\
461-
-Dremote.release.repository=$privateRepo\
462-
-Drepository.credentials.id=$privateCred\
463-
-Dscalac.args.optimise=-optimise\
464-
-Ddocs.skip=1\
465-
-Dlocker.skip=1\
466-
$publishLockerPrivateTask >> $baseDir/logs/builds 2>&1
467-
468-
469-
echo "### Building modules using locker"
233+
echo "### Building the distribution"
470234

471-
# build, test and publish modules with this core
472-
# publish to our internal repo (so we can resolve the modules in the scala build below)
473-
# we only need to build the modules necessary to build Scala itself
474-
# since the version of locker and quick are the same
475-
publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-private-repo")' "set every publishTo := Some(\"private-repo\" at \"$privateRepo\")")
476-
buildTasks=($publishPrivateTask)
477-
buildModules
478-
479-
constructUpdatedModuleVersions
480-
481-
#### QUICK
482-
483-
echo "### Bootstrapping Scala using locker"
484-
485-
# # TODO: close all open staging repos so that we can be reaonably sure the only open one we see after publishing below is ours
486-
# # the ant call will create a new one
487-
#
488-
# Rebuild Scala with these modules so that all binary versions are consistent.
489-
# Update versions.properties to new modules.
490-
# Sanity check: make sure the Scala test suite passes / docs can be generated with these modules.
491-
# don't skip locker (-Dlocker.skip=1), or stability will fail
492-
# overwrite "locker" version of scala at private-repo with bootstrapped version
493-
cd $baseDir
494-
rm -rf build/ # must leave everything else in $baseDir for downstream jobs
495-
496-
ant -Dstarr.version=$SCALA_VER\
497-
-Dextra.repo.url=$privateRepo\
235+
ant -Dextra.repo.url=$privateRepo\
498236
-Dmaven.version.suffix=$SCALA_VER_SUFFIX\
499-
${updatedModuleVersions[@]} \
500-
-Dupdate.versions=1\
501237
-Dscaladoc.git.commit=$SCALADOC_SOURCE_LINKS_VER\
502238
-Dremote.snapshot.repository=NOPE\
503239
-Dremote.release.repository=$privateRepo\
504240
-Drepository.credentials.id=$privateCred\
505-
-Dscalac.args.optimise=-optimise\
506-
$antBuildTask $publishPrivateTask
241+
distpack-opt
242+
243+
(cd $WORKSPACE/dists/ && tar cvzhf ../scala-dist-$SCALA_VER.tar.gz .)
244+
s3Upload scala-dist-$SCALA_VER.tar.gz
245+
}
507246

508-
# clear ivy cache (and to be sure, local as well), so the next round of sbt builds sees the fresh scala
509-
rm -rf $baseDir/ivy2
247+
s3Upload() {
248+
file="$1"
249+
contentType="application/x-compressed-tar"
250+
host=$(grep host ~/.s3credentials | cut -d= -f2)
510251

511-
# TODO: create PR with following commit (note that release will have been tagged already)
512-
# git commit versions.properties -m"Bump versions.properties for $SCALA_VER."
252+
s3curl --id typesafe --contentType "${contentType}" --put "${file}" -- -k https://${host}/scala/tmp/${file}
513253
}
514254

515255
# assumes we just bootstrapped, and current directory is $baseDir
@@ -519,17 +259,8 @@ bootstrap() {
519259
publishSonatype() {
520260
# stage to sonatype, along with all modules -Dmaven.version.suffix/-Dbuild.release not necessary,
521261
# since we're just publishing an existing build
522-
echo "### Publishing core to sonatype"
523-
ant -Dmaven.version.number=$SCALA_VER $publishSonatypeTaskCore
524-
525-
echo "### Publishing modules to sonatype"
526-
# build/test/publish scala core modules to sonatype (this will start a new staging repo)
527-
# (was hoping we could make everything go to the same staging repo, but it's not timing that causes two staging repos to be opened)
528-
# NOTE: only publish those for which versions are set
529-
# test and publish to sonatype, assuming you have ~/.sbt/0.13/sonatype.sbt and ~/.sbt/0.13/plugin/gpg.sbt
530-
publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-sonatype")' "set pgpPassphrase := Some(Array.empty)")
531-
buildTasks=($publishSonatypeTaskModules)
532-
buildModules
262+
echo "### Publishing to sonatype"
263+
(cd $WORKSPACE/dists/maven/latest/ && ant deploy.signed)
533264

534265
open=$(st_stagingReposOpen)
535266
allOpenUrls=$(echo $open | jq '.repositoryURI' | tr -d \")
@@ -547,17 +278,8 @@ publishSonatype() {
547278

548279
determineScalaVersion
549280

550-
deriveModuleVersions
551-
552-
bootstrap
281+
distro
553282

554283
if [ "$publishToSonatype" == "yes" ]
555284
then publishSonatype
556-
else # build modules one more time, just to mimic the regular build as much when running as nightly
557-
echo "### Rebuilding modules with quick, publishing to $baseDir/ivy/local"
558-
buildTasks=(publish-local)
559-
# buildScalacheck always uses publishPrivateTask (not buildTasks). we override it to avoid publishing to private-repo.
560-
publishPrivateTask="publish-local"
561-
forceRebuild="yes"
562-
buildModules
563285
fi

0 commit comments

Comments
 (0)
0