From 1386a7b0bc9e832d45f925202e1382717cf4a706 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 25 Apr 2016 11:59:38 -0500 Subject: [PATCH 001/429] melting-pot.sh: add a usage example to the help Hopefully this helps you, @tpietzsch. --- melting-pot.sh | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index 5998306..0bcdc9f 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -209,7 +209,31 @@ parseArguments() { -s, --skipBuild Skips the final build step. Useful for automated testing. -h, --help - Display this usage information." + Display this usage information. + +--== Example ==-- + + sh melting-pot.sh net.imagej:imagej-common:0.15.1 \\ + -r http://maven.imagej.net/content/groups/public \\ + -c org.scijava:scijava-common:2.44.2 \\ + -i 'org.scijava:*,net.imagej:*,net.imglib2:*,io.scif:*' \\ + -e net.imglib2:imglib2-roi \\ + -v -f -s + +This command tests net.imagej:imagej-common:0.15.1 along with all of its +dependencies, pulled from its usual ImageJ Maven repository location. + +The -c flag is used to override the org.scijava:scijava-common +dependency to use version 2.44.2 instead of its declared version 2.42.0. + +Note that such overrides do not need to be release versions; you can +also test SNAPSHOTs the same way. + +The -i option is used to include all imagej-common dependencies with +groupIds org.scijava, net.imagej, net.imglib2 and io.scif in the pot. + +The -e flag is used to exclude net.imglib2:imglib2-roi from the pot. +" exit 1 fi From 0b360e568269817eacc723b432b0ea572e46bb68 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 20 Oct 2016 10:31:20 -0500 Subject: [PATCH 002/429] Rename scijava-expression-parser to parsington See: https://github.com/scijava/parsington/commit/d67b4c31feab01888da08ac5f22d0c7ca8766973 --- release-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-version.sh b/release-version.sh index 97cc8fd..eb2e21f 100755 --- a/release-version.sh +++ b/release-version.sh @@ -113,10 +113,10 @@ org.scijava:jep:*|\ org.scijava:junit-benchmarks:*|\ org.scijava:minimaven:*|\ org.scijava:native-lib-loader:*|\ +org.scijava:parsington:*|\ org.scijava:pom-scijava:*|\ org.scijava:scijava-common:*|\ org.scijava:scijava-config:*|\ -org.scijava:scijava-expression-parser:*|\ org.scijava:scijava-log-slf4j:*|\ org.scijava:scijava-maven-plugin:*|\ org.scijava:swing-checkbox-tree:*|\ From bc2556189d48088940e4ed20ebaee1adac59ef1b Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 6 Nov 2016 16:10:06 -0600 Subject: [PATCH 003/429] release-version.sh: add pom-scijava-base This is a new parent pom to pom-scijava, as discussed at: http://forum.imagej.net/t/split-boms-from-parent-configuration/2563 --- release-version.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/release-version.sh b/release-version.sh index eb2e21f..23383eb 100755 --- a/release-version.sh +++ b/release-version.sh @@ -114,6 +114,7 @@ org.scijava:junit-benchmarks:*|\ org.scijava:minimaven:*|\ org.scijava:native-lib-loader:*|\ org.scijava:parsington:*|\ +org.scijava:pom-scijava-base:*|\ org.scijava:pom-scijava:*|\ org.scijava:scijava-common:*|\ org.scijava:scijava-config:*|\ From 11890e6268cffed126bdb44b92f9c03fad0e5379 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 14 Dec 2016 18:16:55 +0100 Subject: [PATCH 004/429] melting-pot.sh: add a handy die function --- melting-pot.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 0bcdc9f..d4303c0 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -90,6 +90,11 @@ error() { stderr "[ERROR] $@" } +die() { + error $1 + exit $2 +} + unknownArg() { error "Unknown option: $@" usage=1 @@ -99,7 +104,7 @@ checkPrereqs() { while [ $# -gt 0 ] do which $1 > /dev/null 2> /dev/null - test $? -ne 0 && echo "Missing prerequisite: $1" && exit 255 + test $? -ne 0 && die "Missing prerequisite: $1" 255 shift done } @@ -244,7 +249,7 @@ The -e flag is used to exclude net.imglib2:imglib2-roi from the pot. createDir() { test -z "$force" -a -e "$1" && - error "Directory already exists: $1" && exit 2 + die "Directory already exists: $1" 2 rm -rf "$1" mkdir -p "$1" @@ -514,7 +519,7 @@ meltDown() { # Fetch the project source code. info "$1: fetching project source" local dir="$(retrieveSource "$1" "$branch")" - test ! -d "$dir" && error "Could not fetch project source" && exit 3 + test ! -d "$dir" && die "Could not fetch project source" 3 # Get the project dependencies. info "$1: determining project dependencies" From 99a204a8b1b58b022de7e70ff1f014e73e452962 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 14 Dec 2016 18:17:18 +0100 Subject: [PATCH 005/429] melting-pot.sh: add a bunch more error checking Hopefully this is useful, especially in conjunction with the -d flag, to understand better what is going wrong, when something goes wrong. And to terminate execution more quickly instead of going off the rails. --- melting-pot.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index d4303c0..f7a1f2b 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -319,13 +319,15 @@ downloadPOM() { -DgroupId="$g" \ -DartifactId="$a" \ -Dversion="$v" \ - -Dpackaging=pom > /dev/null + -Dpackaging=pom > /dev/null || + die "Problem fetching $g:$a:$v from $remoteRepos" 4 } # Gets the POM path for the given GAV, ensuring it exists locally. pom() { local pomPath="$(pomPath "$1")" test -f "$pomPath" || downloadPOM "$1" + test -f "$pomPath" || die "Cannot access POM: $pomPath" 9 echo "$pomPath" } @@ -350,6 +352,7 @@ xpath() { # specified XPath expression of the form "//$2/$3/$4/...". pomValue() { local pomPath="$(pom "$1")" + test "$pomPath" || die "Cannot discern POM path for $1" 6 shift local value="$(xpath "$pomPath" $@)" if [ "$value" ] @@ -381,6 +384,7 @@ scmTag() { # Fetches the source code for the given GAV. Returns the directory. retrieveSource() { local scmURL="$(scmURL "$1")" + test "$scmURL" || die "Cannot glean SCM URL for $1" 10 local scmBranch test "$2" && scmBranch="$2" || scmBranch="$(scmTag "$1")" local dir="$(groupId "$1")/$(artifactId "$1")" @@ -393,8 +397,9 @@ retrieveSource() { deps() { cd "$1" debug "mvn dependency:list" - mvn dependency:list | - grep '^\[INFO\] [^ ]' | sed 's/\[INFO\] //' | sort + local depList=$(mvn dependency:list) || + die "Problem fetching dependencies!" 5 + echo "$depList" | grep '^\[INFO\] [^ ]' | sed 's/\[INFO\] //' | sort cd - > /dev/null } @@ -452,6 +457,7 @@ pruneReactor() { do info "Checking relevance of component $dir" local deps="$(deps "$dir")" + test "$deps" || die "Cannot glean dependencies for '$dir'" 8 # Determine whether the component depends on a changed GAV. local keep @@ -524,6 +530,7 @@ meltDown() { # Get the project dependencies. info "$1: determining project dependencies" local deps="$(deps "$dir")" + test "$deps" || die "Cannot glean project dependencies" 7 local args="-Denforcer.skip" From f72564996e571aeae1aba3ab0006fe9dfa819813 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 10 Mar 2017 13:28:28 -0600 Subject: [PATCH 006/429] Purge no-longer-needed verify-code-snippets script This script was used to synchronize code between GitHub and web pages, including MediaWiki and Drupal sites. But we no longer need it. Instead, we now have this MediaWiki extension: https://github.com/imagej/mediawiki-GitHub Unfortunately, it is a fork of the original project, but the changes are quite small: https://github.com/JeroenDeDauw/GitHub/compare/master...imagej:master And it works great for what the ImageJ wiki needs. I know of no other uses of this script in the wilds of the SciJava ecosystem, so I am boldly retiring it. Complain loudly if you need it. --- verify-code-snippets.pl | 145 ---------------------------------------- 1 file changed, 145 deletions(-) delete mode 100755 verify-code-snippets.pl diff --git a/verify-code-snippets.pl b/verify-code-snippets.pl deleted file mode 100755 index 22b2094..0000000 --- a/verify-code-snippets.pl +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/perl - -# -# verify-code-snippets.pl -# - -# Script to check that code snippets on a given web page match the linked code. -# For the script to do anything, the page must declare a link to a source file -# on GitHub, of the form - -# Usage: verify-code-snippets.pl http://domain.name/path/to/page-to-check ... - -use HTML::Entities; - -use strict; - -# constants -my $urlChar = '[\w\-\@?^=%&/~\+#\.]'; -my $codeLink = "https://github.com/$urlChar*/blob/$urlChar*"; -my $lineRange = 10; - -# parse environment variables -my $debug = $ENV{'DEBUG'}; - -# parse command line arguments -my @urls = @ARGV; - -# process all URLs given -my $returnValue = 0; -for my $url (@urls) { - my $result = processURL($url); - if ($result) { - $returnValue = $result; - } -} -exit $returnValue; - -# Processes the given URL for code snippets, checking any it finds. -sub processURL($) { - my ($pageURL) = @_; - - debug("Downloading $pageURL\n"); - my @page = `curl -Ls "$pageURL"`; - - # decode encoded HTML characters (e.g., '&' -> '&') - for (my $pageIndex = 0; $pageIndex < @page; $pageIndex++) { - # HACK: decode ' ' and ' ' to ' ' (not 0xa0 or similar) - $page[$pageIndex] =~ s/ / /g; - $page[$pageIndex] =~ s/ / /g; - - $page[$pageIndex] = decode_entities($page[$pageIndex]); - } - - # scan document for code links - my $failCount = 0; - for (my $pageIndex = 0; $pageIndex < @page; $pageIndex++) { - my $pageLine = $page[$pageIndex]; - if ($pageLine =~ /$codeLink/) { - my $codeURL = makeRaw($&); - debug("Downloading $codeURL"); - my @code = `curl -Ls $codeURL`; - - # HACK: strip off leading copyright header comment - while ($code[0] =~ /^[\/ ]\*/ || $code[0] =~ /^$/) { shift @code; } - - # search for matching first line of code within a few lines - my $codeOffset = findMatch(\@page, $code[0], $pageIndex); - - if ($codeOffset < 0) { - debug("NO CODE SNIPPET FOUND\n"); - next; - } - - my $lineNo = $codeOffset + 1; - debug("Checking code snippet at line $lineNo"); - my $match = valuesEqual(\@page, $codeOffset, \@code); - - if ($match) { - debug("CODE SNIPPET MATCHES\n"); - } - else { - $failCount++; - } - } - } - - # return number of non-matching code snippets - if ($failCount > 0) { - error("Found $failCount non-matching code snippets."); - } - return $failCount; -} - -# Converts a cooked GitHub link to a raw code link -sub makeRaw($) { - my ($url) = @_; - $url =~ s/github\.com/raw.github.com/; - $url =~ s/blob\///; - return $url; -} - -# Searches the given list for a matching string from the specified index. -sub findMatch($$$) { - my ($list, $string, $index) = @_; - for (my $i = $index; $i < $index + $lineRange; $i++) { - if ($$list[$i] eq $string) { - return $i; - } - } - return -1; -} - -# Checks whether the two arrays match (with a1 offset by the given index). -sub valuesEqual($$$) { - my ($a1, $offset, $a2) = @_; - my $index = $offset; - for my $v2 (@$a2) { - my $v1 = sterilize($$a1[$index++]); - if ($v1 ne $v2) { - my $lineNo = $index - $offset + 1; - debug("LINE $lineNo DOES NOT MATCH:\n\t$v1\t$v2"); - return 0; - } - } - return 1; -} - -# Hacky routine to fix discrepancies in HTML code... -sub sterilize($) { - my ($value) = @_; - $value =~ s/<\/pre>$//; - return $value; -} - -# Emits the given message on stderr when the debug flag is set. -sub debug($) { - $debug || return; - error(@_); -} - -# Emits the given message on stderr. -sub error($) { - my ($message) = @_; - print STDERR "$message\n"; -} From 7986287b8c60386e91c62c0f5f307ed9c4ac0920 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 13 Apr 2017 11:20:04 -0500 Subject: [PATCH 007/429] Remove the "no snapshots" git hook The maven-enforcer-plugin now checks this during the build, via the scijava-maven-plugin's requireReproducibleBuilds rule. --- hooks/README.txt | 1 - hooks/pre-commit-no-snapshots.sh | 9 -------- install-git-hooks.sh | 38 -------------------------------- 3 files changed, 48 deletions(-) delete mode 100644 hooks/README.txt delete mode 100755 hooks/pre-commit-no-snapshots.sh delete mode 100755 install-git-hooks.sh diff --git a/hooks/README.txt b/hooks/README.txt deleted file mode 100644 index ae8578b..0000000 --- a/hooks/README.txt +++ /dev/null @@ -1 +0,0 @@ -This folder contains hooks installed by the bin/install-git-hooks.sh script. diff --git a/hooks/pre-commit-no-snapshots.sh b/hooks/pre-commit-no-snapshots.sh deleted file mode 100755 index bab5ceb..0000000 --- a/hooks/pre-commit-no-snapshots.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -# Fail if any new SNAPSHOT references are introduced into pom.xml files. -# In general, SNAPSHOT references result in unstable/nonrepeatable builds. -if git diff --staged -U0 $(git ls-files | grep pom.xml) | grep '^\+.*SNAPSHOT'; -then - echo 'Potential SNAPSHOT references!' - exit 1 -fi diff --git a/install-git-hooks.sh b/install-git-hooks.sh deleted file mode 100755 index 548a761..0000000 --- a/install-git-hooks.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# -# install-git-hooks.sh -# - -# A script to install recommended git hooks into your .git/hooks folder. -# It will append them without overwriting any existing hooks you have. - -HOOK_DIR="$(dirname "$0")/hooks" - -PRE_COMMIT=.git/hooks/pre-commit - -# must be run from toplevel of repository -if [ ! -d .git ]; -then - echo "Must be run from toplevel of a git repository!" - exit 1 -fi - -# create pre-commit hook if it does not already exist -if [ ! -e "$PRE_COMMIT" ]; -then - echo "Creating: $PRE_COMMIT" - echo '#!/bin/sh' >> "$PRE_COMMIT" - echo >> "$PRE_COMMIT" - chmod +x "$PRE_COMMIT" -fi - -# inject bin/hooks calls into relevant git hook scripts -for f in "$HOOK_DIR/pre-commit-"*; -do - if ! grep -q "$f" "$PRE_COMMIT" - then - echo "Installing: $f" - echo "$f" >> "$PRE_COMMIT" - fi -done From 526bb600ef1d0f417ed9ac37ed17cb89edf02184 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 13 Apr 2017 11:21:40 -0500 Subject: [PATCH 008/429] Remove GitHub service installation script We now use Gitter instead of IRC for developer chat. We now use Travis instead of Jenkins for CI. --- add-default-github-hooks.sh | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100755 add-default-github-hooks.sh diff --git a/add-default-github-hooks.sh b/add-default-github-hooks.sh deleted file mode 100755 index 479691f..0000000 --- a/add-default-github-hooks.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh - -# Use this script to add the IRC and Jenkins webhooks to the GitHub repositories. -# You will need to add credentials to your $HOME/.netrc for this to work. And if -# you use two-factor authentication, you'll also need to pass GITHUB_TWO_FACTOR. - -die () { - echo "$*" >&2 - exit 1 -} - -test $# = 1 || -die "Usage: $0 /" - -repository="$1" - -TWO_FACTOR_HEADER="${GITHUB_TWO_FACTOR:+X-GitHub-OTP: }$GITHUB_TWO_FACTOR" -mycurl () { - curl --netrc --header "$TWO_FACTOR_HEADER" "$@" "$url" -} - -url="https://api.github.com/repos/$repository/hooks" -hooks="$(mycurl)" - -# IRC -if ! echo "$hooks" | grep '^ "name": "irc",$' -then - echo "Adding IRC hook" - mycurl -XPOST -d '{"name":"irc","active":true,"events":["push","pull_request"],"config":{"server":"irc.freenode.net","port":"6667","room":"#imagejdev","message_without_join":"1","notice":"1"}}' -fi - -# Jenkins -if ! echo "$hooks" | grep '^ "name": "jenkinsgit",$' -then - echo "Adding Jenkins hook" - mycurl -XPOST -d '{"name":"jenkinsgit","active":true,"events":["push"],"config":{"jenkins_url":"http://jenkins.imagej.net/"}}' -fi From 933ba1ad2312a79f7dee0fbaa7c2e982d20809a3 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 2 May 2017 14:36:12 -0500 Subject: [PATCH 009/429] Remove old release table generator With the switch from Jenkins to Travis CI, this tooling stopped working. As such, it has now been replaced by new code at: https://github.com/scijava/status.scijava.org And the status table moved from status.imagej.net to status.scijava.org. --- check-release.pl | 201 ------------------------------------- prettify-status.pl | 244 --------------------------------------------- show-releases.sh | 21 ---- 3 files changed, 466 deletions(-) delete mode 100755 check-release.pl delete mode 100755 prettify-status.pl delete mode 100755 show-releases.sh diff --git a/check-release.pl b/check-release.pl deleted file mode 100755 index 58f49d0..0000000 --- a/check-release.pl +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/perl - -# check-release.pl - checks whether the tip of the master branch -# has changed since the specified release. - -use strict; -use File::Basename qw(dirname); -use Getopt::Long; - -# -- Constants -- - -my $cachePath = "$ENV{HOME}/.m2/repository"; - -# -- Options -- - -my $release_version; -my $verbose; -my $debug; - -GetOptions( - 'release-version=s' => \$release_version, - 'verbose' => \$verbose, - 'debug' => \$debug) -or die ('Usage: ' . $0 . - ' [--release-version ]' . - ' [--verbose]' . - ' [--debug]'); - -# -- Subroutines -- - -sub debug($) { - my $msg = shift; - if ($debug) { print STDERR "[DEBUG] $msg\n"; } -} - -sub error($$) { - my $no = shift; - my $msg = shift; - print STDERR "[ERROR] $msg\n"; - exit $no; -} - -sub tagExists($) { - my $tag = shift; - return `git tag -l | grep '^$tag\$'` ? 1 : 0; -} - -sub latestRelease($) { - my $ga = shift; - - debug("Downloading latest release metadata"); - - # download the latest release POM from the Maven repository - `mvn dependency:get -Dartifact=$ga:LATEST -Dpackaging=pom`; - - # extract the latest release version from local repository cache metadata - my $gaPath = $ga; - $gaPath =~ s/[\.:]/\//g; - my $release = `grep release "$cachePath/$gaPath"/*.xml | head -n 1`; - chomp $release; - $release =~ s/.*//; - $release =~ s/<\/release>.*//; - return $release; -} - -sub releaseRef($$$) { - my $groupId = shift; - my $artifactId = shift; - my $releaseVersion = shift; - - # look for a suitable tag - my $tag = "$artifactId-$releaseVersion"; - if (tagExists($tag)) { return $tag; } - if ($tag =~ /_$/) { - $tag =~ s/_$//; - if (tagExists($tag)) { return $tag; } - } - if ($tag =~ /^pom-/) { - $tag =~ s/^pom-//; - if (tagExists($tag)) { return $tag; } - } - $tag = "v$releaseVersion"; - if (tagExists($tag)) { return $tag; } - - # no tag found; extract ref from the JAR - return extractRef($groupId, $artifactId, $releaseVersion); -} - -sub extractRef($$$) { - my $groupId = shift; - my $artifactId = shift; - my $version = shift; - - my $ga = "$groupId:$artifactId"; - my $gaPath = $ga; - $gaPath =~ s/[\.:]/\//g; - my $jarPath = "$cachePath/$gaPath/$version/$artifactId-$version.jar"; - - if (! -e $jarPath) { - # download the version from the Maven repository - debug("Fetching JAR: '$jarPath'"); - `mvn dependency:get -Dartifact=$ga:$version`; - } - - if (! -e $jarPath) { - # the requested GAV could not be found - return ''; - } - - # extract Implementation-Build from the JAR manifest - debug("Extracting release ref from JAR"); - my $implBuild = `unzip -q -c "$jarPath" META-INF/MANIFEST.MF | grep Implementation-Build`; - chomp $implBuild; - $implBuild =~ s/.* ([0-9a-f]+).*/\1/; - return $implBuild; -} - -# -- Main -- - -# add SciJava scripts to the search path -$ENV{PATH} .= ':' . dirname($0); - -if (! -e "pom.xml") { - error(1, "No pom.xml: " . `pwd`); -} - -# determine the project's GAV -my $gav = `maven-helper.sh gav-from-pom pom.xml`; -chomp $gav; -my ($groupId, $artifactId, $version) = split(':', $gav); -my $ga = "$groupId:$artifactId"; - -debug("groupId = $groupId"); -debug("artifactId = $artifactId"); - -# make sure the latest tags are available -`git fetch --tags 2> /dev/null`; - -if (!$release_version) { - # no release version specified; use the latest release - $release_version = latestRelease($ga); -} - -debug("Release version = $release_version"); - -if (!$release_version || $release_version =~ /\-SNAPSHOT$/) { - error(2, "$ga: No release version"); -} - -my $releaseRef = releaseRef($groupId, $artifactId, $release_version); - -if (!$releaseRef) { - error(3, "$ga: No ref for version $release_version"); -} - -debug("Release ref = $releaseRef"); - -my $sourceDirs = `git ls-files */src/main/ src/main/ | sed 's-/main/.*-/main-' | uniq | tr '\n' ' '`; - -if (!$sourceDirs) { - error(4, "[ERROR] $ga: No sources to compare"); -} - -debug("Source directories = $sourceDirs"); - -my @commits = `git rev-list $releaseRef...origin/master -- $sourceDirs`; -my $commitCount = @commits; - -# ignore commits which are known to be irrelevant -foreach my $commit (@commits) { - chomp $commit; - my $commitMessage = `git log -1 --oneline $commit`; - chomp $commitMessage; - if ($commitMessage =~ /^[a-z0-9]{7} [Hh]appy .*[Nn]ew [Yy]ear/) { - # Ignore "Happy New Year" copyright header updates - debug("Ignoring Happy New Year commit: $commit"); - $commitCount--; - } - elsif ($commitMessage =~ /^[a-z0-9]{7} [Uu]pdate.*[Cc]opyright.*20[0-9]{2}/) { - # Ignore "Update copyrights" copyright header updates - debug("Ignoring copyright year update commit: $commit"); - $commitCount--; - } - elsif ($commitMessage =~ /^[a-z0-9]{7} [Uu]pdate license (blurb|header)s$/) { - # Ignore "Update license headers" updates - debug("Ignoring 'Update license headers' commit: $commit"); - $commitCount--; - } - elsif ($commitMessage =~ /^[a-z0-9]{7} Organize imports$/) { - # Ignore "Organize imports" updates - debug("Ignoring 'Organize imports' commit: $commit"); - $commitCount--; - } -} - -if ($verbose || $commitCount > 0) { - # new commits on master; a release is potentially needed - my @allCommits = `git rev-list $releaseRef...origin/master`; - my $totalCommits = @allCommits; - print "$ga: $commitCount/$totalCommits commits on master since $release_version\n"; -} diff --git a/prettify-status.pl b/prettify-status.pl deleted file mode 100755 index 52a3aa3..0000000 --- a/prettify-status.pl +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/perl - -# prettify-status.pl - converts plaintext status to HTML. -# -# Usage: prettify-status.pl < status.txt > status.html - -use strict; - -my %orgs = ( - 'org.scijava' => 'scijava', - 'io.scif' => 'scifio', - 'net.imagej' => 'imagej', - 'net.imglib2' => 'imglib', - 'sc.fiji' => 'fiji', -); - -sub rowClass($$) { - my $index = shift; - my $count = shift; - my $rowClass = $index % 2 == 0 ? 'even' : 'odd'; - if ($index == 0) { $rowClass .= ' first'; } - if ($index == $count - 1) { $rowClass .= ' last'; } - return $rowClass; -} - -# parse status output -my @unknown = (); -my @ahead = (); -my @released = (); -my @warnings = (); - -my @lines = <>; - -for my $line (sort @lines) { - chomp $line; - if ($line =~ /([^:]+):([^:]+): (\d+)\/(\d+) commits on (\w+) since (.*)/) { - my $groupId = $1; - my $artifactId = $2; - my $commitCount = $3; - my $totalCommits = $4; - my $branch = $5; - my $version = $6; - my $tag = $version ? "$artifactId-$version" : ""; - my $org = $orgs{$groupId}; - if (!$org) { $org = $groupId; } - my $repo = $artifactId; - $repo =~ s/_$//; - my $link = "https://github.com/$orgs{$groupId}/$repo"; - - my $data = { - groupId => $groupId, - artifactId => $artifactId, - commitCount => $commitCount, - totalCommits => $totalCommits, - branch => $branch, - version => $version, - tag => $tag, - org => $org, - repo => $repo, - }; - - if (not $org) { - my $warning = { %$data }; - $warning->{line} = "No known GitHub org for groupId '$groupId'\n"; - push @warnings, $warning; - } - - if (!$version) { - # release status is unknown - $data->{line} = "\n" . - "$artifactId\n"; - push @unknown, $data; - } - elsif ($commitCount > 0) { - # a release is needed - $data->{line} = "\n" . - "$artifactId\n" . - "$commitCount/$totalCommits\n" . - "$version\n"; - push @ahead, $data; - } - else { - # everything is up to date - my $tagLink = $tag ? "$version" : "-"; - $data->{line} = "\n" . - "$artifactId\n" . - "$tagLink\n"; - push @released, $data; - } - } - else { - my $data = {}; - $data->{line} = $line; - push @warnings, $data; - } -} - -# dump prettified version - -print < - -SciJava software status - - - - - - -HEADER - -print "
\n"; -print "

Projects

\n"; -print "
    \n"; -for my $org (sort values %orgs) { - print "
  • $org
  • \n"; -} -print "
\n"; -print "
\n"; - -if (@warnings > 0) { - print "
\n"; - print "

Warnings

\n"; - print "
    \n"; - my $rowIndex = 0; - my $rowCount = @warnings; - for my $row (@warnings) { - my $org = $row->{org}; - my $line = $row->{line}; - my $rowClass = rowClass($rowIndex++, $rowCount); - print "
  • \n$line\n
  • \n"; - } - print "
\n"; - print "
\n\n"; -} - -if (@released > 0) { - print "
\n"; - print "

Released

\n"; - print "\n"; - print "\n"; - print "\n"; - print "\n"; - print "\n"; - print "\n"; - my $lastGroupId = ''; - my $rowIndex = 0; - my $rowCount = @released; - for my $row (@released) { - my $org = $row->{org}; - my $groupId = $row->{groupId}; - if ($lastGroupId ne $groupId) { - print "\n"; - print "\n"; - print "\n"; - $lastGroupId = $groupId; - } - my $line = $row->{line}; - my $rowClass = rowClass($rowIndex++, $rowCount); - print "\n$line\n"; - } - print "
 ProjectLatest version
" . - "$org
\n"; - print "
\n\n"; -} - -if (@unknown > 0) { - print "
\n"; - print "

Unknown

\n"; - print "\n"; - print "\n"; - print "\n"; - print "\n"; - print "\n"; - my $lastGroupId = ''; - my $rowIndex = 0; - my $rowCount = @unknown; - for my $row (@unknown) { - my $org = $row->{org}; - my $groupId = $row->{groupId}; - if ($lastGroupId ne $groupId) { - print "\n"; - print "\n"; - print "\n"; - $lastGroupId = $groupId; - } - my $line = $row->{line}; - my $rowClass = rowClass($rowIndex++, $rowCount); - print "\n$line\n"; - } - print "
 Project
" . - "$org
\n"; - print "
\n\n"; -} - -if (@ahead > 0) { - print "
\n"; - print "

Ahead

\n"; - print "\n"; - print "\n"; - print "\n"; - print "\n"; - print "\n"; - print "\n"; - print "\n"; - my $lastGroupId = ''; - my $rowIndex = 0; - my $rowCount = @ahead; - for my $row (@ahead) { - my $org = $row->{org}; - my $groupId = $row->{groupId}; - if ($lastGroupId ne $groupId) { - print "\n"; - print "\n"; - print "\n"; - $lastGroupId = $groupId; - } - my $line = $row->{line}; - my $rowClass = rowClass($rowIndex++, $rowCount); - print "\n$line\n"; - } - print "
 ProjectCommitsLatest version
" . - "$org
\n"; - print "
\n\n"; -} - -print "
\n"; -print "

See also

\n"; -print "\n"; -print "
\n\n"; - -print "
 
\n\n"; - -print "\n"; -print "\n"; diff --git a/show-releases.sh b/show-releases.sh deleted file mode 100755 index 2ad18a9..0000000 --- a/show-releases.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh - -die () { - echo "$*" >&2 - exit 1 -} - -check () { - (cd "$1" && check-release.pl) -} - -while [ $# -gt 0 ] -do - if [ -d "$1" ] - then - check "$1" - else - echo "Invalid directory: $1" - fi - shift -done From e694e5eb0d230c973fe3690e9a36e9bf1cbc1fe8 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 5 May 2017 10:34:01 -0500 Subject: [PATCH 010/429] Remove obsolete version update script We do not use it anymore. The Versions Maven plugin can do many of the same things. For example, you can see which versions need updating: mvn -U versions:display-dependency-updates And you can even (in theory) update them automatically: mvn -U versions:use-latest-releases Although I have not had good luck with this latter invocation. See also: http://status.scijava.org/ --- bump-versions-in-pom.sh | 211 ---------------------------------------- 1 file changed, 211 deletions(-) delete mode 100755 bump-versions-in-pom.sh diff --git a/bump-versions-in-pom.sh b/bump-versions-in-pom.sh deleted file mode 100755 index 355a32c..0000000 --- a/bump-versions-in-pom.sh +++ /dev/null @@ -1,211 +0,0 @@ -#!/bin/sh - -die () { - echo "$*" >&2 - exit 1 -} - -skip_commit= -bump_parent= -while test $# -gt 0 -do - case "$1" in - --skip-commit) - skip_commit=t - ;; - --parent|--bump-parent) - bump_parent=t - ;; - --default|--default-properties) - # handle later - break - ;; - -*) - die "Unknown option: $1" - ;; - *) - break - ;; - esac - shift -done - -require_clean_worktree () { - test -z "$skip_commit" || - return - - git rev-parse HEAD@{u} > /dev/null 2>&1 || - die "No upstream configured for the current branch" - - git update-index -q --refresh && - git diff-files --quiet --ignore-submodules && - git diff-index --cached --quiet --ignore-submodules HEAD -- || - die "There are uncommitted changes!" -} - -commit () { - test -n "$skip_commit" || { - git commit -s -m "$@" || - die "Could not commit" - } -} - -maven_helper="$(cd "$(dirname "$0")" && pwd)/maven-helper.sh" && -test -f "$maven_helper" || -die "Could not find maven-helper.sh" - -bump_parent_if_needed () { - gav="$(sh "$maven_helper" parent-gav-from-pom pom.xml)" || - test -z "$gav" || - die "Could not determine parent: $gav" - - test -n "$gav" || { - echo "No parent to bump" >&2 - return 0 - } - - version="${gav#*:*:}" && - test "$version" != "$gav" || - die "Could not determine version: $gav" - - latest="$(sh "$maven_helper" latest-version ${gav%:$version})" && - test -n "$latest" || - die "Could not determine latest ${gav%:$version} version" - - test $version != $latest || { - echo "Parent is already the newest version: $gav" >&2 - return 0 - } - - sed "//,/<\/parent>/s/\(\)$version\(<\/version>\)/\1$latest\2/" \ - pom.xml > pom.xml.new && - mv -f pom.xml.new pom.xml || - die "Could not edit pom.xml" - - return 1 -} - -test -z "$bump_parent" || { - require_clean_worktree - - test -f pom.xml || - die "Not found: pom.xml" - - gav="$(sh "$maven_helper" gav-from-pom pom.xml)" || - die "Could not extract GAV from pom.xml" - - case "$gav" in - *-SNAPSHOT) - ;; - *) - die "Not a -SNAPSHOT version: $gav" - ;; - esac - - bump_parent_if_needed || - commit "Bump parent to $latest" pom.xml - - exit -} - -test "a--default" != "a$*" && -test "a--default-properties" != "a$*" || -set $(sed -ne '/!-- Open Microscopy Environment/q' -e '/^ /,/<\/properties>/s/.*<\([^>\/]*\.version\)>.*/\1 --latest/p' pom.xml) - -test $# -ge 2 && -test 0 = $(($#%2)) || -die "Usage: $0 [--skip-commit] (--parent | --default | ...)" - -pom=pom.xml - -require_clean_worktree - -sed_quote () { - echo "$1" | sed "s/[]\/\"\'\\\\(){}[\!\$ ;]/\\\\&/g" -} - -grep "" pom.xml > /dev/null 2>&1 && -bump_parent_if_needed - -gav="$(sh "$maven_helper" gav-from-pom $pom)" -old_version=${gav##*:} -new_version="${old_version%-SNAPSHOT}" -test "$old_version" != "$new_version" || -new_version=${old_version%.*}.$((1 + ${old_version##*.}))-SNAPSHOT - -message="$(printf "%s\n" "The following changes were made:")" -while test $# -ge 2 -do - must_change=t - latest_message= - property="$1" - value="$2" - if test "a--latest" = "a$value" - then - must_change= - artifactId="${property%.version}" - case "$artifactId" in - imagej1) artifactId=ij;; - nar) artifactId=nar-maven-plugin;; - esac - - test ! -t 0 || - printf '\rLooking at %s...\033[K\r' "$artifactId" - ga="$(sed -n '//{ -N; -s/.*\([^<]*\).*'"$artifactId"'<.*/\1/p -}' pom.xml | head -n 1):$artifactId" - latest_message=" (latest $ga)" - value="$(sh "$maven_helper" latest-version "$ga")" - test -n "$value" || { - echo "Warning: no version found for $ga" >&2 - shift - shift - continue - } - fi - - p="$(sed_quote "$property")" - v="$(sed_quote "$value")" - # Set the primary property version - sed \ - -e "/^ /,/^ <\/properties>/s/\(<$p>\)[^<]*\(<\/$p>\)/\1$v\2/" \ - $pom > $pom.new && - if ! cmp $pom $pom.new - then - message="$(printf '%s\n\t%s = %s%s' \ - "$message" "$property" "$value" "$latest_message")" - elif test -n "$must_change" - then - die "Property $property not found in $pom" - fi && - mv $pom.new $pom || - die "Failed to set property $property = $value" - - shift - shift -done - -! git diff --quiet $pom || { - echo "No properties changed!" >&2 - # help detect when no commit is required by --default - exit 128 -} - -case "$old_version" in -*-SNAPSHOT) ;; -*) - mv $pom $pom.new && - sed \ - -e "s/^\(\\t\)$old_version\(<\/version>\)/\1$new_version\2/" \ - $pom.new > $pom && - ! cmp $pom $pom.new || - die "Failed to increase version of $pom" - - rm $pom.new || - die "Failed to remove intermediate $pom.new" - ;; -esac - -commit "Bump component versions" \ - -m "$message" $pom From 6f1ef6df22246ab7b5ebb7520f47158352844d9a Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 5 May 2017 10:43:43 -0500 Subject: [PATCH 011/429] Remove script for creating Jenkins jobs We are moving away from Jenkins, in general. --- create-jenkins-job.sh | 123 ------------------------------------------ 1 file changed, 123 deletions(-) delete mode 100755 create-jenkins-job.sh diff --git a/create-jenkins-job.sh b/create-jenkins-job.sh deleted file mode 100755 index 18f511c..0000000 --- a/create-jenkins-job.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/sh - -# Use this script to create a Jenkins job - -die () { - echo "$*" >&2 - exit 1 -} - -test $# = 2 || -die "Usage: $0 ( | --deploy-to-imagej=)" - -name="$1" -jenkinsurl="http://jenkins.imagej.net:8080" -createurl="$jenkinsurl/view/Fiji/createItem?name=$name" - -case "$2" in ---deploy-to-imagej=*) - name2="$(echo "$name" | tr '_' ' ')" - url="${2#--deploy-to-imagej=}" - config="$(cat << EOF)" - - - Builds and deploys $name2 every time changes are pushed to the <a href="$url">$name2 Git repository</a>. - - -1 - 15 - -1 - -1 - - false - - - hudson.model.Item.Workspace:anonymous - hudson.model.Item.Workspace:authenticated - hudson.model.Run.Update:authenticated - hudson.model.Item.Configure:authenticated - hudson.scm.SCM.Tag:authenticated - hudson.model.Item.Delete:authenticated - hudson.model.Item.Build:authenticated - hudson.model.Item.Read:anonymous - hudson.model.Item.Read:authenticated - hudson.model.Run.Delete:authenticated - - - - - 2 - - - origin - +refs/heads/*:refs/remotes/origin/* - $url - - - - - master - - - false - - - - - - true - false - false - false - (Default) - - - @yearly - false - - - false - - - git clean -dfx - - - mvn -DupdateReleaseInfo=true -Pdeploy-to-imagej deploy - - - - - **/target/*.jar - false - false - - - imagej-builds@imagej.net - false - true - - - - - FAILURE_AND_FIXED - false - false - false - false - false - - ONLY_CONFIGURATIONS - - - - - -EOF - ;; -*) - config="$(cat "$2")" || - die "Could not read $2" - ;; -esac - -#crumb="$(curl "$jenkinsurl/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,\":\",//crumb)")" -curl --netrc -XPOST -H Content-Type:text/xml -d "$config" "$createurl" From 1553c15014d781bda679df3f653cd23b1509abda Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 5 May 2017 10:45:25 -0500 Subject: [PATCH 012/429] Remove ImageJ-Jenkins-specific release scripts The Groovy script was used by the ImageJ Jenkins's internal Release-Version job. But a recent Jenkins security upgrade now prevents such scripts from being run without jumping through additional hoops. For this reason and others, we are moving away from using Jenkins for releases, in favor of only command-line-oriented tooling. --- git-info-for-jenkins-job.groovy | 39 ---------------- let-jenkins-release.sh | 81 --------------------------------- 2 files changed, 120 deletions(-) delete mode 100755 git-info-for-jenkins-job.groovy delete mode 100755 let-jenkins-release.sh diff --git a/git-info-for-jenkins-job.groovy b/git-info-for-jenkins-job.groovy deleted file mode 100755 index 0b07cca..0000000 --- a/git-info-for-jenkins-job.groovy +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/jenkins-cli - -/** - * Prints author information for use by Git about the user who started a given - * Jenkins build. Example usage in an 'Execute shell' build step: - * - * eval "$(jenkins-cli groovy git-info-for-job.groovy $JOB_NAME $BUILD_NUMBER)" - */ - -if (this.args.length != 2) { - throw new IllegalArgumentException("Usage: author-ident ") -} - -projectName = this.args[0] -buildNumber = Integer.parseInt(this.args[1]) - -map = jenkins.model.Jenkins.instance.getItemMap() -project = map.get(projectName) -build = project.getBuildByNumber(buildNumber) - -emailMap = [ - "Curtis Rueden": "ctrueden@wisc.edu", - "Johannes Schindelin": "johannes.schindelin@gmx.de", - "Mark Hiner": "hinerm@gmail.com", - "Tobias Pietzsch" : "tobias.pietzsch@gmail.com" -] - -for (cause in build.getCauses()) try { - name = cause.getUserName() - println('GIT_AUTHOR_NAME="' + name + '"') - println('export GIT_AUTHOR_NAME') - email = emailMap[name] - if (email == null) { - email = name.toLowerCase().replaceAll(' ', '.') + '@jenkins.imagej.net' - } - println('GIT_AUTHOR_EMAIL="' + email + '"') - println('export GIT_AUTHOR_EMAIL') - break -} catch (e) { /* ignore */ } diff --git a/let-jenkins-release.sh b/let-jenkins-release.sh deleted file mode 100755 index fca30e9..0000000 --- a/let-jenkins-release.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/sh - -die () { - echo "$*" >&2 - exit 1 -} - -simulate= -while test $# -gt 0 -do - case "$1" in - --simulate) - simulate=t - ;; - *) - die "Unhandled option: $1" - ;; - esac - shift -done - -url="$(git config remote.origin.url)" || -die "Could not obtain the current URL" - -case "$url" in -http://github.com/*|https://github.com/*|git://github.com/*|ssh://github.com/*|ssh://git@github.com/*) - repo=${url#*://*github.com/} - ;; -git@github.com:*|github.com:*|github:*) - repo=${url#*:} - ;; -*) - die "Not a GitHub URL: $url" - ;; -esac -repo=${repo%/} -repo=${repo%.git} -repository_url=$repo - -test refs/heads/master = "$(git rev-parse --symbolic-full-name HEAD)" || -die "Not on master branch!" - -git update-index -q --refresh && -git diff-files --quiet --ignore-submodules && -git diff-index --cached --quiet --ignore-submodules HEAD -- || -die "There are uncommitted changes!" - -git fetch --tags || -die "Could not fetch" - -commit="$(git rev-parse HEAD)" && -upstream="$(git rev-parse origin/master)" || -die "Could not obtain current revision" - -test $commit = $upstream || -die "Not up-to-date: $(printf '\n%s' "$(git log --graph --oneline \ - --left-right --boundary ...$upstream)")" - -pom="$(cat pom.xml)" && -snapshot="$(echo "$pom" | - sed -n 's|^ \(.*\)|\1|p')" || -die "Could not obtain version" - -version=${snapshot%-SNAPSHOT} -test "$version" != "$snapshot" || -die "Not a -SNAPSHOT version: $snapshot" - -test ! -t 0 || { - printf 'Version? [%s]: ' "$version" - read line - test -z "$line" || version="$line" -} - -jenkins_url=http://jenkins.imagej.net/job/Release-Version/buildWithParameters -params="REPOSITORY_URL=$repository_url&COMMIT=$commit&VERSION_STRING=$version" -if test -z "$simulate" -then - curl --netrc -X POST "$jenkins_url?$params" -else - echo curl --netrc -X POST "$jenkins_url?$params" -fi From 801dd4ef59e43cd5ce1da6f3301b07c4317efa4c Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 5 May 2017 10:51:09 -0500 Subject: [PATCH 013/429] Remove MediaWiki mirror script Nothing is using it anymore. We do not mirror the ImageJ wiki right now. --- mediawiki-mirror.sh | 139 -------------------------------------------- 1 file changed, 139 deletions(-) delete mode 100755 mediawiki-mirror.sh diff --git a/mediawiki-mirror.sh b/mediawiki-mirror.sh deleted file mode 100755 index df9ab2d..0000000 --- a/mediawiki-mirror.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/sh - -if test $# != 2 -then - cat >&2 << EOF -Usage: $0 - -This script tries to keep a mirror of a MediaWiki up-to-date. It does so by -inspecting the recent changes and calling a slightly modified MediaWiki API -to allow for exporting more than one revision of a page. - -It is incomplete in that it does not mirror the logs, nor accounts. It is also -prone to miss edits that happened while the script ran previously. - -Make sure that the script is run in the directory containing the mirror's -LocalSettings.php. -EOF - exit 1 -fi - -normalize_mediawiki_url () { - case "$1" in - */.php*) - echo "${1%.php*}" - ;; - */) - echo "$1" - ;; - *) - echo "${1*/*}"/ - ;; - esac -} - -ORIGINAL="$(normalize_mediawiki_url "$1")"/api.php?action= -MIRROR="$(normalize_mediawiki_url "$2")"/api.php?action= - -latest_update () { - curl -s "$1query&list=recentchanges&rclimit=1&format=yaml" | - sed -n -e '/^ timestamp: |-$/{N;s/.*\n *//p}' -} - -pages_with_changes_since () { - curl -s "$1query&list=recentchanges&rcend=$2&rclimit=500&format=dbg" | - sed -n "/^ 'type' => 'edit'/{N;N;s/.*'title' => //p}" | - sort | - uniq -} - -urlencode () { - printf "%s" "$*" | - xxd -plain | - tr -d '\n' | - sed 's/\(..\)/%\1/g' -} - -get_dump () { - curl -s "$1query&export=1&exportnowrap=1&format=xml&offset=$2&titles=$3" -} - -# make sure we are in the correct directory - -test -f maintenance/importDump.php && -test -f maintenance/dumpUploads.php && -test -f LocalSettings.php || { - echo "Not in a MediaWiki directory!" >&2 - exit 1 -} - -# when was our local latest update - -since="$(latest_update $MIRROR)" - -# get updates for these pages - -pages="$(pages_with_changes_since $ORIGINAL $since | - sed -e "s/^'//" -e "s/',$//" -e "s/ /%20/g" | - tr '\n' ' ')" - -test -n "$pages" || exit 0 - -# export from the original Wiki - -for page in $pages -do - echo "Importing $page incrementally..." - get_dump $ORIGINAL $since $page | - php maintenance/importDump.php -done - -# copy images - -IMAGEURL=${ORIGINAL%index.php?action=} - -php maintenance/dumpUploads.php | -while read path -do - # ignore if there were no changes - case " $pages " in - *" File:${path##*/} "*) - ;; - *) - continue - ;; - esac - - # ignore if we have an up-to-date copy - if test -f "$path" - then - local_mtime="$(stat -c %Y "$path")" - remote_mtime="$(date -d "$(curl -s --head "$IMAGEURL$path" | - sed -n 's/^Last-Modified: //p')" +%s)" - if test -z "$remote_mtime" || - test $local_mtime -ge "$remote_mtime" - then - continue - fi - fi - - # make fan-out directories if necessary - case "$path" in - */*) - directory="${path%/*}" - mkdir -p "$directory" - ;; - *) - directory=. - ;; - esac - - echo "Downloading $path..." - (cd "$directory" && - curl -s -O "$IMAGEURL$path") -done - -# update recent changes - -php maintenance/rebuildrecentchanges.php - From e2cf84196b4bb2cc8a124c196634f704bc2f4f9f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 5 May 2017 12:45:31 -0500 Subject: [PATCH 014/429] release-version: change how GPG credentials work Rather than requiring them to be set as environment variables (GPG_KEYNAME and GPG_PASSPHRASE), we now instead check for them as system properties (gpg.keyname and gpg.passphrase) in ~/.m2/settings.xml. And we provide a hint to the documentation if it needs configuring. --- release-version.sh | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/release-version.sh b/release-version.sh index 23383eb..c47e201 100755 --- a/release-version.sh +++ b/release-version.sh @@ -12,6 +12,33 @@ maven_helper () { die "Could not find maven-helper in '$MAVEN_HELPER'" } +verify_gpg_settings () { + gpg=$(xmllint --xpath '//settings/profiles/profile' "$HOME/.m2/settings.xml") + test "$gpg" && id=$(echo "$gpg" | xmllint --xpath '//profile/id/text()' -) + test "$gpg" && keyname=$(echo "$gpg" | xmllint --xpath '//profile/properties/gpg.keyname' -) + test "$gpg" && passphrase=$(echo "$gpg" | xmllint --xpath '//profile/properties/gpg.passphrase' -) + test "$keyname" -a "$passphrase" || + die 'GPG configuration not found in settings.xml. Please add it: + + + + gpg + + + ${env.HOME}/.gnupg + + + + (your GPG email address) + (your GPG passphrase) + + + + + +See also: https://github.com/scijava/pom-scijava/wiki/GPG-Signing' +} + IMAGEJ_BASE_REPOSITORY=-DaltDeploymentRepository=imagej.releases::default::dav:http://maven.imagej.net/content/repositories IMAGEJ_RELEASES_REPOSITORY=$IMAGEJ_BASE_REPOSITORY/releases IMAGEJ_THIRDPARTY_REPOSITORY=$IMAGEJ_BASE_REPOSITORY/thirdparty @@ -95,8 +122,7 @@ org.scijava:pom-jython-shaded:*) ARTIFACT_ID=${BASE_GAV#*:pom-} ARTIFACT_ID=${ARTIFACT_ID%:*} test -n "$TAG" || TAG=-Dtag=$ARTIFACT_ID-$VERSION - test -n "$GPG_KEYNAME" || die "Need to set GPG_KEYNAME" - test -n "$GPG_PASSPHRASE" || die "Need to set GPG_PASSPHRASE" + verify_gpg_settings PROFILE=-Psonatype-oss-release INVALIDATE_NEXUS=t ;; @@ -122,8 +148,7 @@ org.scijava:scijava-log-slf4j:*|\ org.scijava:scijava-maven-plugin:*|\ org.scijava:swing-checkbox-tree:*|\ org.scijava:vecmath:*) - test -n "$GPG_KEYNAME" || die "Need to set GPG_KEYNAME" - test -n "$GPG_PASSPHRASE" || die "Need to set GPG_PASSPHRASE" + verify_gpg_settings PROFILE=-Psonatype-oss-release INVALIDATE_NEXUS=t ;; @@ -183,13 +208,9 @@ if test -z "$SKIP_DEPLOY" then $DRY_RUN git checkout $tag && $DRY_RUN mvn $PROFILE \ - -Dgpg.keyname="$GPG_KEYNAME" \ - -Dgpg.passphrase="$GPG_PASSPHRASE" \ -DperformRelease \ clean verify && $DRY_RUN mvn $PROFILE \ - -Dgpg.keyname="$GPG_KEYNAME" \ - -Dgpg.passphrase="$GPG_PASSPHRASE" \ $ALT_REPOSITORY \ -DperformRelease -DupdateReleaseInfo=true \ deploy && From 1aaab1adf738b009e914997f7fa9dd8717e327d2 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 5 May 2017 13:32:03 -0500 Subject: [PATCH 015/429] release-version: check if version is SemVer-valid This logic will need some TLC later, since it currently does not recognize things like "1.2.3-SNAPSHOT -> 2.0.0-beta-1". --- release-version.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/release-version.sh b/release-version.sh index c47e201..8cb2838 100755 --- a/release-version.sh +++ b/release-version.sh @@ -12,6 +12,14 @@ maven_helper () { die "Could not find maven-helper in '$MAVEN_HELPER'" } +VALID_SEMVER_BUMP="$(cd "$(dirname "$0")" && pwd)/valid-semver-bump.sh" + +valid_semver_bump () { + test -f "$VALID_SEMVER_BUMP" || + die "Could not find valid-semver-bump in '$VALID_SEMVER_BUMP'" + sh -$- "$VALID_SEMVER_BUMP" "$@" || die +} + verify_gpg_settings () { gpg=$(xmllint --xpath '//settings/profiles/profile' "$HOME/.m2/settings.xml") test "$gpg" && id=$(echo "$gpg" | xmllint --xpath '//profile/id/text()' -) @@ -86,9 +94,9 @@ do shift done +pomVersion="$(sed -n 's/^ \(.*\)-SNAPSHOT<\/version>$/\1/p' pom.xml)" test $# = 1 || test ! -t 0 || { - version="$(sed -n 's/^ \(.*\)-SNAPSHOT<\/version>$/\1/p' \ - pom.xml)" + version=$pomVersion printf 'Version? [%s]: ' "$version" read line test -z "$line" || version="$line" @@ -108,6 +116,7 @@ case "$VERSION" in *) die "Version '$VERSION' does not start with a digit!" esac +valid_semver_bump "$pomVersion" "$VERSION" # defaults From c232732370ca19edb5e62fce07fa4171c1bb20f4 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 5 May 2017 13:32:47 -0500 Subject: [PATCH 016/429] release-version: check for .netrc credentials If they are not present, then the script will not be able to invalidate the remote cache after deploying the release. Currently, we use an account called "jenkins-expire-cache" to do this, which is now a misnomer, and also pretty hacky. It would be better to simply use the current user's credentials for doing it, and ensure that that user has the relevant permission on the server side. But I don't have time to research how to do that at the moment. --- release-version.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/release-version.sh b/release-version.sh index 8cb2838..51c4e8d 100755 --- a/release-version.sh +++ b/release-version.sh @@ -47,6 +47,17 @@ verify_gpg_settings () { See also: https://github.com/scijava/pom-scijava/wiki/GPG-Signing' } +verify_netrc_settings () { + grep -q 'machine maven.imagej.net' "$HOME/.netrc" 2>/dev/null && + grep -q 'login jenkins-expire-cache' "$HOME/.netrc" 2>/dev/null || + die 'maven.imagej.net cache expiration credentials not found in .netrc. Please add it: +machine maven.imagej.net +login jenkins-expire-cache +password (the-correct-password) + +See also: https://github.com/scijava/pom-scijava/wiki/Adding-Maven-Users' +} + IMAGEJ_BASE_REPOSITORY=-DaltDeploymentRepository=imagej.releases::default::dav:http://maven.imagej.net/content/repositories IMAGEJ_RELEASES_REPOSITORY=$IMAGEJ_BASE_REPOSITORY/releases IMAGEJ_THIRDPARTY_REPOSITORY=$IMAGEJ_BASE_REPOSITORY/thirdparty @@ -94,6 +105,8 @@ do shift done +verify_netrc_settings + pomVersion="$(sed -n 's/^ \(.*\)-SNAPSHOT<\/version>$/\1/p' pom.xml)" test $# = 1 || test ! -t 0 || { version=$pomVersion From 2813a1c42ccf5dad162574556f5d7552dce245c9 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 5 May 2017 13:34:07 -0500 Subject: [PATCH 017/429] release-version: use https for repository protocol --- release-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-version.sh b/release-version.sh index 51c4e8d..721b600 100755 --- a/release-version.sh +++ b/release-version.sh @@ -58,7 +58,7 @@ password (the-correct-password) See also: https://github.com/scijava/pom-scijava/wiki/Adding-Maven-Users' } -IMAGEJ_BASE_REPOSITORY=-DaltDeploymentRepository=imagej.releases::default::dav:http://maven.imagej.net/content/repositories +IMAGEJ_BASE_REPOSITORY=-DaltDeploymentRepository=imagej.releases::default::dav:https://maven.imagej.net/content/repositories IMAGEJ_RELEASES_REPOSITORY=$IMAGEJ_BASE_REPOSITORY/releases IMAGEJ_THIRDPARTY_REPOSITORY=$IMAGEJ_BASE_REPOSITORY/thirdparty From 619973fdbc79824ac6473f4c7652311b58b9cc50 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 15 May 2017 14:08:21 -0500 Subject: [PATCH 018/429] melting-pot: allow testing local projects Without this, you had to push a project to remote branch before you could test it locally, which seems unnecessarily convoluted. --- melting-pot.sh | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index f7a1f2b..d16273d 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -184,6 +184,7 @@ parseArguments() { The project to build, including dependencies, with consistent versions. + Can be either G:A:V form, or a local directory pointing at a project. -b, --branch Override the branch/tag of the project to build. By default, the branch used will be the tag named \"artifactId-version\". @@ -242,6 +243,9 @@ The -e flag is used to exclude net.imglib2:imglib2-roi from the pot. exit 1 fi + # If project is a local directory path, get its absolute path. + test -d "$project" && project=$(cd "$project" && pwd) + # Assign default parameter values. test "$outputDir" || outputDir="melting-pot" test "$repoBase" || repoBase="$HOME/.m2/repository" @@ -523,9 +527,21 @@ generatePOM() { # specified version for the corresponding GA. meltDown() { # Fetch the project source code. - info "$1: fetching project source" - local dir="$(retrieveSource "$1" "$branch")" - test ! -d "$dir" && die "Could not fetch project source" 3 + if [ -d "$1" ] + then + # Use local directory for the specified project. + test -d "$1" || die "No such directory: $1" 11 + test -f "$1/pom.xml" || die "Not a Maven project: $1" 12 + info "$1: local Maven project" + mkdir -p "LOCAL" + local dir="LOCAL/PROJECT" + ln -s "$1" "$dir" + else + # Treat specified project as a GAV. + info "$1: fetching project source" + local dir="$(retrieveSource "$1" "$branch")" + test -d "$dir" || die "Could not fetch project source" 3 + fi # Get the project dependencies. info "$1: determining project dependencies" From 323422ab404941f185fc0a467e5a72f0238e4819 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 15 May 2017 15:48:29 -0500 Subject: [PATCH 019/429] release-version: fail fast if push URL is git:/ I have been bitten by this too many times. Let's check for it proactively, to avoid any further embarrassment. --- release-version.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/release-version.sh b/release-version.sh index 721b600..b36f11b 100755 --- a/release-version.sh +++ b/release-version.sh @@ -47,6 +47,15 @@ verify_gpg_settings () { See also: https://github.com/scijava/pom-scijava/wiki/GPG-Signing' } +verify_git_settings () { + if [ ! "$SKIP_PUSH" ] + then + local push=$(git remote -v | grep origin | grep '(push)') + test "$push" || die 'No push URL found for remote origin' + echo "$push" | grep -q 'git:/' && die 'Remote origin is read-only' + fi +} + verify_netrc_settings () { grep -q 'machine maven.imagej.net' "$HOME/.netrc" 2>/dev/null && grep -q 'login jenkins-expire-cache' "$HOME/.netrc" 2>/dev/null || @@ -105,6 +114,7 @@ do shift done +verify_git_settings verify_netrc_settings pomVersion="$(sed -n 's/^ \(.*\)-SNAPSHOT<\/version>$/\1/p' pom.xml)" From 0101d5d1024fe167544e9f12a21fde88b3472d04 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 24 May 2017 13:33:00 -0500 Subject: [PATCH 020/429] release-version.sh: fix apparent syntax error The error I saw was: release-version.sh: 53: local: git://github.com/scijava/scijava-grab: bad variable name --- release-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-version.sh b/release-version.sh index b36f11b..6520133 100755 --- a/release-version.sh +++ b/release-version.sh @@ -50,7 +50,7 @@ See also: https://github.com/scijava/pom-scijava/wiki/GPG-Signing' verify_git_settings () { if [ ! "$SKIP_PUSH" ] then - local push=$(git remote -v | grep origin | grep '(push)') + push=$(git remote -v | grep origin | grep '(push)') test "$push" || die 'No push URL found for remote origin' echo "$push" | grep -q 'git:/' && die 'Remote origin is read-only' fi From f84d70296deb78d96cbce597a579a0d1f1b20640 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 18 Aug 2017 16:30:09 -0500 Subject: [PATCH 021/429] Add a javadoc update script for use by Travis CI This way, we don't have to replicate this logic, and then later keep it in sync, across the various SciJava javadoc repositories. --- travis-javadoc.sh | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 travis-javadoc.sh diff --git a/travis-javadoc.sh b/travis-javadoc.sh new file mode 100644 index 0000000..f27ce24 --- /dev/null +++ b/travis-javadoc.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +# +# travis-javadoc.sh - A script to build the javadocs of a SciJava-based project. +# + +if [ "$TRAVIS_SECURE_ENV_VARS" = true \ + -a "$TRAVIS_PULL_REQUEST" = false \ + -a "$TRAVIS_BRANCH" = master ] +then + project=$1 + + # Build the javadocs. + mvn -Pbuild-javadocs && + test -d target/apidocs && + + # Configure SSH. SSH_PRIVATE_KEY must contain the private + # RSA key for communicating with the git remote. + mkdir -p "$HOME/.ssh" && + echo "$SSH_PRIVATE_KEY" > "$HOME/.ssh/id_rsa" && + echo 'github.com,192.30.252.130 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' > "$HOME/.ssh/known_hosts" && + + # Configure git settings. + git config --global user.email "travis@travis-ci.org" + git config --global user.name "Travis CI" + + # Clone the javadoc.scijava.org repository. + git clone --quiet --depth 1 git@github.com:scijava/javadoc.scijava.org > /dev/null && + + # Update the relevant javadocs. + cd javadoc.scijava.org && + rm -rf "$project" && + mv ../target/apidocs "$project" && + + # Commit and push the changes. + git add "$project" && + git commit -m "Update $project javadocs (Travis build $TRAVIS_BUILD_NUMBER)" && + git push -q origin gh-pages > /dev/null + + echo "Update complete." +else + echo "Skipping non-canonical branch $TRAVIS_BRANCH" +fi From c05360299c556c8e0c4897bdb5f50d4907c20e32 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 18 Aug 2017 16:30:09 -0500 Subject: [PATCH 022/429] travis-javadoc.sh: use encrypted file, not env var The private SSH key was too long for Travis's environment variable support, yielding the error: data too large - consider using travis encrypt-file or travis env set --- travis-javadoc.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/travis-javadoc.sh b/travis-javadoc.sh index f27ce24..60b445d 100644 --- a/travis-javadoc.sh +++ b/travis-javadoc.sh @@ -14,10 +14,10 @@ then mvn -Pbuild-javadocs && test -d target/apidocs && - # Configure SSH. SSH_PRIVATE_KEY must contain the private - # RSA key for communicating with the git remote. + # Configure SSH. The file .travis/javadoc.scijava.org.enc must contain + # an encrypted private RSA key for communicating with the git remote. mkdir -p "$HOME/.ssh" && - echo "$SSH_PRIVATE_KEY" > "$HOME/.ssh/id_rsa" && + openssl aes-256-cbc -K "$encrypted_cb3727795fd5_key" -iv "$encrypted_cb3727795fd5_iv" -in '.travis/javadoc.scijava.org.enc' -out "$HOME/.ssh/id_rsa" -d && echo 'github.com,192.30.252.130 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' > "$HOME/.ssh/known_hosts" && # Configure git settings. From 7c0f2e45b404965f2ab734355014b0644ef11e3b Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 24 Aug 2017 11:43:08 -0500 Subject: [PATCH 023/429] travis-javadoc.sh: fix typo in javadoc profile --- travis-javadoc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis-javadoc.sh b/travis-javadoc.sh index 60b445d..2e4d208 100644 --- a/travis-javadoc.sh +++ b/travis-javadoc.sh @@ -11,7 +11,7 @@ then project=$1 # Build the javadocs. - mvn -Pbuild-javadocs && + mvn -Pbuild-javadoc && test -d target/apidocs && # Configure SSH. The file .travis/javadoc.scijava.org.enc must contain From d522505eafa40053f425b95b0c42e26367eabfb6 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 24 Aug 2017 11:43:28 -0500 Subject: [PATCH 024/429] travis-javadoc.sh: add missing ampersands We only want to keep going if these commands succeed. --- travis-javadoc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travis-javadoc.sh b/travis-javadoc.sh index 2e4d208..5f49449 100644 --- a/travis-javadoc.sh +++ b/travis-javadoc.sh @@ -21,8 +21,8 @@ then echo 'github.com,192.30.252.130 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' > "$HOME/.ssh/known_hosts" && # Configure git settings. - git config --global user.email "travis@travis-ci.org" - git config --global user.name "Travis CI" + git config --global user.email "travis@travis-ci.org" && + git config --global user.name "Travis CI" && # Clone the javadoc.scijava.org repository. git clone --quiet --depth 1 git@github.com:scijava/javadoc.scijava.org > /dev/null && From b5a44d431022e57912a897431d9022809a9c891f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 24 Aug 2017 11:43:46 -0500 Subject: [PATCH 025/429] travis-javadoc.sh: add more github.com host keys This reduces the chance we will see the message: Warning: Permanently added the RSA host key for IP address '192.30.253.113' to the list of known hosts. I.e.: it reduces the risk of a MITM attack between Travis and GitHub. --- travis-javadoc.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/travis-javadoc.sh b/travis-javadoc.sh index 5f49449..5967e0a 100644 --- a/travis-javadoc.sh +++ b/travis-javadoc.sh @@ -18,7 +18,17 @@ then # an encrypted private RSA key for communicating with the git remote. mkdir -p "$HOME/.ssh" && openssl aes-256-cbc -K "$encrypted_cb3727795fd5_key" -iv "$encrypted_cb3727795fd5_iv" -in '.travis/javadoc.scijava.org.enc' -out "$HOME/.ssh/id_rsa" -d && - echo 'github.com,192.30.252.130 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' > "$HOME/.ssh/known_hosts" && + echo 'github.com,192.30.253.113 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> "$HOME/.ssh/known_hosts" && + echo '192.30.253.112 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> "$HOME/.ssh/known_hosts" && + echo '192.30.252.120 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> "$HOME/.ssh/known_hosts" && + echo '192.30.252.121 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> "$HOME/.ssh/known_hosts" && + echo '192.30.252.122 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> "$HOME/.ssh/known_hosts" && + echo '192.30.252.123 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> "$HOME/.ssh/known_hosts" && + echo '192.30.252.128 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> "$HOME/.ssh/known_hosts" && + echo '192.30.252.129 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> "$HOME/.ssh/known_hosts" && + echo '192.30.252.130 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> "$HOME/.ssh/known_hosts" && + echo '192.30.252.131 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> "$HOME/.ssh/known_hosts" && + echo '192.30.252.142 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> "$HOME/.ssh/known_hosts" && # Configure git settings. git config --global user.email "travis@travis-ci.org" && From 5ee884093d2f755db9f21ab8abe672520b4bf558 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 24 Aug 2017 11:51:53 -0500 Subject: [PATCH 026/429] travis-javadoc.sh: fix perms of private RSA key The SSH client requires strict permissions; otherwise, the following error is issued: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @ WARNING: UNPROTECTED PRIVATE KEY FILE! @ @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ Permissions 0664 for '/home/travis/.ssh/id_rsa' are too open. It is required that your private key files are NOT accessible by others. This private key will be ignored. bad permissions: ignore key: /home/travis/.ssh/id_rsa --- travis-javadoc.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/travis-javadoc.sh b/travis-javadoc.sh index 5967e0a..b0ccf9c 100644 --- a/travis-javadoc.sh +++ b/travis-javadoc.sh @@ -18,6 +18,7 @@ then # an encrypted private RSA key for communicating with the git remote. mkdir -p "$HOME/.ssh" && openssl aes-256-cbc -K "$encrypted_cb3727795fd5_key" -iv "$encrypted_cb3727795fd5_iv" -in '.travis/javadoc.scijava.org.enc' -out "$HOME/.ssh/id_rsa" -d && + chmod 400 "$HOME/.ssh/id_rsa" && echo 'github.com,192.30.253.113 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> "$HOME/.ssh/known_hosts" && echo '192.30.253.112 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> "$HOME/.ssh/known_hosts" && echo '192.30.252.120 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> "$HOME/.ssh/known_hosts" && From 2072587cfd5e64f1a4fcc98a95959070d9334760 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 24 Aug 2017 13:57:57 -0500 Subject: [PATCH 027/429] travis-javadoc.sh: return non-zero upon failure We want the Travis build to fail if something explodes. --- travis-javadoc.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/travis-javadoc.sh b/travis-javadoc.sh index b0ccf9c..68da30e 100644 --- a/travis-javadoc.sh +++ b/travis-javadoc.sh @@ -45,8 +45,12 @@ then # Commit and push the changes. git add "$project" && - git commit -m "Update $project javadocs (Travis build $TRAVIS_BUILD_NUMBER)" && - git push -q origin gh-pages > /dev/null + success=1 + + test "$success" || exit 1 + + git commit -m "Update $project javadocs (Travis build $TRAVIS_BUILD_NUMBER)" + git push -q origin gh-pages > /dev/null || exit 2 echo "Update complete." else From 444b2e889e6749e0ce42823925a7b146653d31f5 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 1 Sep 2017 11:36:28 -0500 Subject: [PATCH 028/429] travis-javadoc.sh: strip out date stamps Otherwise, the javadocs always differ trivially. E.g.: - + All Classes (ImageJ Javadocs 2.0.0-SNAPSHOT API) - + --- travis-javadoc.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/travis-javadoc.sh b/travis-javadoc.sh index 68da30e..f1a8e1d 100644 --- a/travis-javadoc.sh +++ b/travis-javadoc.sh @@ -13,6 +13,8 @@ then # Build the javadocs. mvn -Pbuild-javadoc && test -d target/apidocs && + # Strip out date stamps, to avoid spurious changes being committed. + sed -i'' -e '/\(\n\t\tdeploy-to-imagej<\/releaseProfiles>\1/igs' "$tmpFile" + update pom.xml 'POM: deploy releases to the ImageJ repository' + fi + + # update the README + if ! grep -q "travis-ci.org/$repoSlug" README.md >/dev/null 2>&1 + then + info "Adding Travis badge to README.md" + echo "[![](https://travis-ci.org/$repoSlug.svg?branch=master)](https://travis-ci.org/$repoSlug)" >"$tmpFile" + echo >>"$tmpFile" + test -f README.md && cat README.md >>"$tmpFile" + update README.md 'Travis: add badge to README.md' + fi + + # encrypt key/value pairs in variables file + if [ -f "$varsFile" ] + then + while read p; do + test "${p:0:1}" = "#" && continue + info "Encrypting ${p%%=*}" + echo yes | $EXEC travis encrypt "$p" --add env.global + done <"$varsFile" + $EXEC git commit "$travisConfig" -m "Travis: add encrypted environment variables" + else + warn "No $varsFile found. Travis will not have any environment variables set!" + fi + + # encrypt GPG keypair + if [ -f "$signingKeySourceFile" ] + then + info "Encrypting $signingKeyDestFile" + # NB: We have to copy the file first, so that --add does the right thing. + $EXEC cp "$signingKeySourceFile" "$signingKeyDestFile" + $EXEC travis encrypt-file "$signingKeyDestFile" "$signingKeyDestFile.enc" --add + $EXEC rm -f "$signingKeyDestFile" + $EXEC git add "$travisConfig" "$signingKeyDestFile.enc" + $EXEC git commit -m "Travis: add encrypted GPG signing keypair" + else + warn "No $signingKeySourceFile found. Travis will not be able to do GPG signing!" + fi +} + +test -d "$credentialsDir" || + die "This script requires configuration stored in $credentialsDir,\n" \ + "including $varsFile for needed environment variables,\n" \ + "and $signingKeySourceFile for signing of artifacts.\n" \ + "Please contact a SciJava administrator to receive a copy of this content." + +# check prerequisites +check git sed xmllint travis + +# parse arguments +EXEC=: +while test $# -gt 0 +do + case "$1" in + -f) EXEC=;; + --) break;; + -*) echo "Ignoring unknown option: $1" >&2; break;; + *) break;; + esac + shift +done + +test "$EXEC" && warn "Simulation only. Run with -f flag to go for real." + +# process arguments +if [ $# -gt 0 ] +then + for d in $@ + do ( + echo "[$d]" + process "$d" + ) done +else + process . +fi From a9bb401978e04875384258ac5ef47a8dc275376c Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 22 Oct 2017 07:15:44 -0500 Subject: [PATCH 057/429] travisify.sh: also remove .travis/notify.sh It was never used for anything real. The ImageJ Jenkins, which is going to be retired soon, does not need to know when a Travis build happens. --- travisify.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/travisify.sh b/travisify.sh index 6a50a83..0ae0a98 100755 --- a/travisify.sh +++ b/travisify.sh @@ -12,6 +12,7 @@ travisDir=.travis travisConfig=.travis.yml travisBuildScript=$travisDir/build.sh travisSettingsFile=$travisDir/settings.xml +travisNotifyScript=$travisDir/notify.sh credentialsDir=$HOME/.scijava/credentials varsFile=$credentialsDir/vars signingKeySourceFile=$credentialsDir/scijava-ci-signing.asc @@ -114,12 +115,10 @@ EOL update "$travisBuildScript" # Remove obsolete Travis-related files. - if [ -f "$travisSettingsFile" ] - then - info "Removing obsolete $travisSettingsFile (travis-build.sh generates it now)." - $EXEC git rm -f "$travisSettingsFile" - $EXEC git ci -m "Travis: remove obsolete $travisSettingsFile" - fi + test -f "$travisSettingsFile" && info "Removing obsolete $travisSettingsFile (travis-build.sh generates it now)" + test -f "$travisNotifyScript" && info "Removing obsolete $travisNotifyScript (ImageJ Jenkins is going away)" + $EXEC git rm -f "$travisSettingsFile" "$travisNotifyScript" + $EXEC git diff-index --quiet HEAD -- || $EXEC git ci -m "Travis: remove obsolete files" # Upgrade version of pom-scijava. version=$(xmllint --xpath "//*[local-name()='project']/*[local-name()='parent']/*[local-name()='version']" pom.xml|sed 's/[^>]*>//'|sed 's/<.*//') From eb767d57475d8cef2212d3af4280f1e92029454c Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 22 Oct 2017 12:59:15 -0500 Subject: [PATCH 058/429] release-version.sh: remove local deploy option It is not part of the new workflow, and is one more thing we would have to test and debug. Supporting it is not trivial, so away it goes. --- release-version.sh | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/release-version.sh b/release-version.sh index 3162e8c..fac9400 100755 --- a/release-version.sh +++ b/release-version.sh @@ -73,7 +73,6 @@ IMAGEJ_THIRDPARTY_REPOSITORY=$IMAGEJ_BASE_REPOSITORY/thirdparty BATCH_MODE=--batch-mode SKIP_PUSH= -DEPLOY= TAG= DEV_VERSION= EXTRA_ARGS= @@ -87,7 +86,6 @@ do --dry-run) DRY_RUN=echo;; --no-batch-mode) BATCH_MODE=;; --skip-push) SKIP_PUSH=t;; - --deploy) DEPLOY=t;; --tag=*) ! git rev-parse --quiet --verify refs/tags/"${1#--*=}" || die "Tag ${1#--*=} exists already!" @@ -215,21 +213,3 @@ then $DRY_RUN git push "$REMOTE" HEAD $tag fi || exit - -# TODO - Evaluate whether to use "mvn release:perform" when doing local deploy. -if test "$DEPLOY" -then - $DRY_RUN git checkout $tag && - $DRY_RUN mvn $PROFILE \ - -DperformRelease \ - clean verify && - $DRY_RUN mvn $PROFILE \ - $ALT_REPOSITORY \ - -DperformRelease -DupdateReleaseInfo=true \ - deploy && - $DRY_RUN git checkout @{-1} - if test -n "$INVALIDATE_NEXUS" - then - $DRY_RUN maven_helper invalidate-cache "${BASE_GAV%:*}" - fi -fi From 0a40f394a8f27c2b4a53f54fa4e5e1fc9e0724c8 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 22 Oct 2017 13:01:22 -0500 Subject: [PATCH 059/429] release-version.sh: simplify GPG config validation We don't need a properly configured GPG configuration on the client side anymore. But Travis needs a GPG signing keypair, which lives in .travis/signingkey.asc.enc. Fail the release if it's missing. --- release-version.sh | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/release-version.sh b/release-version.sh index fac9400..cf1f89a 100755 --- a/release-version.sh +++ b/release-version.sh @@ -21,29 +21,8 @@ valid_semver_bump () { } verify_gpg_settings () { - gpg=$(xmllint --xpath '//settings/profiles/profile' "$HOME/.m2/settings.xml") - test "$gpg" && id=$(echo "$gpg" | xmllint --xpath '//profile/id/text()' -) - test "$gpg" && keyname=$(echo "$gpg" | xmllint --xpath '//profile/properties/gpg.keyname' -) - test "$gpg" && passphrase=$(echo "$gpg" | xmllint --xpath '//profile/properties/gpg.passphrase' -) - test "$keyname" -a "$passphrase" || - die 'GPG configuration not found in settings.xml. Please add it: - - - - gpg - - - ${env.HOME}/.gnupg - - - - (your GPG email address) - (your GPG passphrase) - - - - - + test -f .travis/signingkey.asc.enc || + die 'GPG configuration not found. Please use travisify.sh to add it. See also: https://github.com/scijava/pom-scijava/wiki/GPG-Signing' } From 53ac7bae63a5f7e70af322c67c76c2d964180537 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 22 Oct 2017 13:08:27 -0500 Subject: [PATCH 060/429] travisify.sh: make the new build.sh executable --- travisify.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/travisify.sh b/travisify.sh index 0ae0a98..fd244c1 100755 --- a/travisify.sh +++ b/travisify.sh @@ -112,6 +112,7 @@ EOL curl -fsLO https://raw.githubusercontent.com/scijava/scijava-scripts/master/travis-build.sh sh travis-build.sh EOL + chmod +x "$tmpFile" update "$travisBuildScript" # Remove obsolete Travis-related files. From f1d45ee77d67926af6828d7a628ccbbdefeec8fe Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 22 Oct 2017 13:13:26 -0500 Subject: [PATCH 061/429] travis-build.sh: import the key without passphrase I did not believe http://www.debonair.io/post/maven-cd/ when it said you can just run "gpg --fast-import .travis/signingkey.asc". Surely, I thought, you must also give the passphrase, since when you try to use this command locally, it demands one? But I guess something is different with the Travis configuration, because the command says: gpg: key : public key "SciJava CI " imported gpg: key : secret key imported gpg: key : "SciJava CI " not changed gpg: can't open `--passphrase': No such file or directory gpg: can't open `[secure]': No such file or directory So let's try without specifying the passphrase, and see if it works. --- travis-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis-build.sh b/travis-build.sh index 228451f..4765db8 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -60,7 +60,7 @@ if [ "$TRAVIS_SECURE_ENV_VARS" = true \ -a -f "$keyFile" ] then echo "== Importing GPG keypair ==" - gpg --batch --fast-import "$keyFile" --passphrase "$GPG_PASSPHRASE" + gpg --batch --fast-import "$keyFile" fi # Run the build. From 199784faf9d2562946fae7b90e2145f42b488e82 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 22 Oct 2017 13:41:35 -0500 Subject: [PATCH 062/429] travisify.sh: eliminate bashism --- travisify.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/travisify.sh b/travisify.sh index fd244c1..ab6d369 100755 --- a/travisify.sh +++ b/travisify.sh @@ -161,7 +161,10 @@ EOL if [ -f "$varsFile" ] then while read p; do - test "${p:0:1}" = "#" && continue + # Skip comments. (Cannot use ${p:0:1} because it's bash-specific.) + case "$p" in + '#'*) continue;; + esac info "Encrypting ${p%%=*}" echo yes | $EXEC travis encrypt "$p" --add env.global done <"$varsFile" From 03621c890ba6f24d4fe62f2bb6421845fafef2d9 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 23 Oct 2017 12:44:05 -0500 Subject: [PATCH 063/429] travisify.sh: strip .git suffix from repo slug Including the .git suffix when cloning is optional. Either way, we don't want it present in the repo slug. --- travisify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travisify.sh b/travisify.sh index ab6d369..4fcf545 100755 --- a/travisify.sh +++ b/travisify.sh @@ -66,7 +66,7 @@ process() { cd "$1" # -- Git sanity checks -- - repoSlug=$(git remote -v | grep origin | head -n1 | sed 's/.*github.com.\([^ ]*\) .*/\1/') + repoSlug=$(git remote -v | grep origin | head -n1 | sed 's/.*github.com.\([^ ]*\) .*/\1/' | sed 's/\.git$//') test "$repoSlug" && info "Repository = $repoSlug" || die 'Could not determine GitHub repository slug' git fetch >/dev/null git diff-index --quiet HEAD -- || die "Dirty working copy" From e08f11036c79bb29615168c0a617de188ecd0d5d Mon Sep 17 00:00:00 2001 From: Hadrien Mary Date: Wed, 25 Oct 2017 17:09:16 -0400 Subject: [PATCH 064/429] Use pom.xml for repoSlug --- travisify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travisify.sh b/travisify.sh index 4fcf545..a63ed61 100755 --- a/travisify.sh +++ b/travisify.sh @@ -66,7 +66,7 @@ process() { cd "$1" # -- Git sanity checks -- - repoSlug=$(git remote -v | grep origin | head -n1 | sed 's/.*github.com.\([^ ]*\) .*/\1/' | sed 's/\.git$//') + repoSlug=$(grep '' pom.xml 2>/dev/null | sed 's/[^>]*>//' | sed 's/<.*//' | cut -d'/' -f4,5) test "$repoSlug" && info "Repository = $repoSlug" || die 'Could not determine GitHub repository slug' git fetch >/dev/null git diff-index --quiet HEAD -- || die "Dirty working copy" From 61166d779b23d88880dc9b2729fbdd457b1346ae Mon Sep 17 00:00:00 2001 From: Hadrien Mary Date: Wed, 25 Oct 2017 17:24:16 -0400 Subject: [PATCH 065/429] Add repoSlug to travis encrypt-file --- travisify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travisify.sh b/travisify.sh index a63ed61..9600bc5 100755 --- a/travisify.sh +++ b/travisify.sh @@ -179,7 +179,7 @@ EOL info "Encrypting $signingKeyDestFile" # NB: We have to copy the file first, so that --add does the right thing. $EXEC cp "$signingKeySourceFile" "$signingKeyDestFile" - $EXEC travis encrypt-file "$signingKeyDestFile" "$signingKeyDestFile.enc" --add + $EXEC travis encrypt-file "$signingKeyDestFile" "$signingKeyDestFile.enc" --add --repo $repoSlug $EXEC rm -f "$signingKeyDestFile" $EXEC git add "$travisConfig" "$signingKeyDestFile.enc" $EXEC git commit -m "Travis: add encrypted GPG signing keypair" From d6b8e2f2df02d331a7b336dd9af3616514a9d6c5 Mon Sep 17 00:00:00 2001 From: Hadrien Mary Date: Wed, 25 Oct 2017 17:55:33 -0400 Subject: [PATCH 066/429] Use --repo with the travis encrypt command --- travisify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travisify.sh b/travisify.sh index 9600bc5..3704003 100755 --- a/travisify.sh +++ b/travisify.sh @@ -166,7 +166,7 @@ EOL '#'*) continue;; esac info "Encrypting ${p%%=*}" - echo yes | $EXEC travis encrypt "$p" --add env.global + echo yes | $EXEC travis encrypt "$p" --add env.global --repo $repoSlug done <"$varsFile" $EXEC git commit "$travisConfig" -m "Travis: add encrypted environment variables" else From 0643a6faeaa7ed733d0a254a120373c51f81a04f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 26 Oct 2017 23:03:09 -0500 Subject: [PATCH 067/429] travisify.sh: quote repoSlug variable It is conceivable, though highly unlikely, that the repoSlug extracted from the pom.xml will have whitespace, and/or be somehow antagonistally formed. So best practice security-wise is to quote it. --- travisify.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travisify.sh b/travisify.sh index 3704003..966632d 100755 --- a/travisify.sh +++ b/travisify.sh @@ -166,7 +166,7 @@ EOL '#'*) continue;; esac info "Encrypting ${p%%=*}" - echo yes | $EXEC travis encrypt "$p" --add env.global --repo $repoSlug + echo yes | $EXEC travis encrypt "$p" --add env.global --repo "$repoSlug" done <"$varsFile" $EXEC git commit "$travisConfig" -m "Travis: add encrypted environment variables" else @@ -179,7 +179,7 @@ EOL info "Encrypting $signingKeyDestFile" # NB: We have to copy the file first, so that --add does the right thing. $EXEC cp "$signingKeySourceFile" "$signingKeyDestFile" - $EXEC travis encrypt-file "$signingKeyDestFile" "$signingKeyDestFile.enc" --add --repo $repoSlug + $EXEC travis encrypt-file "$signingKeyDestFile" "$signingKeyDestFile.enc" --add --repo "$repoSlug" $EXEC rm -f "$signingKeyDestFile" $EXEC git add "$travisConfig" "$signingKeyDestFile.enc" $EXEC git commit -m "Travis: add encrypted GPG signing keypair" From 1664dd9bbc2059f11b2cfdea21a69fdd16b3a4d8 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 26 Oct 2017 23:04:17 -0500 Subject: [PATCH 068/429] travisify.sh: remove no-longer-needed yes response The "yes" was used to confirm the detected repository slug. But now we specify it explicitly, so it should no longer be necessary... I say without testing it. --- travisify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travisify.sh b/travisify.sh index 966632d..700aa50 100755 --- a/travisify.sh +++ b/travisify.sh @@ -166,7 +166,7 @@ EOL '#'*) continue;; esac info "Encrypting ${p%%=*}" - echo yes | $EXEC travis encrypt "$p" --add env.global --repo "$repoSlug" + $EXEC travis encrypt "$p" --add env.global --repo "$repoSlug" done <"$varsFile" $EXEC git commit "$travisConfig" -m "Travis: add encrypted environment variables" else From c12abb6877ed541e925da837994e8535f5d09625 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 26 Oct 2017 23:12:57 -0500 Subject: [PATCH 069/429] travisify.sh: use xmllint to extract repoSlug We already require xmllint for the other POM extractions. It is more robust in that it will always find the correct child element, whereas grep alone might conceivably pick up anything such as an XML comment. --- travisify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travisify.sh b/travisify.sh index 700aa50..c5b64e0 100755 --- a/travisify.sh +++ b/travisify.sh @@ -66,7 +66,7 @@ process() { cd "$1" # -- Git sanity checks -- - repoSlug=$(grep '' pom.xml 2>/dev/null | sed 's/[^>]*>//' | sed 's/<.*//' | cut -d'/' -f4,5) + repoSlug=$(xmllint --xpath "//*[local-name()='project']/*[local-name()='scm']/*[local-name()='connection']" pom.xml|sed 's_.*github.com[:/]\(.*\)<.*_\1_') test "$repoSlug" && info "Repository = $repoSlug" || die 'Could not determine GitHub repository slug' git fetch >/dev/null git diff-index --quiet HEAD -- || die "Dirty working copy" From 885c60f324c4e9a0d5a34837007e8bed80a306c3 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 29 Oct 2017 22:10:49 -0500 Subject: [PATCH 070/429] travisify.sh: decrypt only when applicable The "travis encrypt-file" directive adds the openssl decryption command to before_install, but does not guard it. So if the key and iv variables are unavailable, the decryption -- and hence the build -- will fail. We need to guard against this situation. --- travisify.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/travisify.sh b/travisify.sh index c5b64e0..67ecbd7 100755 --- a/travisify.sh +++ b/travisify.sh @@ -180,6 +180,9 @@ EOL # NB: We have to copy the file first, so that --add does the right thing. $EXEC cp "$signingKeySourceFile" "$signingKeyDestFile" $EXEC travis encrypt-file "$signingKeyDestFile" "$signingKeyDestFile.enc" --add --repo "$repoSlug" + # NB: Decrypt only when secure env vars are set. + # Without this adjustment, all PR builds will fail. + $EXEC perl -0777 -i -pe 's/\n- (openssl aes-256-cbc)/\n- test "\$TRAVIS_SECURE_ENV_VARS" = true &&\n \1/igs' .travis.yml $EXEC rm -f "$signingKeyDestFile" $EXEC git add "$travisConfig" "$signingKeyDestFile.enc" $EXEC git commit -m "Travis: add encrypted GPG signing keypair" @@ -195,7 +198,7 @@ test -d "$credentialsDir" || "Please contact a SciJava administrator to receive a copy of this content." # check prerequisites -check git sed xmllint travis +check git sed perl xmllint travis # parse arguments EXEC=: From 9190157727111bacb2b28fc56e28990899682f49 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 30 Oct 2017 16:14:18 -0500 Subject: [PATCH 071/429] travis-build.sh: decrypt the GPG signing keypair For this to happen, the caller passes key and iv values to the script. --- travis-build.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/travis-build.sh b/travis-build.sh index 4765db8..12b461d 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -55,6 +55,14 @@ EOL # Import the GPG signing key. keyFile=.travis/signingkey.asc +key=$1 +iv=$2 +if [ "$key" -a "$iv" -a -f "$keyFile.enc" ] +then + # NB: Key and iv values were given as arguments. + echo "== Decrypting GPG keypair ==" + openssl aes-256-cbc -K "$key" -iv "$iv" -in "$keyFile.enc" -out "$keyFile" -d +fi if [ "$TRAVIS_SECURE_ENV_VARS" = true \ -a "$TRAVIS_PULL_REQUEST" = false \ -a -f "$keyFile" ] From e4a363752a5e22808b6381ba3c0ca141a1a68c64 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 30 Oct 2017 16:37:16 -0500 Subject: [PATCH 072/429] travisify.sh: change how GPG decryption is done We now pass the key and iv values to the travis-build.sh script, rather than letting the "travis encrypt-file" command add it to the .travis.yml file directly. Because when the travis tool does it, it does not ensure the encrypted environment variables are actually set. --- travisify.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/travisify.sh b/travisify.sh index 67ecbd7..4fb79c0 100755 --- a/travisify.sh +++ b/travisify.sh @@ -177,15 +177,16 @@ EOL if [ -f "$signingKeySourceFile" ] then info "Encrypting $signingKeyDestFile" - # NB: We have to copy the file first, so that --add does the right thing. - $EXEC cp "$signingKeySourceFile" "$signingKeyDestFile" - $EXEC travis encrypt-file "$signingKeyDestFile" "$signingKeyDestFile.enc" --add --repo "$repoSlug" - # NB: Decrypt only when secure env vars are set. - # Without this adjustment, all PR builds will fail. - $EXEC perl -0777 -i -pe 's/\n- (openssl aes-256-cbc)/\n- test "\$TRAVIS_SECURE_ENV_VARS" = true &&\n \1/igs' .travis.yml - $EXEC rm -f "$signingKeyDestFile" - $EXEC git add "$travisConfig" "$signingKeyDestFile.enc" - $EXEC git commit -m "Travis: add encrypted GPG signing keypair" + if [ -z "$EXEC" ] + then + encryptResult=$(travis encrypt-file "$signingKeySourceFile" "$signingKeyDestFile.enc" --repo "$repoSlug" | grep openssl) + key=$(echo "$encryptResult" | cut -d' ' -f4) + iv=$(echo "$encryptResult" | cut -d' ' -f6) + sed -i.bak "s/\(sh travis-build.sh\)/\1 $key $iv/" "$travisBuildScript" + rm -f "$travisBuildScript.bak" + git add "$travisBuildScript" "$signingKeyDestFile.enc" + git commit -m "Travis: add encrypted GPG signing keypair" + fi else warn "No $signingKeySourceFile found. Travis will not be able to do GPG signing!" fi @@ -198,7 +199,7 @@ test -d "$credentialsDir" || "Please contact a SciJava administrator to receive a copy of this content." # check prerequisites -check git sed perl xmllint travis +check git sed cut perl xmllint travis # parse arguments EXEC=: From cbc3ed2e729f5bd1aa94272b6223e9c2afdaa3b9 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 1 Nov 2017 16:01:32 -0500 Subject: [PATCH 073/429] travisify.sh: fix removal of obsolete files If .travis/notify.sh was missing, the script did not generate the correct commit removing only .travis/settings.xml. The script is now more cautious, calling "git rm -f" on each file only if it is present. --- travisify.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/travisify.sh b/travisify.sh index 4fb79c0..2da33b7 100755 --- a/travisify.sh +++ b/travisify.sh @@ -116,9 +116,16 @@ EOL update "$travisBuildScript" # Remove obsolete Travis-related files. - test -f "$travisSettingsFile" && info "Removing obsolete $travisSettingsFile (travis-build.sh generates it now)" - test -f "$travisNotifyScript" && info "Removing obsolete $travisNotifyScript (ImageJ Jenkins is going away)" - $EXEC git rm -f "$travisSettingsFile" "$travisNotifyScript" + if [ -f "$travisSettingsFile" ] + then + info "Removing obsolete $travisSettingsFile (travis-build.sh generates it now)" + $EXEC git rm -f "$travisSettingsFile" + fi + if [ -f "$travisNotifyScript" ] + then + info "Removing obsolete $travisNotifyScript (ImageJ Jenkins is going away)" + $EXEC git rm -f "$travisNotifyScript" + fi $EXEC git diff-index --quiet HEAD -- || $EXEC git ci -m "Travis: remove obsolete files" # Upgrade version of pom-scijava. From a014cc8952d6f380111b2309778aa8714e476723 Mon Sep 17 00:00:00 2001 From: Jan Eglinger Date: Fri, 10 Nov 2017 11:38:15 +0100 Subject: [PATCH 074/429] Replace quotes for xmllint on Windows Git Bash --- travisify.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/travisify.sh b/travisify.sh index 2da33b7..55735f3 100755 --- a/travisify.sh +++ b/travisify.sh @@ -66,7 +66,7 @@ process() { cd "$1" # -- Git sanity checks -- - repoSlug=$(xmllint --xpath "//*[local-name()='project']/*[local-name()='scm']/*[local-name()='connection']" pom.xml|sed 's_.*github.com[:/]\(.*\)<.*_\1_') + repoSlug=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="scm"]/*[local-name()="connection"]' pom.xml|sed 's_.*github.com[:/]\(.*\)<.*_\1_') test "$repoSlug" && info "Repository = $repoSlug" || die 'Could not determine GitHub repository slug' git fetch >/dev/null git diff-index --quiet HEAD -- || die "Dirty working copy" @@ -78,7 +78,7 @@ process() { # die "Mismatch with upstream branch (local ahead?)" # -- POM sanity checks -- - parent=$(xmllint --xpath "//*[local-name()='project']/*[local-name()='parent']/*[local-name()='artifactId']" pom.xml|sed 's/[^>]*>//'|sed 's/<.*//') + parent=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="parent"]/*[local-name()="artifactId"]' pom.xml|sed 's/[^>]*>//'|sed 's/<.*//') test "$parent" = "pom-scijava" || die "Not pom-scijava parent: $parent" @@ -129,7 +129,7 @@ EOL $EXEC git diff-index --quiet HEAD -- || $EXEC git ci -m "Travis: remove obsolete files" # Upgrade version of pom-scijava. - version=$(xmllint --xpath "//*[local-name()='project']/*[local-name()='parent']/*[local-name()='version']" pom.xml|sed 's/[^>]*>//'|sed 's/<.*//') + version=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="parent"]/*[local-name()="version"]' pom.xml|sed 's/[^>]*>//'|sed 's/<.*//') # HACK: Using a lexicographic comparison here is imperfect. if [ "$version" \< "$pomMinVersion" ] then From bc599386ad72fdf676b6f845b6d9522fdd2a7c1b Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 10 Nov 2017 12:41:29 -0600 Subject: [PATCH 075/429] release-version.sh: always add release.properties Even if release.properties is in the .gitignore, we want it to be added here, for this special purpose. See #17. --- release-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-version.sh b/release-version.sh index cf1f89a..d26daf5 100755 --- a/release-version.sh +++ b/release-version.sh @@ -180,7 +180,7 @@ test -n "$tag" && sed -i.bak -e 's|^scm.url=scm\\:git\\:git@github.com\\:|scm.url=scm\\:git\\:git\\://github.com/|' release.properties && rm release.properties.bak && $DRY_RUN git checkout "$tag" && -$DRY_RUN git add release.properties && +$DRY_RUN git add -f release.properties && $DRY_RUN git commit --amend --no-edit && $DRY_RUN git tag -d "$tag" && $DRY_RUN git tag "$tag" HEAD && From 33b3414eb62c3a8f4802009cc6d26b987dc03a21 Mon Sep 17 00:00:00 2001 From: Stefan Helfrich Date: Fri, 17 Nov 2017 09:33:23 +0100 Subject: [PATCH 076/429] travis-build.sh: install gpg on osx --- travis-build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/travis-build.sh b/travis-build.sh index 12b461d..8a49451 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -53,6 +53,12 @@ cat >"$settingsFile" < EOL +# Install GPG on OSX/macOS +if [ "$TRAVIS_OS_NAME" = osx ] +then + brew install gnupg2 +fi + # Import the GPG signing key. keyFile=.travis/signingkey.asc key=$1 From 8e79f3b765071ba93cdcad6868bb5e568a17a02a Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 27 Nov 2017 16:30:55 -0600 Subject: [PATCH 077/429] release-version.sh: extract version more reliably If the POM did not use tabs as leading indentation, it failed. Invoking mvn directly is more reliable, if slower. --- release-version.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release-version.sh b/release-version.sh index d26daf5..36cd275 100755 --- a/release-version.sh +++ b/release-version.sh @@ -96,7 +96,8 @@ verify_git_settings test -n "$INVALIDATE_NEXUS" && verify_netrc_settings -pomVersion="$(sed -n 's/^ \(.*\)-SNAPSHOT<\/version>$/\1/p' pom.xml)" +devVersion=$(mvn -Dexec.executable='echo' -Dexec.args='${project.version}' exec:exec -q) +pomVersion=${devVersion%-SNAPSHOT} test $# = 1 || test ! -t 0 || { version=$pomVersion printf 'Version? [%s]: ' "$version" From 072f5e129acd830db8afa9a5ee0997e3adf6d8c5 Mon Sep 17 00:00:00 2001 From: Tobias Pietzsch Date: Thu, 30 Nov 2017 16:04:12 +0100 Subject: [PATCH 078/429] Raise error if the repo slug ends in ".git" If the pom gives the scm connection as ".../project.git" instead of ".../project" then 'travis encrypt' fails with an unhelpful error message. This prints something more helpful. --- travisify.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/travisify.sh b/travisify.sh index 55735f3..dbf616e 100755 --- a/travisify.sh +++ b/travisify.sh @@ -68,6 +68,7 @@ process() { # -- Git sanity checks -- repoSlug=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="scm"]/*[local-name()="connection"]' pom.xml|sed 's_.*github.com[:/]\(.*\)<.*_\1_') test "$repoSlug" && info "Repository = $repoSlug" || die 'Could not determine GitHub repository slug' + [[ "$repoSlug" == *.git ]] && die 'GitHub repository slug ('$repoSlug') ends in ".git". Please fix the pom' git fetch >/dev/null git diff-index --quiet HEAD -- || die "Dirty working copy" branch=$(git rev-parse --abbrev-ref HEAD) From bda6217d7c007650a29ab20d57007eb396fefdb0 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Nov 2017 09:11:52 -0600 Subject: [PATCH 079/429] travisify: fix bashism We want to stay POSIX-friendly. --- travisify.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/travisify.sh b/travisify.sh index dbf616e..a112b16 100755 --- a/travisify.sh +++ b/travisify.sh @@ -68,7 +68,11 @@ process() { # -- Git sanity checks -- repoSlug=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="scm"]/*[local-name()="connection"]' pom.xml|sed 's_.*github.com[:/]\(.*\)<.*_\1_') test "$repoSlug" && info "Repository = $repoSlug" || die 'Could not determine GitHub repository slug' - [[ "$repoSlug" == *.git ]] && die 'GitHub repository slug ('$repoSlug') ends in ".git". Please fix the pom' + case "$repoSlug" in + *.git) + die "GitHub repository slug ('$repoSlug') ends in '.git'; please fix the POM" + ;; + esac git fetch >/dev/null git diff-index --quiet HEAD -- || die "Dirty working copy" branch=$(git rev-parse --abbrev-ref HEAD) From 69e75ebd0cdcdf3a999200e5ad58f72765cfb619 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 13 Dec 2017 19:36:01 +0100 Subject: [PATCH 080/429] Print travis_fold around the script --- travis-build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/travis-build.sh b/travis-build.sh index 8a49451..446d62e 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -4,6 +4,8 @@ # travis-build.sh - A script to build and/or release SciJava-based projects. # +echo travis_fold:start:travis-build.sh + dir="$(dirname "$0")" echo "== Configuring Maven ==" @@ -94,3 +96,4 @@ else echo "== Building the artifact locally only ==" mvn -B install fi +echo travis_fold:end:travis-build.sh From 5eb1673053b7f945f519e5147fa26dc2a3c2d611 Mon Sep 17 00:00:00 2001 From: Tobias Pietzsch Date: Wed, 10 Jan 2018 19:13:14 +0100 Subject: [PATCH 081/429] Add -p flag to skip "parent == pom-scijava" check --- travisify.sh | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/travisify.sh b/travisify.sh index a112b16..34ef03a 100755 --- a/travisify.sh +++ b/travisify.sh @@ -84,8 +84,11 @@ process() { # -- POM sanity checks -- parent=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="parent"]/*[local-name()="artifactId"]' pom.xml|sed 's/[^>]*>//'|sed 's/<.*//') - test "$parent" = "pom-scijava" || - die "Not pom-scijava parent: $parent" + if [ -z "$SKIPPARENTCHECK" ] + then + test "$parent" = "pom-scijava" || + die "Not pom-scijava parent: $parent. Run with -p flag to skip this check." + fi # -- Travis sanity checks -- test -e "$travisDir" -a ! -d "$travisDir" && die "$travisDir is not a directory" @@ -134,15 +137,18 @@ EOL $EXEC git diff-index --quiet HEAD -- || $EXEC git ci -m "Travis: remove obsolete files" # Upgrade version of pom-scijava. - version=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="parent"]/*[local-name()="version"]' pom.xml|sed 's/[^>]*>//'|sed 's/<.*//') - # HACK: Using a lexicographic comparison here is imperfect. - if [ "$version" \< "$pomMinVersion" ] + if [ -z "$SKIPPARENTCHECK" ] then - info 'Upgrading pom-scijava version' - sed "s|^ $version$| $pomMinVersion|" pom.xml >"$tmpFile" - update pom.xml "POM: update pom-scijava parent to $pomMinVersion" - else - info "Version of pom-scijava ($version) is OK" + version=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="parent"]/*[local-name()="version"]' pom.xml|sed 's/[^>]*>//'|sed 's/<.*//') + # HACK: Using a lexicographic comparison here is imperfect. + if [ "$version" \< "$pomMinVersion" ] + then + info 'Upgrading pom-scijava version' + sed "s|^ $version$| $pomMinVersion|" pom.xml >"$tmpFile" + update pom.xml "POM: update pom-scijava parent to $pomMinVersion" + else + info "Version of pom-scijava ($version) is OK" + fi fi releaseProfile=$(grep '' pom.xml 2>/dev/null | sed 's/[^>]*>//' | sed 's/<.*//') @@ -215,10 +221,12 @@ check git sed cut perl xmllint travis # parse arguments EXEC=: +SKIPPARENTCHECK= while test $# -gt 0 do case "$1" in -f) EXEC=;; + -p) SKIPPARENTCHECK=true;; --) break;; -*) echo "Ignoring unknown option: $1" >&2; break;; *) break;; From 148638059be8436cfc16805e8f1cc147a4f60816 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 25 Jan 2018 09:22:59 -0600 Subject: [PATCH 082/429] travis-build.sh: preserve the exit code Commit 69e75ebd0cdcdf3a999200e5ad58f72765cfb619 resulted in the script always returning success, because the final echo statement succeeded. --- travis-build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/travis-build.sh b/travis-build.sh index 446d62e..62547f8 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -86,14 +86,18 @@ if [ "$TRAVIS_SECURE_ENV_VARS" = true \ then echo "== Building and deploying master SNAPSHOT ==" mvn -B -Pdeploy-to-imagej deploy + success=$? elif [ "$TRAVIS_SECURE_ENV_VARS" = true \ -a "$TRAVIS_PULL_REQUEST" = false \ -a -f release.properties ] then echo "== Cutting and deploying release version ==" mvn -B release:perform + success=$? else echo "== Building the artifact locally only ==" mvn -B install + success=$? fi echo travis_fold:end:travis-build.sh +exit $success From c5033d55c95c85d6b2daa9ca0956031d45ca1a88 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 28 Feb 2018 15:35:13 -0600 Subject: [PATCH 083/429] melting-pot: include local projects in reactor Without this test, the isProject check fails, and LOCAL/PROJECT is not included in the multi-module build. --- melting-pot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index d16273d..76c99e9 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -484,7 +484,7 @@ pruneReactor() { # Tests if the given directory contains the appropriate source code. isProject() { local a="$(xpath "$1/pom.xml" project artifactId)" - test "$a" = "$(basename "$1")" && echo 1 + test "$1" = "LOCAL/PROJECT" -o "$a" = "$(basename "$1")" && echo 1 } # Generates an aggregator POM for all modules in the current directory. From 819059d11c5429f02fd9eb1945894ccbfd7dcab0 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 28 Feb 2018 15:46:48 -0600 Subject: [PATCH 084/429] melting-pot: fix optional dependency detection --- melting-pot.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index 76c99e9..806b2b5 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -403,7 +403,8 @@ deps() { debug "mvn dependency:list" local depList=$(mvn dependency:list) || die "Problem fetching dependencies!" 5 - echo "$depList" | grep '^\[INFO\] [^ ]' | sed 's/\[INFO\] //' | sort + echo "$depList" | grep '^\[INFO\] [^ ]' | + sed 's/\[INFO\] //' | sed 's/ *(optional) *$//' | sort cd - > /dev/null } From 7a6cb68466b3536407121f016f2e9af387aa616a Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 28 Feb 2018 16:03:13 -0600 Subject: [PATCH 085/429] Remove unused sj-hierarchy script The current strategy is to use melting-pot on a top-level project such as net.imagej:imagej or sc.fiji:fiji. More dependencies can be mixed in as needed as well using that script. --- .gitignore | 1 - sj-blacklist.txt | 152 ----------------- sj-hierarchy.pl | 430 ----------------------------------------------- 3 files changed, 583 deletions(-) delete mode 100644 .gitignore delete mode 100644 sj-blacklist.txt delete mode 100755 sj-hierarchy.pl diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 6aef28c..0000000 --- a/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/sj-hierarchy.cache diff --git a/sj-blacklist.txt b/sj-blacklist.txt deleted file mode 100644 index aad3bee..0000000 --- a/sj-blacklist.txt +++ /dev/null @@ -1,152 +0,0 @@ -# A list of obsolete artifacts. - -io.scif:bf-formats -io.scif:bom-scifio -io.scif:jai_imageio -io.scif:ome-formats -io.scif:pom-ome-formats -io.scif:pom-scifio-lifesci -io.scif:scifio-bio-formats -loci:bf-autogen -loci:bio-formats -loci:bio-formats-base -loci:deep-zoom-plugin -loci:jai_imageio -loci:jimi -loci:loci-base -loci:loci-checks -loci:loci-common -loci:loci-legacy -loci:loci_plugins -loci:loci_tools -loci:lwf-stubs -loci:mdbtools-java -loci:metakit -loci:ome-editor -loci:ome-io -loci:ome-notes -loci:ome-xml -loci:ome_plugins -loci:ome_tools -loci:poi-loci -loci:pom-scifio -loci:scifio -loci:scifio-devel -loci:scifio-tools -loci:slim-plugin -loci:specification -loci:test-suite -loci:turbojpeg -loci:wiscscan-utils -loci:xsd-fu -net.imagej:bom-imagej -net.imagej:ij-app -net.imagej:ij-commands -net.imagej:ij-commands-app -net.imagej:ij-commands-data -net.imagej:ij-commands-debug -net.imagej:ij-commands-display -net.imagej:ij-commands-misc -net.imagej:ij-commands-upload -net.imagej:ij-core -net.imagej:ij-data -net.imagej:ij-ext -net.imagej:ij-ext-awt -net.imagej:ij-ext-pivot -net.imagej:ij-ext-swing -net.imagej:ij-ext-swt -net.imagej:ij-io -net.imagej:ij-launcher -net.imagej:ij-legacy -net.imagej:ij-log-slf4j -net.imagej:ij-minimaven -net.imagej:ij-omero -net.imagej:ij-omero-server -net.imagej:ij-ops -net.imagej:ij-options -net.imagej:ij-platform -net.imagej:ij-platform-macosx -net.imagej:ij-platform-windows -net.imagej:ij-platforms -net.imagej:ij-plugins-app -net.imagej:ij-plugins-data -net.imagej:ij-plugins-debug -net.imagej:ij-plugins-display -net.imagej:ij-plugins-misc -net.imagej:ij-script-beanshell -net.imagej:ij-script-java -net.imagej:ij-scripting-beanshell -net.imagej:ij-scripting-clojure -net.imagej:ij-scripting-java -net.imagej:ij-scripting-javascript -net.imagej:ij-scripting-jruby -net.imagej:ij-scripting-jython -net.imagej:ij-text -net.imagej:ij-tools -net.imagej:ij-ui -net.imagej:ij-ui-awt -net.imagej:ij-ui-common-awt -net.imagej:ij-ui-pivot -net.imagej:ij-ui-swing -net.imagej:ij-ui-swing-base -net.imagej:ij-ui-swing-commands -net.imagej:ij-ui-swing-mdi -net.imagej:ij-ui-swing-plugins -net.imagej:ij-ui-swing-script-editor -net.imagej:ij-ui-swing-sdi -net.imagej:ij-ui-swing-tools -net.imagej:ij-ui-swing-updater -net.imagej:ij-ui-swt -net.imagej:ij-updater-core -net.imagej:ij-updater-ssh -net.imagej:ij-updater-webdav -net.imagej:ij-util-awt -net.imagej:ij-util-swing -net.imagej:minimaven -net.imagej:pom-ij-base -net.imagej:pom-ij-core -net.imagej:pom-ij-core-commands -net.imagej:pom-ij-core-plugins -net.imagej:pom-ij-extra -net.imagej:pom-ij-omero -net.imagej:pom-ij-platform -net.imagej:pom-ij-plugins -net.imagej:pom-ij-script -net.imagej:pom-ij-scripting -net.imagej:pom-ij-ui -net.imagej:pom-ij-ui-awt -net.imagej:pom-ij-ui-awt-swing -net.imagej:pom-ij-ui-pivot -net.imagej:pom-ij-ui-swing -net.imagej:pom-ij-ui-swt -net.imagej:pom-ij-uis -net.imagej:pom-ij-updater -net.imagej:pom-ij-uploaders -net.imagej:pom-script -net.imglib2:GameOfDeath2 -net.imglib2:bom-imglib2 -net.imglib2:imglib-pom-base -net.imglib2:imglib2-algorithms -net.imglib2:imglib2-algorithms-gpl -net.imglib2:imglib2-algorithms-legacy -net.imglib2:imglib2-base -net.imglib2:imglib2-io -net.imglib2:pom-imglib -net.imglib2:pom-imglib2-algorithms -net.imglib2:pom-imglib2-examples -ome:bio-formats -ome:loci-common -ome:loci-legacy -ome:loci_plugins -ome:ome-io -ome:ome-java -ome:ome-java-deprecated -ome:ome_plugins -ome:omero-client -ome:omero-common -ome:poi-loci -ome:pom-scifio -ome:scifio -ome:scifio-devel -ome:scifio-tools -sc.fiji:cookbook diff --git a/sj-hierarchy.pl b/sj-hierarchy.pl deleted file mode 100755 index c59ed47..0000000 --- a/sj-hierarchy.pl +++ /dev/null @@ -1,430 +0,0 @@ -#!/usr/bin/perl - -# -# sj-hierarchy.pl -# - -# Script to print out hierarchy of Maven projects built on pom-scijava. -# Requires XML::Simple to be installed. - -# Usage: sj-hierarchy.pl - -use strict; -use XML::Simple; - -# -- Constants -- - -my @groupIds = ( - 'io.scif', - 'loci', - 'net.imagej', - 'net.imglib2', - 'org.scijava', - 'sc.fiji', -); - -my $home = $ENV{"HOME"}; -my $dir = `dirname "$0"`; -chop $dir; - -my %blacklist; -my %versions; -my %pomXML; -my %pomTree; - -my $quiet; -my $verbose; - -# -- Main -- - -{ - my $doCI; - my $doHelp; - my $doList; - my $doParents; - my $doSCM; - my $doStats; - my $doTree; - - for my $cmd (@ARGV) { - if ($cmd =~ /^-\w+$/) { - for my $c (split(//, $cmd)) { - if ($c eq 'c') { $doCI = 1; } - elsif ($c eq 'g') { $doSCM = 1; } - elsif ($c eq 'l') { $doList = 1; } - elsif ($c eq 'p') { $doParents = 1; } - elsif ($c eq 's') { $doStats = 1; } - elsif ($c eq 't') { $doTree = 1; } - elsif ($c eq 'q') { $quiet = 1; } - elsif ($c eq 'v') { $verbose = 1; } - } - } - elsif ($cmd eq '--ci') { $doCI = 1; } - elsif ($cmd eq '--help') { $doHelp = 1; } - elsif ($cmd eq '--list') { $doList = 1; } - elsif ($cmd eq '--parents') { $doParents = 1; } - elsif ($cmd eq '--scm') { $doSCM = 1; } - elsif ($cmd eq '--stats') { $doStats = 1; } - elsif ($cmd eq '--tree') { $doTree = 1; } - elsif ($cmd eq '--quiet') { $quiet = 1; } - elsif ($cmd eq '--verbose') { $verbose = 1; } - else { warning("Invalid argument: $cmd"); } - } - - $doCI || $doList || $doParents || $doSCM || $doStats || $doTree || - ($doHelp = 1); - - if ($doHelp) { - print STDERR "Usage: sj-hierachy.pl [-cglpstqv]\n"; - print STDERR "\n"; - print STDERR " -c, --ci : list involved CI URLs\n"; - print STDERR " -g, --scm : list involved SCM URLs\n"; - print STDERR " -l, --list : list SciJava artifacts\n"; - print STDERR " -p, --parents : show table of artifact parents\n"; - print STDERR " -s, --stats : show some statistics about the artifacts\n"; - print STDERR " -t, --tree : display artifacts in a tree structure\n"; - print STDERR " -q, --quiet : emit fewer details to stderr\n"; - print STDERR " -v, --verbose : emit more details to stderr\n"; - exit 1; - } - - parse_blacklist(); - resolve_artifacts(); - - if ($doCI) { - list_cis(); - } - if ($doList) { - show_list(); - } - if ($doParents) { - show_parents(); - } - if ($doTree) { - show_tree(); - } - if ($doSCM) { - list_scms(); - } - if ($doStats) { - report_statistics(); - } -} - -# -- Subroutines -- - -sub parse_blacklist() { - open(BLACKLIST, "$dir/sj-blacklist.txt"); - my @list = ; - close(BLACKLIST); - for my $ga (@list) { - chop $ga; - if ($ga && $ga !~ /^#/) { - $blacklist{$ga} = 1; - } - } -} - -sub resolve_artifacts($) { - my ($printList) = @_; - - my $cacheFile = "$dir/sj-hierarchy.cache"; - if (-e $cacheFile) { - # build versions table from cache - info("Reading artifact list from cache..."); - - open(CACHE, "$cacheFile"); - my @gavs = ; - close(CACHE); - for my $gav (@gavs) { - chop $gav; - my ($groupId, $artifactId, $version) = split(':', $gav); - my $ga = "$groupId:$artifactId"; - if ($blacklist{$ga}) { - detail("Ignoring blacklisted artifact: $ga"); - next; - } - $versions{$ga} = $version; - detail($gav); - } - } - else { - # build versions table from remote repository - info("Scanning for artifacts..."); - - open(CACHE, ">$cacheFile"); - for my $groupId (@groupIds) { - my @artifactIds = artifacts($groupId); - for my $artifactId (@artifactIds) { - my $ga = "$groupId:$artifactId"; - if ($blacklist{$ga}) { - detail("Ignoring blacklisted artifact: $ga"); - next; - } - - # determine the latest version - my $version = version($ga); - if ($version =~ /-SNAPSHOT$/) { - detail("Ignoring SNAPSHOT-only artifact: $ga"); - } - elsif ($version) { - info("$ga:$version"); - print CACHE "$ga:$version\n"; - } - } - } - close(CACHE); - } -} - -# Displays a list of artifacts -sub show_list() { - for my $ga (sort keys %versions) { - output("$ga:$versions{$ga}"); - } -} - -# Displays a list of artifacts -sub show_parents() { - my $width = 0; - for my $ga (keys %versions) { - my $gav = "$ga:" . version($ga); - my $w = length($gav); - if ($w > $width) { $width = $w; } - } - $width++; - - for my $ga (sort keys %versions) { - my $gav = "$ga:" . version($ga); - my $parent = parent($ga); - my $pGAV = $parent eq ':' ? '(none)' : "$parent:" . version($parent); - printf("%${width}s : %s\n", $gav, $pGAV); - } -} - -# Displays a parent-child tree of POMs -sub show_tree() { - # build the tree - for my $ga (keys %versions) { - my $version = version($ga); - $version || next; - my $parent = parent($ga); - if (!$pomTree{$parent}) { - my @children = (); - $pomTree{$parent} = \@children; - } - my $children = $pomTree{$parent}; - push @{$children}, $ga; - } - - # display the tree - dump_tree("org.scijava:pom-scijava", 0); -} - -# Makes a list of CIs associated with the artifacts. -sub list_cis() { - for my $ga (keys %versions) { - my $version = version($ga); - $version || next; - my $ci = ci($ga); - if (!$ci) { - warning("No CI for artifact: $ga:$version"); - next; - } - output("$ga: $ci"); - } -} - -# Makes a list of SCMs associated with the artifacts. -sub list_scms() { - my %scms; - for my $ga (keys %versions) { - my $version = version($ga); - $version || next; - my $scm = scm($ga); - if (!$scm) { - warning("No SCM for artifact: $ga:$version"); - next; - } - if ($scm !~ /^scm:git:/) { - warning("Unsupported SCM for artifact: $ga:$version"); - next; - } - $scm =~ s/^scm:git://; - $scms{$scm} = 1; - } - for my $scm (sort keys %scms) { - output($scm); - } -} - -# Reports some statistics. -sub report_statistics() { - my @gavs; - for my $ga (keys %versions) { - push @gavs, "$ga:$versions{$ga}"; - } - for my $groupId (@groupIds) { - my @total = grep(/^$groupId/, @gavs); - my @poms = grep(/:pom-/, @total); - my @snapshots = grep(/-SNAPSHOT$/, @total); - my @pomSnapshots = grep(/:pom-/, @snapshots); - - my $totalCount = scalar @total; - my $pomCount = scalar @poms; - my $snapshotCount = scalar @snapshots; - my $pomSnapshotCount = scalar @pomSnapshots; - - my $releaseCount = $totalCount - $snapshotCount; - my $pomReleaseCount = $pomCount - $pomSnapshotCount; - output("$groupId: $totalCount total ($releaseCount releases, " . - "$snapshotCount snapshots); $pomCount poms " . - "($pomReleaseCount releases, $pomSnapshotCount snapshots)"); - } -} - -# Recursively prints out the POM tree -sub dump_tree($$) { - my ($ga, $indent) = @_; - output(lead($indent) . "* $ga"); - my $children = $pomTree{$ga}; - if (!$children) { return; } - for my $child (@{$children}) { - dump_tree($child, $indent + 2); - } -} - -# Computes the latest version of a GA, caching the result. -sub version($) { - my ($ga) = @_; - if (!$versions{$ga}) { - $versions{$ga} = execute("$dir/maven-helper.sh latest-version \"$ga\""); - } - return $versions{$ga}; -} - -# Obtains the path to the given GAV's POM in the local repository. -sub pom_path($) { - my ($gav) = @_; - my ($groupId, $artifactId, $version) = split(':', $gav); - $groupId =~ s/\./\//g; - my $pomPath = "$home/.m2/repository/$groupId/" . - "$artifactId/$version/$artifactId-$version.pom"; - unless (-e $pomPath) { - # download POM to local repository cache - info("Resolving $gav from remote repository"); - execute("mvn org.apache.maven.plugins:maven-dependency-plugin:2.8:get " . - "-Dartifact=$gav:pom -DremoteRepositories=imagej.public::default::" . - "http://maven.imagej.net/content/groups/public -Dtransitive=false"); - } - return $pomPath; -} - -# Obtains the POM XML for the given GA, caching the result. -sub pom_xml($) { - my ($ga) = @_; - if (!$pomXML{$ga}) { - my $version = version($ga); - $pomXML{$ga} = XMLin(pom_path("$ga:$version")); - } - return $pomXML{$ga}; -} - -# Computes the parent of a GA. -sub parent($) { - my ($ga) = @_; - my $xml = pom_xml($ga); - my $groupId = $xml->{parent}->{groupId}; - my $artifactId = $xml->{parent}->{artifactId}; - return "$groupId:$artifactId"; -} - -# Computes the CI of a GA. -sub ci($) { - my ($ga) = @_; - my $xml = pom_xml($ga); - my $ci = $xml->{ciManagement}->{url}; - if (!$ci) { - my $parent = parent($ga); - return $parent && $parent ne ':' ? ci($parent) : undef; - } - return $ci; -} - -# Computes the SCM of a GA. -sub scm($) { - my ($ga) = @_; - my $xml = pom_xml($ga); - my $scm = $xml->{scm}->{connection}; - if (!$scm) { - my $parent = parent($ga); - return $parent && $parent ne ':' ? scm($parent) : undef; - } - return $scm; -} - -# Obtains a list of artifacts in the given group. -sub artifacts($) { - my ($groupId) = @_; - - my $url = $groupId; - $url =~ s/\./\//g; - $url = "http://maven.imagej.net/content/groups/public/$url/"; - - my @links = `curl -fs "$url"`; - my @artifacts = (); - for my $link (@links) { - if ($link =~ //) { - chop $link; - $link =~ s/ *<[^>]*>(.*)\/<\/a>/$1/; - push @artifacts, $link; - } - } - - return @artifacts; -} - -# Executes the given shell command. -sub execute($) { - my ($command) = @_; - detail($command); - my $result = `$command`; - chop $result; - return $result; -} - -sub lead($) { - my ($indent) = @_; - my $lead = ''; - for (my $i = 0; $i < $indent; $i++) { $lead .= ' '; } - return $lead; -} - -sub output($) { - my ($message) = @_; - print "$message\n"; -} - -sub info($) { - my ($message) = @_; - unless ($quiet) { - print STDERR "==> $message\n"; - } -} - -sub detail($) { - my ($message) = @_; - if ($verbose) { - print STDERR "==> $message\n"; - } -} - -sub warning($) { - my ($message) = @_; - print STDERR "[WARNING] $message\n"; -} - -sub error($) { - my ($message) = @_; - print STDERR "[ERROR] $message\n"; -} From 52b9d67860e9b6e679f69f06a42c099945b72c2e Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Mar 2018 09:59:23 -0600 Subject: [PATCH 086/429] melting-pot: save build command to shell script This makes it easier to repeat it as needed. --- melting-pot.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/melting-pot.sh b/melting-pot.sh index 806b2b5..4dd9cfe 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -591,6 +591,7 @@ meltDown() { generatePOM # Build everything. + echo "mvn $args test" > build.sh if [ "$skipBuild" ] then info "Skipping the build; the command would have been:" From 7a18feecc419e7a57b0b2402011bb713430f5ab6 Mon Sep 17 00:00:00 2001 From: Niko Ehrenfeuchter Date: Mon, 26 Mar 2018 13:51:57 +0200 Subject: [PATCH 087/429] Make verify_gpg_settings respect --skip-gpg --- release-version.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/release-version.sh b/release-version.sh index 36cd275..1e13da2 100755 --- a/release-version.sh +++ b/release-version.sh @@ -21,9 +21,12 @@ valid_semver_bump () { } verify_gpg_settings () { - test -f .travis/signingkey.asc.enc || - die 'GPG configuration not found. Please use travisify.sh to add it. + if [ ! "$SKIP_GPG" ] + then + test -f .travis/signingkey.asc.enc || + die 'GPG configuration not found. Please use travisify.sh to add it. See also: https://github.com/scijava/pom-scijava/wiki/GPG-Signing' + fi } verify_git_settings () { @@ -52,6 +55,7 @@ IMAGEJ_THIRDPARTY_REPOSITORY=$IMAGEJ_BASE_REPOSITORY/thirdparty BATCH_MODE=--batch-mode SKIP_PUSH= +SKIP_GPG= TAG= DEV_VERSION= EXTRA_ARGS= @@ -84,6 +88,7 @@ do SKIP_PUSH=t ALT_REPOSITORY=$IMAGEJ_THIRDPARTY_REPOSITORY;; --skip-gpg) + SKIP_GPG=t EXTRA_ARGS="$EXTRA_ARGS -Dgpg.skip=true";; -*) echo "Unknown option: $1" >&2; break;; *) break;; From 600e054e99d9864164b55fea105ee6fb1689209f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 10 Apr 2018 16:36:13 -0500 Subject: [PATCH 088/429] release-version.sh: simplify skip-gpg test --- release-version.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/release-version.sh b/release-version.sh index 1e13da2..2d8c4a9 100755 --- a/release-version.sh +++ b/release-version.sh @@ -21,10 +21,8 @@ valid_semver_bump () { } verify_gpg_settings () { - if [ ! "$SKIP_GPG" ] - then - test -f .travis/signingkey.asc.enc || - die 'GPG configuration not found. Please use travisify.sh to add it. + test "$SKIP_GPG" -o -f .travis/signingkey.asc.enc || + die 'GPG configuration not found. Please use travisify.sh to add it. See also: https://github.com/scijava/pom-scijava/wiki/GPG-Signing' fi } From cb20c6ec12d5cc403b695b68a4f8ca1500dc4cc2 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 10 Apr 2018 16:36:36 -0500 Subject: [PATCH 089/429] release-version.sh: remove obsolete netrc check This script no longer invalidates the Nexus repository, so the check is unnecessary. --- release-version.sh | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/release-version.sh b/release-version.sh index 2d8c4a9..77b2e7c 100755 --- a/release-version.sh +++ b/release-version.sh @@ -36,17 +36,6 @@ verify_git_settings () { fi } -verify_netrc_settings () { - grep -q 'machine maven.imagej.net' "$HOME/.netrc" 2>/dev/null && - grep -q 'login jenkins-expire-cache' "$HOME/.netrc" 2>/dev/null || - die 'maven.imagej.net cache expiration credentials not found in .netrc. Please add it: -machine maven.imagej.net -login jenkins-expire-cache -password (the-correct-password) - -See also: https://github.com/scijava/pom-scijava/wiki/Adding-Maven-Users' -} - IMAGEJ_BASE_REPOSITORY=-DaltDeploymentRepository=imagej.releases::default::dav:https://maven.imagej.net/content/repositories IMAGEJ_RELEASES_REPOSITORY=$IMAGEJ_BASE_REPOSITORY/releases IMAGEJ_THIRDPARTY_REPOSITORY=$IMAGEJ_BASE_REPOSITORY/thirdparty @@ -59,7 +48,6 @@ DEV_VERSION= EXTRA_ARGS= ALT_REPOSITORY= PROFILE=-Pdeploy-to-imagej -INVALIDATE_NEXUS= DRY_RUN= while test $# -gt 0 do @@ -96,9 +84,6 @@ done verify_git_settings -test -n "$INVALIDATE_NEXUS" && - verify_netrc_settings - devVersion=$(mvn -Dexec.executable='echo' -Dexec.args='${project.version}' exec:exec -q) pomVersion=${devVersion%-SNAPSHOT} test $# = 1 || test ! -t 0 || { @@ -131,8 +116,7 @@ die "Could not obtain GAV coordinates for base project" # If releasing to OSS Sonatype, enable some extra stuff mvn -Dexec.executable='echo' -Dexec.args='${releaseProfiles}' exec:exec -q | grep -q 'sonatype-oss-release' && - verify_gpg_settings && - INVALIDATE_NEXUS=t + verify_gpg_settings git update-index -q --refresh && git diff-files --quiet --ignore-submodules && From 6e8c398bc6ed9be59888a30544d9d2f0a454bda1 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 10 Apr 2018 16:42:16 -0500 Subject: [PATCH 090/429] travisify.sh: tweak variable name --- travisify.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/travisify.sh b/travisify.sh index 34ef03a..cafb134 100755 --- a/travisify.sh +++ b/travisify.sh @@ -84,7 +84,7 @@ process() { # -- POM sanity checks -- parent=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="parent"]/*[local-name()="artifactId"]' pom.xml|sed 's/[^>]*>//'|sed 's/<.*//') - if [ -z "$SKIPPARENTCHECK" ] + if [ -z "$SKIP_PARENT_CHECK" ] then test "$parent" = "pom-scijava" || die "Not pom-scijava parent: $parent. Run with -p flag to skip this check." @@ -137,7 +137,7 @@ EOL $EXEC git diff-index --quiet HEAD -- || $EXEC git ci -m "Travis: remove obsolete files" # Upgrade version of pom-scijava. - if [ -z "$SKIPPARENTCHECK" ] + if [ -z "$SKIP_PARENT_CHECK" ] then version=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="parent"]/*[local-name()="version"]' pom.xml|sed 's/[^>]*>//'|sed 's/<.*//') # HACK: Using a lexicographic comparison here is imperfect. @@ -221,12 +221,12 @@ check git sed cut perl xmllint travis # parse arguments EXEC=: -SKIPPARENTCHECK= +SKIP_PARENT_CHECK= while test $# -gt 0 do case "$1" in -f) EXEC=;; - -p) SKIPPARENTCHECK=true;; + -p) SKIP_PARENT_CHECK=true;; --) break;; -*) echo "Ignoring unknown option: $1" >&2; break;; *) break;; From fbf53137c86056e5cd0997d5536225fdeed7215d Mon Sep 17 00:00:00 2001 From: Jan Eglinger Date: Thu, 12 Apr 2018 12:36:23 +0200 Subject: [PATCH 091/429] Fix syntax error introduced by commit 600e054 --- release-version.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/release-version.sh b/release-version.sh index 77b2e7c..82d7600 100755 --- a/release-version.sh +++ b/release-version.sh @@ -24,7 +24,6 @@ verify_gpg_settings () { test "$SKIP_GPG" -o -f .travis/signingkey.asc.enc || die 'GPG configuration not found. Please use travisify.sh to add it. See also: https://github.com/scijava/pom-scijava/wiki/GPG-Signing' - fi } verify_git_settings () { From 69d29a51db29bb0e09c7af60589a42c0102856bd Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 19 Apr 2018 07:54:46 -0500 Subject: [PATCH 092/429] travis-build.sh: build javadocs also by default When building PRs, we'd like to verify the javadoc is correct. See https://gitter.im/imglib/imglib2?at=5ad79f3e5d7286b43a424494 --- travis-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis-build.sh b/travis-build.sh index 62547f8..75e22d4 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -96,7 +96,7 @@ then success=$? else echo "== Building the artifact locally only ==" - mvn -B install + mvn -B install javadoc:javadoc success=$? fi echo travis_fold:end:travis-build.sh From faf77db0dc5bd99bcac8c37db87949f44467534b Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 3 May 2018 09:03:50 -0700 Subject: [PATCH 093/429] Change travis-ci.org -> travis-ci.com See: https://blog.travis-ci.com/2018-05-02-open-source-projects-on-travis-ci-com-with-github-apps --- travis-javadoc.sh | 2 +- travisify.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/travis-javadoc.sh b/travis-javadoc.sh index a58349d..f107590 100644 --- a/travis-javadoc.sh +++ b/travis-javadoc.sh @@ -53,7 +53,7 @@ then echo '192.30.252.142 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> "$HOME/.ssh/known_hosts" && # Configure git settings. - git config --global user.email "travis@travis-ci.org" && + git config --global user.email "travis@travis-ci.com" && git config --global user.name "Travis CI" && # Clone the javadoc.scijava.org repository. diff --git a/travisify.sh b/travisify.sh index cafb134..0ea3761 100755 --- a/travisify.sh +++ b/travisify.sh @@ -166,10 +166,10 @@ EOL fi # update the README - if ! grep -q "travis-ci.org/$repoSlug" README.md >/dev/null 2>&1 + if ! grep -q "travis-ci.com/$repoSlug" README.md >/dev/null 2>&1 then info "Adding Travis badge to README.md" - echo "[![](https://travis-ci.org/$repoSlug.svg?branch=master)](https://travis-ci.org/$repoSlug)" >"$tmpFile" + echo "[![](https://travis-ci.com/$repoSlug.svg?branch=master)](https://travis-ci.com/$repoSlug)" >"$tmpFile" echo >>"$tmpFile" test -f README.md && cat README.md >>"$tmpFile" update README.md 'Travis: add badge to README.md' From 91e0b2dc9c9e95e19c2bae0a8d12c40334871dc5 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 11 May 2018 08:31:32 -0500 Subject: [PATCH 094/429] melting-pot: make build script accept args So you can say e.g. "build.sh -Dmy.version=1.2.3". --- melting-pot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index 4dd9cfe..32f76ba 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -591,7 +591,7 @@ meltDown() { generatePOM # Build everything. - echo "mvn $args test" > build.sh + echo "mvn $args test \$@" > build.sh if [ "$skipBuild" ] then info "Skipping the build; the command would have been:" From 85f6937cfe794c02788b5c6ec50e515b0d2f9774 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 23 May 2018 09:55:54 -0500 Subject: [PATCH 095/429] travis-build.sh: trigger Maven build by pom.xml If a pom.xml is not present, there is no need to do all the Maven stuff. This change will pave the way for this script to become more general, with additional and/or alternative steps besides only Maven builds. You may want to view this diff with the "-w" flag. --- travis-build.sh | 106 +++++++++++++++++++++++++----------------------- 1 file changed, 55 insertions(+), 51 deletions(-) diff --git a/travis-build.sh b/travis-build.sh index 75e22d4..d811a04 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -8,18 +8,21 @@ echo travis_fold:start:travis-build.sh dir="$(dirname "$0")" -echo "== Configuring Maven ==" +# Build Maven projects. +if [ -f pom.xml ] +then + echo "== Configuring Maven ==" -# NB: Suppress "Downloading/Downloaded" messages. -# See: https://stackoverflow.com/a/35653426/1207769 -export MAVEN_OPTS=-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + # NB: Suppress "Downloading/Downloaded" messages. + # See: https://stackoverflow.com/a/35653426/1207769 + export MAVEN_OPTS=-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -# Populate the settings.xml configuration. -mkdir -p "$HOME/.m2" -settingsFile="$HOME/.m2/settings.xml" -customSettings=.travis/settings.xml -test -f "$customSettings" && cp "$customSettings" "$settingsFile" || -cat >"$settingsFile" <"$settingsFile" < @@ -55,49 +58,50 @@ cat >"$settingsFile" < EOL -# Install GPG on OSX/macOS -if [ "$TRAVIS_OS_NAME" = osx ] -then - brew install gnupg2 -fi + # Install GPG on OSX/macOS + if [ "$TRAVIS_OS_NAME" = osx ] + then + brew install gnupg2 + fi -# Import the GPG signing key. -keyFile=.travis/signingkey.asc -key=$1 -iv=$2 -if [ "$key" -a "$iv" -a -f "$keyFile.enc" ] -then - # NB: Key and iv values were given as arguments. - echo "== Decrypting GPG keypair ==" - openssl aes-256-cbc -K "$key" -iv "$iv" -in "$keyFile.enc" -out "$keyFile" -d -fi -if [ "$TRAVIS_SECURE_ENV_VARS" = true \ - -a "$TRAVIS_PULL_REQUEST" = false \ - -a -f "$keyFile" ] -then - echo "== Importing GPG keypair ==" - gpg --batch --fast-import "$keyFile" -fi + # Import the GPG signing key. + keyFile=.travis/signingkey.asc + key=$1 + iv=$2 + if [ "$key" -a "$iv" -a -f "$keyFile.enc" ] + then + # NB: Key and iv values were given as arguments. + echo "== Decrypting GPG keypair ==" + openssl aes-256-cbc -K "$key" -iv "$iv" -in "$keyFile.enc" -out "$keyFile" -d + fi + if [ "$TRAVIS_SECURE_ENV_VARS" = true \ + -a "$TRAVIS_PULL_REQUEST" = false \ + -a -f "$keyFile" ] + then + echo "== Importing GPG keypair ==" + gpg --batch --fast-import "$keyFile" + fi -# Run the build. -if [ "$TRAVIS_SECURE_ENV_VARS" = true \ - -a "$TRAVIS_PULL_REQUEST" = false \ - -a "$TRAVIS_BRANCH" = master ] -then - echo "== Building and deploying master SNAPSHOT ==" - mvn -B -Pdeploy-to-imagej deploy - success=$? -elif [ "$TRAVIS_SECURE_ENV_VARS" = true \ - -a "$TRAVIS_PULL_REQUEST" = false \ - -a -f release.properties ] -then - echo "== Cutting and deploying release version ==" - mvn -B release:perform - success=$? -else - echo "== Building the artifact locally only ==" - mvn -B install javadoc:javadoc - success=$? + # Run the build. + if [ "$TRAVIS_SECURE_ENV_VARS" = true \ + -a "$TRAVIS_PULL_REQUEST" = false \ + -a "$TRAVIS_BRANCH" = master ] + then + echo "== Building and deploying master SNAPSHOT ==" + mvn -B -Pdeploy-to-imagej deploy + success=$? + elif [ "$TRAVIS_SECURE_ENV_VARS" = true \ + -a "$TRAVIS_PULL_REQUEST" = false \ + -a -f release.properties ] + then + echo "== Cutting and deploying release version ==" + mvn -B release:perform + success=$? + else + echo "== Building the artifact locally only ==" + mvn -B install javadoc:javadoc + success=$? + fi fi echo travis_fold:end:travis-build.sh exit $success From f43b3bd1a63a7dea9de6c6840b4469dec3247118 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 23 May 2018 09:57:03 -0500 Subject: [PATCH 096/429] travis-build.sh: fold the build around Maven steps When we add more build steps, we can have separate folds around them. --- travis-build.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/travis-build.sh b/travis-build.sh index d811a04..b16d903 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -4,13 +4,14 @@ # travis-build.sh - A script to build and/or release SciJava-based projects. # -echo travis_fold:start:travis-build.sh - dir="$(dirname "$0")" # Build Maven projects. if [ -f pom.xml ] then + echo travis_fold:start:travis-build.sh-maven + echo "= Maven build =" + echo echo "== Configuring Maven ==" # NB: Suppress "Downloading/Downloaded" messages. @@ -71,6 +72,7 @@ EOL if [ "$key" -a "$iv" -a -f "$keyFile.enc" ] then # NB: Key and iv values were given as arguments. + echo echo "== Decrypting GPG keypair ==" openssl aes-256-cbc -K "$key" -iv "$iv" -in "$keyFile.enc" -out "$keyFile" -d fi @@ -78,6 +80,7 @@ EOL -a "$TRAVIS_PULL_REQUEST" = false \ -a -f "$keyFile" ] then + echo echo "== Importing GPG keypair ==" gpg --batch --fast-import "$keyFile" fi @@ -87,6 +90,7 @@ EOL -a "$TRAVIS_PULL_REQUEST" = false \ -a "$TRAVIS_BRANCH" = master ] then + echo echo "== Building and deploying master SNAPSHOT ==" mvn -B -Pdeploy-to-imagej deploy success=$? @@ -94,14 +98,17 @@ EOL -a "$TRAVIS_PULL_REQUEST" = false \ -a -f release.properties ] then + echo echo "== Cutting and deploying release version ==" mvn -B release:perform success=$? else + echo echo "== Building the artifact locally only ==" mvn -B install javadoc:javadoc success=$? fi + echo travis_fold:end:travis-build.sh-maven fi -echo travis_fold:end:travis-build.sh + exit $success From 29b60a613c6351e71f8d05e69d492959073c5c59 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 23 May 2018 09:58:14 -0500 Subject: [PATCH 097/429] travis-build.sh: improve exit code checking This way, if a build step fails, we still continue, but record the bad exit code. And then ultimately we fail the build accordingly. --- travis-build.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/travis-build.sh b/travis-build.sh index b16d903..b14943b 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -6,6 +6,15 @@ dir="$(dirname "$0")" +success=0 +checkSuccess() { + # Log non-zero exit code. + test $1 -eq 0 || echo "==> FAILED: EXIT CODE $1" 1>&2 + + # Record the first non-zero exit code. + test $success -eq 0 && success=$1 +} + # Build Maven projects. if [ -f pom.xml ] then @@ -75,6 +84,7 @@ EOL echo echo "== Decrypting GPG keypair ==" openssl aes-256-cbc -K "$key" -iv "$iv" -in "$keyFile.enc" -out "$keyFile" -d + checkSuccess $? fi if [ "$TRAVIS_SECURE_ENV_VARS" = true \ -a "$TRAVIS_PULL_REQUEST" = false \ @@ -83,6 +93,7 @@ EOL echo echo "== Importing GPG keypair ==" gpg --batch --fast-import "$keyFile" + checkSuccess $? fi # Run the build. @@ -93,7 +104,7 @@ EOL echo echo "== Building and deploying master SNAPSHOT ==" mvn -B -Pdeploy-to-imagej deploy - success=$? + checkSuccess $? elif [ "$TRAVIS_SECURE_ENV_VARS" = true \ -a "$TRAVIS_PULL_REQUEST" = false \ -a -f release.properties ] @@ -101,12 +112,12 @@ EOL echo echo "== Cutting and deploying release version ==" mvn -B release:perform - success=$? + checkSuccess $? else echo echo "== Building the artifact locally only ==" mvn -B install javadoc:javadoc - success=$? + checkSuccess $? fi echo travis_fold:end:travis-build.sh-maven fi From 8794ec8b7ae77ef9f5904585de52ea6b08ab4d8c Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 23 May 2018 09:59:30 -0500 Subject: [PATCH 098/429] travis-build.sh: validate Jupyter notebooks This is a first attempt at Travis executing Jupyter notebooks, when the jupyter tool is present in the environment. If any notebooks have cell execution errors, the build should fail. This will be useful e.g. for imagej/tutorials#53. --- travis-build.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/travis-build.sh b/travis-build.sh index b14943b..b489f40 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -122,4 +122,29 @@ EOL echo travis_fold:end:travis-build.sh-maven fi +# Execute Jupyter notebooks. +if which jupyter >/dev/null 2>/dev/null +then + echo travis_fold:start:travis-build.sh-jupyter + echo "= Jupyter notebooks =" + # NB: This part is fiddly. We want to loop over files even with spaces, + # so we use the "find ... -print0 | while read $'\0' ..." idiom. + # However, that runs the piped expression in a subshell, which means + # that any updates to the success variable will not persist outside + # the loop. So we suppress all stdout inside the loop, echoing only + # the final value of success upon completion, and then capture the + # echoed value back into the parent shell's success variable. + success=$(find . -name '*.ipynb' -print0 | { + while read -d $'\0' nbf + do + echo 1>&2 + echo "== $nbf ==" 1>&2 + jupyter nbconvert --execute --stdout "$nbf" >/dev/null + checkSuccess $? + done + echo $success + }) + echo travis_fold:end:travis-build.sh-jupyter +fi + exit $success From 6bafa2e23ef973a45eb6a1039471e853ceceecd8 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 23 May 2018 12:24:57 -0500 Subject: [PATCH 099/429] travis-build.sh: use a cooler fold prefix This is the SciJava build script for Travis CI. The prefix "scijava-" is less likely to clash with others. --- travis-build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/travis-build.sh b/travis-build.sh index b489f40..0738d5e 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -18,7 +18,7 @@ checkSuccess() { # Build Maven projects. if [ -f pom.xml ] then - echo travis_fold:start:travis-build.sh-maven + echo travis_fold:start:scijava-maven echo "= Maven build =" echo echo "== Configuring Maven ==" @@ -119,13 +119,13 @@ EOL mvn -B install javadoc:javadoc checkSuccess $? fi - echo travis_fold:end:travis-build.sh-maven + echo travis_fold:end:scijava-maven fi # Execute Jupyter notebooks. if which jupyter >/dev/null 2>/dev/null then - echo travis_fold:start:travis-build.sh-jupyter + echo travis_fold:start:scijava-jupyter echo "= Jupyter notebooks =" # NB: This part is fiddly. We want to loop over files even with spaces, # so we use the "find ... -print0 | while read $'\0' ..." idiom. @@ -144,7 +144,7 @@ then done echo $success }) - echo travis_fold:end:travis-build.sh-jupyter + echo travis_fold:end:scijava-jupyter fi exit $success From e08a3bfce8c9b9671ad176544fa4b8022d6a3d27 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 23 May 2018 12:27:01 -0500 Subject: [PATCH 100/429] travis-build.sh: add conda installation + config That way, everyone does not need their own complex installation step for conda. Instead, the build script takes care of it, in the same way it takes care of configuring things like GPG for Maven. We do it in such a way that if the ~/miniconda installation folder is cached from previous builds, we can hopefully gracefully update it, rather than needing to download and install a new miniconda every time. --- travis-build.sh | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/travis-build.sh b/travis-build.sh index 0738d5e..47e3f97 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -122,6 +122,45 @@ EOL echo travis_fold:end:scijava-maven fi +# Configure conda environment, if one is needed. +if [ -f environment.yml ] +then + echo travis_fold:start:scijava-conda + echo "= Conda =" + + condaDir=$HOME/miniconda + if [ ! -d "$condaDir" ]; then + echo + echo "== Installing conda ==" + if [ "$TRAVIS_PYTHON_VERSION" = "2.7" ]; then + wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh + else + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + fi + bash miniconda.sh -b -p "$condaDir" + checkSuccess $? + fi + + echo + echo "== Updating conda ==" + export PATH="$condaDir/bin:$PATH" + hash -r && + conda config --set always_yes yes --set changeps1 no && + conda update -q conda && + conda info -a + checkSuccess $? + + echo + echo "== Configuring environment ==" + condaEnv=travis-scijava + test -d "$condaDir/envs/$condaEnv" && condaAction=update || condaAction=create + conda env "$condaAction" -n "$condaEnv" -f environment.yml && + source activate "$condaEnv" + checkSuccess $? + + echo travis_fold:end:scijava-conda +fi + # Execute Jupyter notebooks. if which jupyter >/dev/null 2>/dev/null then From ce7642bfdfa117070a084eba66bb41a15ceb90e0 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 23 May 2018 13:38:42 -0500 Subject: [PATCH 101/429] travis-build.sh: use conda without changing PATH For some reason, the PATH update does not seem to take: travis-build.sh: 148: travis-build.sh: conda: not found While we're at it, I removed the "hash -r" reset since now the location of conda-related executions is always given explicitly. Not sure this will actually fix behavior, but *fingers crossed*. --- travis-build.sh | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/travis-build.sh b/travis-build.sh index 47e3f97..2a570b7 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -126,7 +126,7 @@ fi if [ -f environment.yml ] then echo travis_fold:start:scijava-conda - echo "= Conda =" + echo "= Conda setup =" condaDir=$HOME/miniconda if [ ! -d "$condaDir" ]; then @@ -143,19 +143,17 @@ then echo echo "== Updating conda ==" - export PATH="$condaDir/bin:$PATH" - hash -r && - conda config --set always_yes yes --set changeps1 no && - conda update -q conda && - conda info -a + $condaDir/bin/conda config --set always_yes yes --set changeps1 no && + $condaDir/bin/conda update -q conda && + $condaDir/bin/conda info -a checkSuccess $? echo echo "== Configuring environment ==" condaEnv=travis-scijava test -d "$condaDir/envs/$condaEnv" && condaAction=update || condaAction=create - conda env "$condaAction" -n "$condaEnv" -f environment.yml && - source activate "$condaEnv" + $condaDir/bin/conda env "$condaAction" -n "$condaEnv" -f environment.yml && + source "$condaDir/bin/activate" "$condaEnv" checkSuccess $? echo travis_fold:end:scijava-conda From ab61df97d019fd1ddee3d38884e59c68400a44f4 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 23 May 2018 14:34:54 -0500 Subject: [PATCH 102/429] travis-build.sh: be more precise detecting conda We can look for a $HOME/miniconda/bin/conda program, not just the $HOME/miniconda directory. Maybe Travis already has a miniconda folder there for some reason? Otherwise I don't know why the conda installation is not triggering. Let's see if this change remedies that. --- travis-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis-build.sh b/travis-build.sh index 2a570b7..a98b0d0 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -129,7 +129,7 @@ then echo "= Conda setup =" condaDir=$HOME/miniconda - if [ ! -d "$condaDir" ]; then + if [ ! -f "$condaDir/bin/conda" ]; then echo echo "== Installing conda ==" if [ "$TRAVIS_PYTHON_VERSION" = "2.7" ]; then From 5c340d9199e9b44b0e160319bfc63f42a438bb46 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 23 May 2018 14:47:58 -0500 Subject: [PATCH 103/429] travis-build.sh: remove existing conda dir, if any It seems that Travis already has one! Somehow. Sometimes maybe. So let's be thorough and clean it out before proceeding. --- travis-build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/travis-build.sh b/travis-build.sh index a98b0d0..108e813 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -137,6 +137,7 @@ then else wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh fi + rm -rf "$condaDir" bash miniconda.sh -b -p "$condaDir" checkSuccess $? fi From f8ae524920fd0be726186886a01ab94eb57ae7fe Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 23 May 2018 15:19:21 -0500 Subject: [PATCH 104/429] travis-build.sh: eliminate "source" bashism It seems Travis does not run /bin/sh scripts with bash (which is fine). But the conda recommendation to use source assumes bash. Travis pukes on the conda "source activate" line with: travis-build.sh: 157: travis-build.sh: source: not found Let's try with the POSIX-friendly dot (.) instead, and see what happens! --- travis-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis-build.sh b/travis-build.sh index 108e813..bb528ea 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -154,7 +154,7 @@ then condaEnv=travis-scijava test -d "$condaDir/envs/$condaEnv" && condaAction=update || condaAction=create $condaDir/bin/conda env "$condaAction" -n "$condaEnv" -f environment.yml && - source "$condaDir/bin/activate" "$condaEnv" + . "$condaDir/bin/activate" "$condaEnv" checkSuccess $? echo travis_fold:end:scijava-conda From 7dce67162a4763c25f7cbbd70a35022ebca036f2 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 23 May 2018 15:54:29 -0500 Subject: [PATCH 105/429] travis-build.sh: give up and require bash Using dot apparently caused Travis to immediately terminate the script. We'll stop swimming upstream here and use bash like conda intended. --- travis-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travis-build.sh b/travis-build.sh index bb528ea..04f5cf3 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # travis-build.sh - A script to build and/or release SciJava-based projects. @@ -154,7 +154,7 @@ then condaEnv=travis-scijava test -d "$condaDir/envs/$condaEnv" && condaAction=update || condaAction=create $condaDir/bin/conda env "$condaAction" -n "$condaEnv" -f environment.yml && - . "$condaDir/bin/activate" "$condaEnv" + source "$condaDir/bin/activate" "$condaEnv" checkSuccess $? echo travis_fold:end:scijava-conda From 2b866b011b1dc1ec7c800ba20ee6c117e9ef488f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 25 May 2018 17:13:54 -0500 Subject: [PATCH 106/429] melting-pot: fail fast if git is too old The melting pot construction requires a git new enough to run: git clone --depth 1 Otherwise, the wrong revision (typically master) gets retrieved! :-O --- melting-pot.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/melting-pot.sh b/melting-pot.sh index 32f76ba..c0a4583 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -111,6 +111,8 @@ checkPrereqs() { verifyPrereqs() { checkPrereqs git mvn xmllint + git --version | grep -q 'git version 2' || + die "Please use git v2.x; older versions (<=1.7.9.5 at least) mishandle 'git clone --depth 1'" } parseArguments() { From 05d90209e9a111a1559139a21988ac85fbf1e0c3 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 25 May 2018 17:15:08 -0500 Subject: [PATCH 107/429] melting-pot: pass -B when retrieving dependencies Just to be safe. Otherwise, colors can mess up the grep, maybe? --- melting-pot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index c0a4583..b56110b 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -403,7 +403,7 @@ retrieveSource() { deps() { cd "$1" debug "mvn dependency:list" - local depList=$(mvn dependency:list) || + local depList="$(mvn -B dependency:list)" || die "Problem fetching dependencies!" 5 echo "$depList" | grep '^\[INFO\] [^ ]' | sed 's/\[INFO\] //' | sed 's/ *(optional) *$//' | sort From 6f1e13126c781e0fbe386863737e0be0e56cc914 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 25 May 2018 17:15:50 -0500 Subject: [PATCH 108/429] melting-pot: improve "skipping the build" message There is a shell script now to do it, so reference that. --- melting-pot.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index b56110b..697d9f5 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -596,8 +596,7 @@ meltDown() { echo "mvn $args test \$@" > build.sh if [ "$skipBuild" ] then - info "Skipping the build; the command would have been:" - info "mvn $args test" + info "Skipping the build; run build.sh to do it." else info "Building the project!" # NB: All code is fresh; no need to clean. From 01c606698600747480f35fdfe920175b6fd7ab98 Mon Sep 17 00:00:00 2001 From: Jan Eglinger Date: Thu, 12 Jul 2018 11:36:20 +0200 Subject: [PATCH 109/429] travis-build.sh: Use conda activate Newer versions of conda support activating an environment with `conda activate $env`, so let's use that recommended syntax instead of `source activate $env`. This hopefully fixes the Travis build failure here: https://travis-ci.org/imagej/tutorials/builds/388369917 --- travis-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis-build.sh b/travis-build.sh index 04f5cf3..0f4874c 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -154,7 +154,7 @@ then condaEnv=travis-scijava test -d "$condaDir/envs/$condaEnv" && condaAction=update || condaAction=create $condaDir/bin/conda env "$condaAction" -n "$condaEnv" -f environment.yml && - source "$condaDir/bin/activate" "$condaEnv" + $condaDir/bin/conda activate "$condaEnv" checkSuccess $? echo travis_fold:end:scijava-conda From a796a05057da7b608081d5004c2cae2649ef309b Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 27 Jul 2018 11:26:10 -0500 Subject: [PATCH 110/429] travis-javadoc: remove superfluous doclint flag In newer pom-scijava-base releases, the doclint flag is now specified in the build-javadoc profile, where it should have been all along. This matters because the doclint flag changed in newer versions of the maven-javadoc-plugin, such that the old invocation no longer works. --- travis-javadoc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis-javadoc.sh b/travis-javadoc.sh index f107590..ba2e77a 100644 --- a/travis-javadoc.sh +++ b/travis-javadoc.sh @@ -30,7 +30,7 @@ then mvn -B dependency:tree && # Build the javadocs. - mvn -B -q -Dadditionalparam=-Xdoclint:none -Pbuild-javadoc && + mvn -B -q -Pbuild-javadoc && test -d target/apidocs && # Strip out date stamps, to avoid spurious changes being committed. sed -i'' -e '/\(\n\t\tdeploy-to-imagej<\/releaseProfiles>\1/igs' "$tmpFile" - update pom.xml 'POM: deploy releases to the ImageJ repository' + perl -0777 -i -pe 's/(\n\t<\/properties>\n)/\n\n\t\t\n\t\tdeploy-to-scijava<\/releaseProfiles>\1/igs' "$tmpFile" + update pom.xml 'POM: deploy releases to the SciJava repository' fi # update the README From 66c777b56fa615fa84b56ac9d1e820c4ffdcc807 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 30 Jun 2019 22:07:08 +0200 Subject: [PATCH 145/429] melting-pot: generate another build script layer This layer builds all the modules one by one, all pinned to the same versions, but all separately, rather than in one multi-module build. This means that each build is more isolated and standalone. A succinct summary of each component's success or failure is now emitted, so the situation can be perused more quickly, rather than needing to wade through a giant build log on the console for each component. Each build is now logged to its own build.log file. It also avoids a problem we encountered while attempting to melt the Fiji project recently: the imagej-scripting component has failing tests when they run as part of a multi-module build including imglib2, but has passing tests otherwise. Here is a transcript of the first failure: [INFO] Running net.imagej.scripting.ImageJ2ScriptTest log4j:WARN No appenders could be found for logger (org.bushe.swing.event.EventService). log4j:WARN Please initialize the log4j system properly. log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info. [ERROR] null Traceback (most recent call last): File "/Users/curtis/code/fiji/fiji/melting-pot/net.imagej/imagej-scripting/target/classes/script_templates/ImageJ2/Rotate_Stack.py", line 17, in from net.imglib2.view import Views java.lang.NoClassDefFoundError: net/imglib2/view (wrong name: net/imglib2/View) at java.lang.ClassLoader.defineClass1(Native Method) at java.lang.ClassLoader.defineClass(ClassLoader.java:763) at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) at java.net.URLClassLoader.defineClass(URLClassLoader.java:468) at java.net.URLClassLoader.access$100(URLClassLoader.java:74) at java.net.URLClassLoader$1.run(URLClassLoader.java:369) at java.net.URLClassLoader$1.run(URLClassLoader.java:363) at java.security.AccessController.doPrivileged(Native Method) at java.net.URLClassLoader.findClass(URLClassLoader.java:362) at java.lang.ClassLoader.loadClass(ClassLoader.java:424) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349) at java.lang.ClassLoader.loadClass(ClassLoader.java:411) at java.lang.ClassLoader.loadClass(ClassLoader.java:357) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:348) at org.python.core.Py.loadAndInitClass(Py.java:1107) at org.python.core.Py.findClassInternal(Py.java:1042) at org.python.core.Py.findClassEx(Py.java:1093) at org.python.core.packagecache.SysPackageManager.findClass(SysPackageManager.java:142) at org.python.core.packagecache.PackageManager.findClass(PackageManager.java:33) at org.python.core.packagecache.SysPackageManager.findClass(SysPackageManager.java:130) at org.python.core.PyJavaPackage.__findattr_ex__(PyJavaPackage.java:134) at org.python.core.PyObject.__findattr__(PyObject.java:965) at org.python.core.PyObject.impAttr(PyObject.java:1103) at org.python.core.imp.import_next(imp.java:840) at org.python.core.imp.import_logic(imp.java:905) at org.python.core.imp.import_module_level(imp.java:970) at org.python.core.imp.importName(imp.java:1057) at org.python.core.ImportFunction.__call__(__builtin__.java:1280) at org.python.core.PyObject.__call__(PyObject.java:450) at org.python.core.__builtin__.__import__(__builtin__.java:1232) at org.python.core.imp.importFromAs(imp.java:1149) at org.python.core.imp.importFrom(imp.java:1124) at org.python.pycode._pyx0.f$0(/Users/curtis/code/fiji/fiji/melting-pot/net.imagej/imagej-scripting/target/classes/script_templates/ImageJ2/Rotate_Stack.py:89) at org.python.pycode._pyx0.call_function(/Users/curtis/code/fiji/fiji/melting-pot/net.imagej/imagej-scripting/target/classes/script_templates/ImageJ2/Rotate_Stack.py) at org.python.core.PyTableCode.call(PyTableCode.java:171) at org.python.core.PyCode.call(PyCode.java:18) at org.python.core.Py.runCode(Py.java:1614) at org.python.core.__builtin__.eval(__builtin__.java:497) at org.python.core.__builtin__.eval(__builtin__.java:501) at org.python.util.PythonInterpreter.eval(PythonInterpreter.java:259) at org.python.jsr223.PyScriptEngine.eval(PyScriptEngine.java:57) at org.python.jsr223.PyScriptEngine.eval(PyScriptEngine.java:31) at javax.script.AbstractScriptEngine.eval(AbstractScriptEngine.java:264) at org.scijava.script.ScriptModule.run(ScriptModule.java:160) at org.scijava.module.ModuleRunner.run(ModuleRunner.java:168) at org.scijava.module.ModuleRunner.call(ModuleRunner.java:127) at org.scijava.module.ModuleRunner.call(ModuleRunner.java:66) at org.scijava.thread.DefaultThreadService$3.call(DefaultThreadService.java:238) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) java.lang.NoClassDefFoundError: java.lang.NoClassDefFoundError: net/imglib2/view (wrong name: net/imglib2/View) at org.python.core.Py.JavaError(Py.java:552) at org.python.core.Py.findClassEx(Py.java:1099) at org.python.core.packagecache.SysPackageManager.findClass(SysPackageManager.java:142) at org.python.core.packagecache.PackageManager.findClass(PackageManager.java:33) at org.python.core.packagecache.SysPackageManager.findClass(SysPackageManager.java:130) at org.python.core.PyJavaPackage.__findattr_ex__(PyJavaPackage.java:134) at org.python.core.PyObject.__findattr__(PyObject.java:965) at org.python.core.PyObject.impAttr(PyObject.java:1103) at org.python.core.imp.import_next(imp.java:840) at org.python.core.imp.import_logic(imp.java:905) at org.python.core.imp.import_module_level(imp.java:970) at org.python.core.imp.importName(imp.java:1057) at org.python.core.ImportFunction.__call__(__builtin__.java:1280) at org.python.core.PyObject.__call__(PyObject.java:450) at org.python.core.__builtin__.__import__(__builtin__.java:1232) at org.python.core.imp.importFromAs(imp.java:1149) at org.python.core.imp.importFrom(imp.java:1124) at org.python.pycode._pyx0.f$0(/Users/curtis/code/fiji/fiji/melting-pot/net.imagej/imagej-scripting/target/classes/script_templates/ImageJ2/Rotate_Stack.py:89) at org.python.pycode._pyx0.call_function(/Users/curtis/code/fiji/fiji/melting-pot/net.imagej/imagej-scripting/target/classes/script_templates/ImageJ2/Rotate_Stack.py) at org.python.core.PyTableCode.call(PyTableCode.java:171) at org.python.core.PyCode.call(PyCode.java:18) at org.python.core.Py.runCode(Py.java:1614) at org.python.core.__builtin__.eval(__builtin__.java:497) at org.python.core.__builtin__.eval(__builtin__.java:501) at org.python.util.PythonInterpreter.eval(PythonInterpreter.java:259) at org.python.jsr223.PyScriptEngine.eval(PyScriptEngine.java:57) at org.python.jsr223.PyScriptEngine.eval(PyScriptEngine.java:31) at javax.script.AbstractScriptEngine.eval(AbstractScriptEngine.java:264) at org.scijava.script.ScriptModule.run(ScriptModule.java:160) at org.scijava.module.ModuleRunner.run(ModuleRunner.java:168) at org.scijava.module.ModuleRunner.call(ModuleRunner.java:127) at org.scijava.module.ModuleRunner.call(ModuleRunner.java:66) at org.scijava.thread.DefaultThreadService$3.call(DefaultThreadService.java:238) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.NoClassDefFoundError: net/imglib2/view (wrong name: net/imglib2/View) at java.lang.ClassLoader.defineClass1(Native Method) at java.lang.ClassLoader.defineClass(ClassLoader.java:763) at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) at java.net.URLClassLoader.defineClass(URLClassLoader.java:468) at java.net.URLClassLoader.access$100(URLClassLoader.java:74) at java.net.URLClassLoader$1.run(URLClassLoader.java:369) at java.net.URLClassLoader$1.run(URLClassLoader.java:363) at java.security.AccessController.doPrivileged(Native Method) at java.net.URLClassLoader.findClass(URLClassLoader.java:362) at java.lang.ClassLoader.loadClass(ClassLoader.java:424) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349) at java.lang.ClassLoader.loadClass(ClassLoader.java:411) at java.lang.ClassLoader.loadClass(ClassLoader.java:357) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:348) at org.python.core.Py.loadAndInitClass(Py.java:1107) at org.python.core.Py.findClassInternal(Py.java:1042) at org.python.core.Py.findClassEx(Py.java:1093) ... 35 more The relevant issue in the Jython bug tracker is probably: https://bugs.jython.org/issue2666 The bug may be fixed on the jython 2.7.2 development branch, although I did not test as of this writing. Unfortunately, I was not able to concoct an MCVE; nothing I tried was able to reproduce the problem except for a melting-pot multi-module build that includes imglib2 and imagej-scripting. When running a multi-module build not including imglib2, the apparent difference is that the imagej-scripting project's classpath includes the imglib2 JAR, rather than the linked imglib2 module's target/classes folder. But my naive attempts to create an MCVE with an analogous classpath structure did not yield failure to import classes within Jython. --- melting-pot.sh | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 32de245..a8c50a9 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -592,16 +592,28 @@ meltDown() { info "Generating aggregator POM" generatePOM - # Build everything. + # Generate build scripts. echo "mvn $args \\\\\n test \$@" > build.sh + echo '#!/bin/sh +trap "exit" INT +echo "Melting the pot..." +dir=$(pwd) +for f in */* +do ( + cd "$f" + sh "$dir/build.sh" > build.log 2>&1 && + echo "[SUCCESS] $f" || + echo "[FAILURE] $f" +) done' > melt.sh + + # Build everything. if [ "$skipBuild" ] then - info "Skipping the build; run build.sh to do it." + info "Skipping the build; run melt.sh to do it." else info "Building the project!" # NB: All code is fresh; no need to clean. - debug "mvn $args test" - sh build.sh + sh melt.sh fi info "$1: complete" From 6173aeca4018e88a60430e79cdacef7f4eab12a7 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 1 Jul 2019 08:23:59 +0200 Subject: [PATCH 146/429] melting-pot: improve tag detection --- melting-pot.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index a8c50a9..a5770af 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -384,7 +384,15 @@ scmURL() { # Gets the SCM tag for the given GAV. scmTag() { - echo "$(artifactId "$1")-$(version "$1")" + local tag=$(pomValue "$1" project scm tag) + if [ -z "$tag" -o "$tag" = "HEAD" ] + then + # The value was not set properly, + # so we guess at the tag naming scheme. + echo "$(artifactId "$1")-$(version "$1")" + else + echo "$tag" + fi } # Fetches the source code for the given GAV. Returns the directory. @@ -395,7 +403,8 @@ retrieveSource() { test "$2" && scmBranch="$2" || scmBranch="$(scmTag "$1")" local dir="$(groupId "$1")/$(artifactId "$1")" debug "git clone \"$scmURL\" --branch \"$scmBranch\" --depth 1 \"$dir\"" - git clone "$scmURL" --branch "$scmBranch" --depth 1 "$dir" 2> /dev/null + git clone "$scmURL" --branch "$scmBranch" --depth 1 "$dir" 2> /dev/null || + die "Could not fetch project source for $1" 3 echo "$dir" } @@ -542,8 +551,7 @@ meltDown() { else # Treat specified project as a GAV. info "$1: fetching project source" - local dir="$(retrieveSource "$1" "$branch")" - test -d "$dir" || die "Could not fetch project source" 3 + retrieveSource "$1" "$branch" fi # Get the project dependencies. From 0fbeea89d2c05562bf9ecb5f2911d794689d3467 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 1 Jul 2019 08:40:40 +0200 Subject: [PATCH 147/429] melting-pot: make melt.sh matching pom.xml For multi-module components, */* is not the same as explicitly enumerating the child projects. See mpicbg:mpicbg and mpicbg:mpicbg_ for an example. --- melting-pot.sh | 47 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index a5770af..c4c87ca 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -533,6 +533,35 @@ generatePOM() { echo '' >> pom.xml } +# Generates melt.sh script for all modules in the current directory. +generateScript() { + echo '#!/bin/sh' > melt.sh + echo 'trap "exit" INT' >> melt.sh + echo 'echo "Melting the pot..."' >> melt.sh + echo 'dir=$(pwd)' >> melt.sh + echo 'for f in \' >> melt.sh + local dir + for dir in */* + do + if [ "$(isProject "$dir")" ] + then + echo " $dir \\" >> melt.sh + else + # Check for a child component of a multi-module project. + local childDir="$dir/$(basename "$dir")" + test "$(isProject "$childDir")" && + echo " $childDir \\" >> melt.sh + fi + done + echo >> melt.sh + echo 'do (' >> melt.sh + echo ' cd "$f"' >> melt.sh + echo ' sh "$dir/build.sh" > build.log 2>&1 &&' >> melt.sh + echo ' echo "[SUCCESS] $f" ||' >> melt.sh + echo ' echo "[FAILURE] $f"' >> melt.sh + echo ') done' >> melt.sh +} + # Creates and tests an appropriate multi-module reactor for the given project. # All relevant dependencies which match the inclusion criteria are linked into # the multi-module build, with each changed GAV overridding the originally @@ -596,23 +625,11 @@ meltDown() { # Prune the build, if applicable. test "$prune" && pruneReactor - # Generate the aggregator POM. - info "Generating aggregator POM" - generatePOM - # Generate build scripts. + info "Generating build scripts" + generatePOM echo "mvn $args \\\\\n test \$@" > build.sh - echo '#!/bin/sh -trap "exit" INT -echo "Melting the pot..." -dir=$(pwd) -for f in */* -do ( - cd "$f" - sh "$dir/build.sh" > build.log 2>&1 && - echo "[SUCCESS] $f" || - echo "[FAILURE] $f" -) done' > melt.sh + generateScript # Build everything. if [ "$skipBuild" ] From f245a5fd46660d4ecd5919157dca3f662ba957bc Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 19 Aug 2019 11:33:31 -0500 Subject: [PATCH 148/429] travis-build.sh: centralize deploy detection We only want to attempt deployment if we are building the official repository, not a fork, with deployment credentials available to the environment. Rather than testing for the various TRAVIS_* variables in multiple places, we now check everything once and set a single variable. --- travis-build.sh | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/travis-build.sh b/travis-build.sh index f94080e..c99384b 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -88,6 +88,27 @@ EOL EOL fi + # Determine whether deploying will be possible. + ciURL=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${project.ciManagement.url}' --non-recursive validate exec:exec 2>/dev/null) + ciRepo=${ciURL##*/} + ciPrefix=${ciURL%/*} + ciOrg=${ciPrefix##*/} + deployOK= + if [ "$TRAVIS_SECURE_ENV_VARS" != true ] + then + echo "No deploy -- secure environment variables not available" + elif [ "$TRAVIS_PULL_REQUEST" != false ] + then + echo "No deploy -- pull request detected" + elif [ "$TRAVIS_REPO_SLUG" != "$ciOrg/$ciRepo" ] + then + echo "No deploy -- repository fork: $TRAVIS_REPO_SLUG != $ciOrg/$ciRepo" + # TODO: Detect travis-ci.org versus travis-ci.com? + else + echo "All checks passed for artifact deployment" + deployOK=1 + fi + # Install GPG on OSX/macOS if [ "$TRAVIS_OS_NAME" = osx ] then @@ -106,9 +127,7 @@ EOL openssl aes-256-cbc -K "$key" -iv "$iv" -in "$keyFile.enc" -out "$keyFile" -d checkSuccess $? fi - if [ "$TRAVIS_SECURE_ENV_VARS" = true \ - -a "$TRAVIS_PULL_REQUEST" = false \ - -a -f "$keyFile" ] + if [ "$deployOK" -a -f "$keyFile" ] then echo echo "== Importing GPG keypair ==" @@ -117,17 +136,13 @@ EOL fi # Run the build. - if [ "$TRAVIS_SECURE_ENV_VARS" = true \ - -a "$TRAVIS_PULL_REQUEST" = false \ - -a "$TRAVIS_BRANCH" = master ] + if [ "$deployOK" -a "$TRAVIS_BRANCH" = master ] then echo echo "== Building and deploying master SNAPSHOT ==" mvn -B -Pdeploy-to-scijava deploy checkSuccess $? - elif [ "$TRAVIS_SECURE_ENV_VARS" = true \ - -a "$TRAVIS_PULL_REQUEST" = false \ - -a -f release.properties ] + elif [ "$deployOK" -a -f release.properties ] then echo echo "== Cutting and deploying release version ==" From c0308d2ca5ae14b7fa66315658e3a2fd282d0a8a Mon Sep 17 00:00:00 2001 From: Wei Ouyang Date: Sat, 14 Dec 2019 18:57:18 +0100 Subject: [PATCH 149/429] only include runtime dependencies --- melting-pot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index c4c87ca..cf76e2e 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -412,7 +412,7 @@ retrieveSource() { deps() { cd "$1" debug "mvn dependency:list" - local depList="$(mvn -B dependency:list)" || + local depList="$(mvn -DincludeScope=runtime -B dependency:list)" || die "Problem fetching dependencies!" 5 echo "$depList" | grep '^\[INFO\] [^ ]' | sed 's/\[INFO\] //' | sed 's/ *(optional) *$//' | sort From a441406c55729c1cf46923937a1a424ba29023ed Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 7 Feb 2020 16:18:16 -0600 Subject: [PATCH 150/429] check-branch: use expr instead of let This avoids problems with dash on Ubuntu. --- check-branch.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check-branch.sh b/check-branch.sh index 1da4304..8496e55 100755 --- a/check-branch.sh +++ b/check-branch.sh @@ -20,9 +20,9 @@ do start=$(date +%s) mvn clean verify > "$filename" 2>&1 && result=SUCCESS || result=FAILURE end=$(date +%s) - let time="end-start" + time=$(expr "$end" - "$start") echo "$prefix $commit $result $time" - let count="count+1" + count=$(expr "$count" + 1) done git checkout "$branch" > /dev/null 2>&1 From 23bd0cfa85fa014b23d4919a98d6329b7c3b25f8 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 28 Mar 2020 17:59:39 +0100 Subject: [PATCH 151/429] From version 1.8.12 "Travis encrypt" asks for confirmation From release notes: Ask for confirmation when encrypt and encrypt-file commands receive -a, --add flag --- travisify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travisify.sh b/travisify.sh index 1a85afd..1cfd228 100755 --- a/travisify.sh +++ b/travisify.sh @@ -201,7 +201,7 @@ EOL '#'*) continue;; esac info "Encrypting ${p%%=*}" - $EXEC travis encrypt --$mode "$p" --add env.global --repo "$repoSlug" + yes | $EXEC travis encrypt --$mode "$p" --add env.global --repo "$repoSlug" test $? -eq 0 || die "Failed to encrypt variable '$p'" done <"$varsFile" $EXEC git commit "$travisConfig" -m "Travis: add encrypted environment variables" From 8c7d0f07810484adf357fae2070c9f66f0db1b61 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sat, 18 Apr 2020 13:52:44 -0500 Subject: [PATCH 152/429] travis-build: only mirror Central, not everything See discussion at: https://gitter.im/scenerygraphics/SciView?at=5e9a0e99d240da24337d7aba --- travis-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travis-build.sh b/travis-build.sh index c99384b..ded0dce 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -55,7 +55,7 @@ then EOL - # NB: Use maven.scijava.org as sole mirror if defined in . + # NB: Use maven.scijava.org instead of Central if defined in repositories. # This hopefully avoids intermittent "ReasonPhrase:Forbidden" errors # when the Travis build pings Maven Central; see travis-ci/travis-ci#6593. grep -A 2 '' pom.xml | grep -q 'maven.scijava.org' && @@ -65,7 +65,7 @@ EOL scijava-mirror SciJava mirror https://maven.scijava.org/content/groups/public/ - * + central EOL From a5795b651fce7b646025c9a124c263a30219a5c1 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 12 May 2020 16:46:57 -0500 Subject: [PATCH 153/429] melting-pot: skip non-main classifiers --- melting-pot.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/melting-pot.sh b/melting-pot.sh index c4c87ca..c1ad19f 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -295,6 +295,34 @@ version() { esac } +classifier() { + local result="${1#*:}" # strip groupId + case "$result" in + *:*) + result="${result#*:}" # strip artifactId + case "$result" in + *:*:*:*) + # G:A:P:C:V:S + result="${result#*:}" # strip packaging + ;; + *:*:*) + # G:A:P:V:S + result="" + ;; + *:*) + # G:A:V:C + result="${result#*:}" # strip version + ;; + *) + # G:A:V + result="" + ;; + esac + echo "${result%%:*}" + ;; + esac +} + # Converts the given GAV into a path in the local repository cache. repoPath() { local gPath="$(echo "$(groupId "$1")" | tr :. /)" @@ -598,6 +626,8 @@ meltDown() { local g="$(groupId "$dep")" local a="$(artifactId "$dep")" local v="$(version "$dep")" + local c="$(classifier "$dep")" + test -z "$c" || continue # skip secondary artifacts local gav="$g:$a:$v" test -z "$(isChanged "$gav")" && From 01306e635dd9da03ddb7aa5b4bf2e9cc47e2e4a2 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 19 May 2020 20:47:42 -0500 Subject: [PATCH 154/429] melting-pot: fail if melt.sh fails And make melt.sh use an exit code equal to the number of failing component builds (max 255). --- melting-pot.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index c1ad19f..18d2543 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -582,12 +582,17 @@ generateScript() { fi done echo >> melt.sh + echo 'local failCount=0' echo 'do (' >> melt.sh - echo ' cd "$f"' >> melt.sh - echo ' sh "$dir/build.sh" > build.log 2>&1 &&' >> melt.sh - echo ' echo "[SUCCESS] $f" ||' >> melt.sh - echo ' echo "[FAILURE] $f"' >> melt.sh + echo ' cd "$f"' >> melt.sh + echo ' sh "$dir/build.sh" > build.log 2>&1 &&' >> melt.sh + echo ' echo "[SUCCESS] $f" || {' >> melt.sh + echo ' echo "[FAILURE] $f"' >> melt.sh + echo ' failCount=$((failCount+1))' >> melt.sh + echo ' }' >> melt.sh echo ') done' >> melt.sh + echo 'test "$failCount" -gt 255 && failCount=255' >> melt.sh + echo 'exit "$failCount"' >> melt.sh } # Creates and tests an appropriate multi-module reactor for the given project. @@ -668,7 +673,7 @@ meltDown() { else info "Building the project!" # NB: All code is fresh; no need to clean. - sh melt.sh + sh melt.sh || die "Melt failed" 13 fi info "$1: complete" From 29b8c94517157f43209210051c9a81bad97fc9ae Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 19 May 2020 21:02:15 -0500 Subject: [PATCH 155/429] melting-pot: add exit code for bad git version --- melting-pot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index 18d2543..62c9e7c 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -112,7 +112,7 @@ checkPrereqs() { verifyPrereqs() { checkPrereqs git mvn xmllint git --version | grep -q 'git version 2' || - die "Please use git v2.x; older versions (<=1.7.9.5 at least) mishandle 'git clone --depth 1'" + die "Please use git v2.x; older versions (<=1.7.9.5 at least) mishandle 'git clone --depth 1'" 254 } parseArguments() { From c4507a31b2a8e6fbab870a9824eeef83bc952cc2 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 19 May 2020 21:15:59 -0500 Subject: [PATCH 156/429] melting-pot: fix bug in failure counting --- melting-pot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index 62c9e7c..1c4fe7e 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -567,6 +567,7 @@ generateScript() { echo 'trap "exit" INT' >> melt.sh echo 'echo "Melting the pot..."' >> melt.sh echo 'dir=$(pwd)' >> melt.sh + echo 'failCount=0' >> melt.sh echo 'for f in \' >> melt.sh local dir for dir in */* @@ -582,7 +583,6 @@ generateScript() { fi done echo >> melt.sh - echo 'local failCount=0' echo 'do (' >> melt.sh echo ' cd "$f"' >> melt.sh echo ' sh "$dir/build.sh" > build.log 2>&1 &&' >> melt.sh From dbff71222e8dd79891c3a01ca23e30dc42d97ae4 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sat, 23 May 2020 15:11:13 -0500 Subject: [PATCH 157/429] melting-pot: trim prefix length of toplevel infos --- melting-pot.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 1c4fe7e..dcf18fa 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -606,25 +606,25 @@ meltDown() { # Use local directory for the specified project. test -d "$1" || die "No such directory: $1" 11 test -f "$1/pom.xml" || die "Not a Maven project: $1" 12 - info "$1: local Maven project" + info "Local Maven project: $1" mkdir -p "LOCAL" local dir="LOCAL/PROJECT" ln -s "$1" "$dir" else # Treat specified project as a GAV. - info "$1: fetching project source" + info "Fetching project source" retrieveSource "$1" "$branch" fi # Get the project dependencies. - info "$1: determining project dependencies" + info "Determining project dependencies" local deps="$(deps "$dir")" test "$deps" || die "Cannot glean project dependencies" 7 local args="-Denforcer.skip" # Process the dependencies. - info "$1: processing project dependencies" + info "Processing project dependencies" local dep for dep in $deps do @@ -640,13 +640,13 @@ meltDown() { if [ "$(isIncluded "$gav")" ] then - info "$1: $a: fetching component source" + info "$a: fetching component source" dir="$(retrieveSource "$gav")" fi done # Override versions of changed GAVs. - info "$1: processing changed components" + info "Processing changed components" local TLS=, local gav for gav in $changes @@ -676,7 +676,7 @@ meltDown() { sh melt.sh || die "Melt failed" 13 fi - info "$1: complete" + info "Melt complete: $1" } # -- Main -- From d207db9a94628a70f0503c6d0bdae33bc0ebb0f2 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sat, 23 May 2020 15:20:56 -0500 Subject: [PATCH 158/429] melting-pot: be smarter about faulty SCM tags When the tag is HEAD, we need to guess. But we can ask the git remote for its tags, and use the best matching one, if any. --- melting-pot.sh | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index dcf18fa..897ddec 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -86,6 +86,10 @@ info() { stderr "[INFO] $@" } +warn() { + stderr "[WARNING] $@" +} + error() { stderr "[ERROR] $@" } @@ -416,8 +420,22 @@ scmTag() { if [ -z "$tag" -o "$tag" = "HEAD" ] then # The value was not set properly, - # so we guess at the tag naming scheme. - echo "$(artifactId "$1")-$(version "$1")" + # so we try to guess the tag naming scheme. :-/ + warn "$1: improper scm tag value; scanning remote tags..." + local a=$(artifactId "$1") + local v=$(version "$1") + local scmURL="$(scmURL "$1")" + local allTags=$(git ls-remote --tags "$scmURL" | sed 's/.*refs\/tags\///' || + error "$1: Invalid scm url: $scmURL") + for tag in "$a-$v" "$v" "v$v" + do + echo "$allTags" | grep -q "^$tag$" && { + info "$1: inferred tag: $tag" + echo "$tag" + return + } + done + error "$1: inscrutable tag scheme" else echo "$tag" fi From a97c77e7a241337fadc1a58c3f61b1c0b5ba3a4b Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sat, 23 May 2020 16:39:03 -0500 Subject: [PATCH 159/429] melting-pot: check that cloned source is correct --- melting-pot.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/melting-pot.sh b/melting-pot.sh index 897ddec..45c3fd6 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -451,6 +451,13 @@ retrieveSource() { debug "git clone \"$scmURL\" --branch \"$scmBranch\" --depth 1 \"$dir\"" git clone "$scmURL" --branch "$scmBranch" --depth 1 "$dir" 2> /dev/null || die "Could not fetch project source for $1" 3 + + # Now verify that the cloned pom.xml contains the expected version! + local expectedVersion=$(version "$1") + local actualVersion=$(xpath "$dir/pom.xml" project version) + test "$expectedVersion" = "$actualVersion" || + die "POM for $1 contains wrong version: $actualVersion" 14 + echo "$dir" } From 0de65bacd17a846452c536061d51c2035dfcc4b5 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 24 May 2020 18:43:41 -0500 Subject: [PATCH 160/429] release-version: improve things! * Add a short header comment. * Flesh out the usage guide. * Remove the not-too-useful-anymore --third-party option. * Put all the sanity checks together. * Comment on each block of code more clearly. * Inline short functions only used once, for flow. * Allow version argument to be specified before flags. --- release-version.sh | 135 +++++++++++++++++++++++++++------------------ 1 file changed, 80 insertions(+), 55 deletions(-) diff --git a/release-version.sh b/release-version.sh index 3b9fe7b..5226dfe 100755 --- a/release-version.sh +++ b/release-version.sh @@ -1,38 +1,27 @@ #!/bin/sh +# ============================================================================ +# release-version.sh +# ============================================================================ +# Releases a new version of a component extending the pom-scijava parent. +# +# Authors: Johannes Schindelin & Curtis Rueden +# ============================================================================ + +# -- Functions -- + die () { echo "$*" >&2 exit 1 } -MAVEN_HELPER="$(cd "$(dirname "$0")" && pwd)/maven-helper.sh" - -maven_helper () { - sh -$- "$MAVEN_HELPER" "$@" || - die "Could not find maven-helper in '$MAVEN_HELPER'" -} - -VALID_SEMVER_BUMP="$(cd "$(dirname "$0")" && pwd)/valid-semver-bump.sh" - -valid_semver_bump () { - test -f "$VALID_SEMVER_BUMP" || - die "Could not find valid-semver-bump in '$VALID_SEMVER_BUMP'" - sh -$- "$VALID_SEMVER_BUMP" "$@" || die -} - -verify_git_settings () { - if [ ! "$SKIP_PUSH" ] - then - push=$(git remote -v | grep origin | grep '(push)') - test "$push" || die 'No push URL found for remote origin' - echo "$push" | grep -q 'git:/' && die 'Remote origin is read-only' - fi -} +# -- Constants and settings -- SCIJAVA_BASE_REPOSITORY=-DaltDeploymentRepository=scijava.releases::default::dav:https://maven.scijava.org/content/repositories SCIJAVA_RELEASES_REPOSITORY=$SCIJAVA_BASE_REPOSITORY/releases SCIJAVA_THIRDPARTY_REPOSITORY=$SCIJAVA_BASE_REPOSITORY/thirdparty +# Parse command line options. BATCH_MODE=--batch-mode SKIP_VERSION_CHECK= SKIP_PUSH= @@ -43,6 +32,8 @@ EXTRA_ARGS= ALT_REPOSITORY= PROFILE=-Pdeploy-to-scijava DRY_RUN= +USAGE= +VERSION= while test $# -gt 0 do case "$1" in @@ -64,60 +55,95 @@ do ALT_REPOSITORY=$SCIJAVA_THIRDPARTY_REPOSITORY;; --alt-repository=*|--alt-deployment-repository=*) ALT_REPOSITORY="${1#--*=}";; - --thirdparty=scijava) - BATCH_MODE= - SKIP_PUSH=t - ALT_REPOSITORY=$SCIJAVA_THIRDPARTY_REPOSITORY;; --skip-gpg) SKIP_GPG=t EXTRA_ARGS="$EXTRA_ARGS -Dgpg.skip=true";; - -*) echo "Unknown option: $1" >&2; break;; - *) break;; + --help) + USAGE=t + break;; + -*) + echo "Unknown option: $1" >&2 + USAGE=t + break;; + *) + test -z "$VERSION" || { + echo "Extraneous argument: $1" >&2 + USAGE=t + break + } + VERSION=$1;; esac shift done -verify_git_settings +test "$USAGE" && +die "Usage: $0 [options] [] -devVersion=$(mvn -N -Dexec.executable='echo' -Dexec.args='${project.version}' exec:exec -q) -pomVersion=${devVersion%-SNAPSHOT} -test $# = 1 || test ! -t 0 || { - version=$pomVersion - printf 'Version? [%s]: ' "$version" - read line - test -z "$line" || version="$line" - set "$version" -} +Where is the version to release. If omitted, it will prompt you. + +Options include: + --dry-run - Simulate the release without actually doing it. + --skip-version-check - Violate SemVer version numbering intentionally. + --skip-push - Do not push to the remote git repository. + --dev-version= - Specify next development version explicitly; + e.g.: if you release 2.0.0-beta-1, by default + Maven will set the next development version at + 2.0.0-beta-2-SNAPSHOT, but maybe you want to + set it to 2.0.0-SNAPSHOT instead. + --alt-repository= - Deploy release to a different remote repository. + --skip-gpg - Do not perform GPG signing of artifacts. +" + +# -- Sanity checks -- + +# Check that we have push rights to the repository. +if [ ! "$SKIP_PUSH" ] +then + push=$(git remote -v | grep origin | grep '(push)') + test "$push" || die 'No push URL found for remote origin' + echo "$push" | grep -q 'git:/' && die 'Remote origin is read-only' +fi -test $# = 1 && test "a$1" = "a${1#-}" || -die "Usage: $0 [--no-batch-mode] [--skip-push] [--alt-repository=] [--thirdparty=scijava] [--skip-gpg] [--extra-arg=] " +# Discern the version to release. +currentVersion=$(mvn -N -Dexec.executable='echo' -Dexec.args='${project.version}' exec:exec -q) +pomVersion=${currentVersion%-SNAPSHOT} +test "$VERSION" || test ! -t 0 || { + printf 'Version? [%s]: ' "$pomVersion" + read VERSION + test "$VERSION" || VERSION=$pomVersion +} -VERSION="$1" +# If REMOTE is unset, use origin by default. REMOTE="${REMOTE:-origin}" -# do a quick sanity check on the new version number +# Check that the release version number starts with a digit. +test "$VERSION" || die 'Please specify the version to release!' case "$VERSION" in [0-9]*) ;; *) die "Version '$VERSION' does not start with a digit!" esac -test "$SKIP_VERSION_CHECK" || - valid_semver_bump "$pomVersion" "$VERSION" - -# defaults -BASE_GAV="$(maven_helper gav-from-pom pom.xml)" || -die "Could not obtain GAV coordinates for base project" +# Check that the release version number conforms to SemVer. +VALID_SEMVER_BUMP="$(cd "$(dirname "$0")" && pwd)/valid-semver-bump.sh" +test -f "$VALID_SEMVER_BUMP" || + die "Missing helper script at '$VALID_SEMVER_BUMP'" +test "$SKIP_VERSION_CHECK" || { + sh -$- "$VALID_SEMVER_BUMP" "$pomVersion" "$VERSION" || die +} +# Check that the working copy is clean. git update-index -q --refresh && git diff-files --quiet --ignore-submodules && git diff-index --cached --quiet --ignore-submodules HEAD -- || die "There are uncommitted changes!" +# Check that we are on the master branch. test refs/heads/master = "$(git rev-parse --symbolic-full-name HEAD)" || die "Not on 'master' branch" +# Check that the master branch isn't behind the upstream branch. HEAD="$(git rev-parse HEAD)" && git fetch "$REMOTE" master && FETCH_HEAD="$(git rev-parse FETCH_HEAD)" && @@ -125,13 +151,12 @@ test "$FETCH_HEAD" = HEAD || test "$FETCH_HEAD" = "$(git merge-base $FETCH_HEAD $HEAD)" || die "'master' is not up-to-date" -# Prepare new release without pushing (requires the release plugin >= 2.1) +# Prepare new release without pushing (requires the release plugin >= 2.1). $DRY_RUN mvn $BATCH_MODE release:prepare -DpushChanges=false -Dresume=false $TAG \ $PROFILE $DEV_VERSION -DreleaseVersion="$VERSION" \ "-Darguments=-Dgpg.skip=true ${EXTRA_ARGS# }" && -# Squash the two commits on the current branch produced by the -# maven-release-plugin into one +# Squash the maven-release-plugin's two commits into one. if test -z "$DRY_RUN" then test "[maven-release-plugin] prepare for next development iteration" = \ @@ -144,7 +169,7 @@ then $DRY_RUN git commit -s -m "Bump to next development cycle" fi && -# extract the name of the new tag +# Extract the name of the new tag. if test -z "$DRY_RUN" then tag=$(sed -n 's/^scm.tag=//p' < release.properties) @@ -152,7 +177,7 @@ else tag="" fi && -# rewrite the tag to include release.properties +# Rewrite the tag to include release.properties. test -n "$tag" && # HACK: SciJava projects use SSH (git@github.com:...) for developerConnection. # The release:perform command wants to use the developerConnection URL when @@ -169,7 +194,7 @@ $DRY_RUN git tag -d "$tag" && $DRY_RUN git tag "$tag" HEAD && $DRY_RUN git checkout @{-1} && -# push the current branch and the tag +# Push the current branch and the tag. if test -z "$SKIP_PUSH" then $DRY_RUN git push "$REMOTE" HEAD $tag From c440456e299cda776ad9265aa2903952fb9fe070 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 24 May 2020 20:18:12 -0500 Subject: [PATCH 161/429] release-version: refactor a little bit more --- release-version.sh | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/release-version.sh b/release-version.sh index 5226dfe..d329564 100755 --- a/release-version.sh +++ b/release-version.sh @@ -15,6 +15,12 @@ die () { exit 1 } +no_changes_pending() { + git update-index -q --refresh && + git diff-files --quiet --ignore-submodules && + git diff-index --cached --quiet --ignore-submodules HEAD -- +} + # -- Constants and settings -- SCIJAVA_BASE_REPOSITORY=-DaltDeploymentRepository=scijava.releases::default::dav:https://maven.scijava.org/content/repositories @@ -113,9 +119,6 @@ test "$VERSION" || test ! -t 0 || { test "$VERSION" || VERSION=$pomVersion } -# If REMOTE is unset, use origin by default. -REMOTE="${REMOTE:-origin}" - # Check that the release version number starts with a digit. test "$VERSION" || die 'Please specify the version to release!' case "$VERSION" in @@ -134,14 +137,14 @@ test "$SKIP_VERSION_CHECK" || { } # Check that the working copy is clean. -git update-index -q --refresh && -git diff-files --quiet --ignore-submodules && -git diff-index --cached --quiet --ignore-submodules HEAD -- || -die "There are uncommitted changes!" +no_changes_pending || die 'There are uncommitted changes!' # Check that we are on the master branch. test refs/heads/master = "$(git rev-parse --symbolic-full-name HEAD)" || -die "Not on 'master' branch" + die "Not on 'master' branch" + +# If REMOTE is unset, use origin by default. +REMOTE="${REMOTE:-origin}" # Check that the master branch isn't behind the upstream branch. HEAD="$(git rev-parse HEAD)" && @@ -149,7 +152,8 @@ git fetch "$REMOTE" master && FETCH_HEAD="$(git rev-parse FETCH_HEAD)" && test "$FETCH_HEAD" = HEAD || test "$FETCH_HEAD" = "$(git merge-base $FETCH_HEAD $HEAD)" || -die "'master' is not up-to-date" + die "'master' is not up-to-date" + # Prepare new release without pushing (requires the release plugin >= 2.1). $DRY_RUN mvn $BATCH_MODE release:prepare -DpushChanges=false -Dresume=false $TAG \ @@ -161,7 +165,7 @@ if test -z "$DRY_RUN" then test "[maven-release-plugin] prepare for next development iteration" = \ "$(git show -s --format=%s HEAD)" || - die "maven-release-plugin's commits are unexpectedly missing!" + die "maven-release-plugin's commits are unexpectedly missing!" fi $DRY_RUN git reset --soft HEAD^^ && if ! git diff-index --cached --quiet --ignore-submodules HEAD -- From 7a17be1ec8644683e0bbedef2ac0aeb148e08168 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 24 May 2020 22:10:03 -0500 Subject: [PATCH 162/429] release-version: check for newer parent versions This is helpful to avoid releasing things built on too-old pom-scijava versions, which are less likely to melt correctly with everything else. --- release-version.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/release-version.sh b/release-version.sh index d329564..390a573 100755 --- a/release-version.sh +++ b/release-version.sh @@ -100,6 +100,12 @@ Options include: --skip-gpg - Do not perform GPG signing of artifacts. " +# -- Extract project details -- + +projectDetails=$(mvn -N -Dexec.executable='echo' -Dexec.args='${project.version}::${project.parent.groupId}:${project.parent.artifactId}:${project.parent.version}' exec:exec -q) +parentGAV=${projectDetails#*::} +currentVersion=${projectDetails%::*} + # -- Sanity checks -- # Check that we have push rights to the repository. @@ -111,7 +117,6 @@ then fi # Discern the version to release. -currentVersion=$(mvn -N -Dexec.executable='echo' -Dexec.args='${project.version}' exec:exec -q) pomVersion=${currentVersion%-SNAPSHOT} test "$VERSION" || test ! -t 0 || { printf 'Version? [%s]: ' "$pomVersion" @@ -136,6 +141,17 @@ test "$SKIP_VERSION_CHECK" || { sh -$- "$VALID_SEMVER_BUMP" "$pomVersion" "$VERSION" || die } +# Check that the project extends the latest version of pom-scijava. +MAVEN_HELPER="$(cd "$(dirname "$0")" && pwd)/maven-helper.sh" +test -f "$MAVEN_HELPER" || + die "Missing helper script at '$MAVEN_HELPER'" +test "$SKIP_VERSION_CHECKS" || { + latestParentVersion=$(sh -$- "$MAVEN_HELPER" latest-version "$parentGAV") + currentParentVersion=${parentGAV##*:} + test "$currentParentVersion" = "$latestParentVersion" || + die "Newer version of parent '${parentGAV%:*}' is available: $latestParentVersion" +} + # Check that the working copy is clean. no_changes_pending || die 'There are uncommitted changes!' From e2beadaeb8fc5fa21e28989b8e0bce4b701b2011 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 24 May 2020 22:12:51 -0500 Subject: [PATCH 163/429] release-version: update copyright blurbs if needed --- release-version.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/release-version.sh b/release-version.sh index 390a573..9774d48 100755 --- a/release-version.sh +++ b/release-version.sh @@ -170,6 +170,12 @@ test "$FETCH_HEAD" = HEAD || test "$FETCH_HEAD" = "$(git merge-base $FETCH_HEAD $HEAD)" || die "'master' is not up-to-date" +# Ensure license headers are up-to-date. +mvn license:update-project-license license:update-file-header && +git add LICENSE.txt || + die 'Failed to update copyright blurbs' +no_changes_pending || + die 'Copyright blurbs needed an update -- commit changes and try again' # Prepare new release without pushing (requires the release plugin >= 2.1). $DRY_RUN mvn $BATCH_MODE release:prepare -DpushChanges=false -Dresume=false $TAG \ From 5478ec0d68869afb412f1ac1e90ed3b1fce60846 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 26 May 2020 00:16:08 -0500 Subject: [PATCH 164/429] release-version: fix bug in --skip-version-checks It is supposed to skip the parent POM version check, too. --- release-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-version.sh b/release-version.sh index 9774d48..d0c6f0a 100755 --- a/release-version.sh +++ b/release-version.sh @@ -145,7 +145,7 @@ test "$SKIP_VERSION_CHECK" || { MAVEN_HELPER="$(cd "$(dirname "$0")" && pwd)/maven-helper.sh" test -f "$MAVEN_HELPER" || die "Missing helper script at '$MAVEN_HELPER'" -test "$SKIP_VERSION_CHECKS" || { +test "$SKIP_VERSION_CHECK" || { latestParentVersion=$(sh -$- "$MAVEN_HELPER" latest-version "$parentGAV") currentParentVersion=${parentGAV##*:} test "$currentParentVersion" = "$latestParentVersion" || From 9c9a9935003e67a14e8e041a97a91e306367b8f0 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 26 May 2020 07:55:17 -0500 Subject: [PATCH 165/429] release-version: skip parent check if no parent --- release-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-version.sh b/release-version.sh index d0c6f0a..d8fc791 100755 --- a/release-version.sh +++ b/release-version.sh @@ -145,7 +145,7 @@ test "$SKIP_VERSION_CHECK" || { MAVEN_HELPER="$(cd "$(dirname "$0")" && pwd)/maven-helper.sh" test -f "$MAVEN_HELPER" || die "Missing helper script at '$MAVEN_HELPER'" -test "$SKIP_VERSION_CHECK" || { +test "$SKIP_VERSION_CHECK" -o "$parentGAV" != "${parentGAV#$}" || { latestParentVersion=$(sh -$- "$MAVEN_HELPER" latest-version "$parentGAV") currentParentVersion=${parentGAV##*:} test "$currentParentVersion" = "$latestParentVersion" || From 3b9d9c60dc16467cd8638673cc4a9fa613b95bd4 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 27 May 2020 15:19:16 -0500 Subject: [PATCH 166/429] release-version.sh: handle missing/invalid license If licenseName is not set, or set to N/A, skip copyright blurb update. --- release-version.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/release-version.sh b/release-version.sh index d8fc791..de613b9 100755 --- a/release-version.sh +++ b/release-version.sh @@ -102,9 +102,11 @@ Options include: # -- Extract project details -- -projectDetails=$(mvn -N -Dexec.executable='echo' -Dexec.args='${project.version}::${project.parent.groupId}:${project.parent.artifactId}:${project.parent.version}' exec:exec -q) -parentGAV=${projectDetails#*::} -currentVersion=${projectDetails%::*} +projectDetails=$(mvn -N -Dexec.executable='echo' -Dexec.args='${project.version}/${license.licenseName}/${project.parent.groupId}:${project.parent.artifactId}:${project.parent.version}' exec:exec -q) +currentVersion=${projectDetails%%/*} +projectDetails=${projectDetails#*/} +licenseName=${projectDetails%%/*} +parentGAV=${projectDetails#*/} # -- Sanity checks -- @@ -171,11 +173,13 @@ test "$FETCH_HEAD" = "$(git merge-base $FETCH_HEAD $HEAD)" || die "'master' is not up-to-date" # Ensure license headers are up-to-date. -mvn license:update-project-license license:update-file-header && -git add LICENSE.txt || - die 'Failed to update copyright blurbs' -no_changes_pending || - die 'Copyright blurbs needed an update -- commit changes and try again' +test -z "$licenseName" -o "$licenseName" = "N/A" || { + mvn license:update-project-license license:update-file-header && + git add LICENSE.txt || + die 'Failed to update copyright blurbs' + no_changes_pending || + die 'Copyright blurbs needed an update -- commit changes and try again' +} # Prepare new release without pushing (requires the release plugin >= 2.1). $DRY_RUN mvn $BATCH_MODE release:prepare -DpushChanges=false -Dresume=false $TAG \ From 2c49dc109ae690a65debdb8e2bf2e40849f3eab2 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 28 May 2020 08:21:00 -0500 Subject: [PATCH 167/429] release-version: add flag to skip license update And add a hint about how to exclude erroneously touched files. --- release-version.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/release-version.sh b/release-version.sh index de613b9..7f27aaf 100755 --- a/release-version.sh +++ b/release-version.sh @@ -30,6 +30,7 @@ SCIJAVA_THIRDPARTY_REPOSITORY=$SCIJAVA_BASE_REPOSITORY/thirdparty # Parse command line options. BATCH_MODE=--batch-mode SKIP_VERSION_CHECK= +SKIP_LICENSE_UPDATE= SKIP_PUSH= SKIP_GPG= TAG= @@ -46,6 +47,7 @@ do --dry-run) DRY_RUN=echo;; --no-batch-mode) BATCH_MODE=;; --skip-version-check) SKIP_VERSION_CHECK=t;; + --skip-license-update) SKIP_LICENSE_UPDATE=t;; --skip-push) SKIP_PUSH=t;; --tag=*) ! git rev-parse --quiet --verify refs/tags/"${1#--*=}" || @@ -90,6 +92,7 @@ Where is the version to release. If omitted, it will prompt you. Options include: --dry-run - Simulate the release without actually doing it. --skip-version-check - Violate SemVer version numbering intentionally. + --skip-license-update - Skips update of the copyright blurbs. --skip-push - Do not push to the remote git repository. --dev-version= - Specify next development version explicitly; e.g.: if you release 2.0.0-beta-1, by default @@ -173,12 +176,17 @@ test "$FETCH_HEAD" = "$(git merge-base $FETCH_HEAD $HEAD)" || die "'master' is not up-to-date" # Ensure license headers are up-to-date. -test -z "$licenseName" -o "$licenseName" = "N/A" || { +test "$SKIP_LICENSE_UPDATE" -o -z "$licenseName" -o "$licenseName" = "N/A" || { mvn license:update-project-license license:update-file-header && git add LICENSE.txt || die 'Failed to update copyright blurbs' no_changes_pending || - die 'Copyright blurbs needed an update -- commit changes and try again' + die 'Copyright blurbs needed an update -- commit changes and try again. +Or if the license headers are being added erroneously to certain files, +exclude them by setting license.excludes in your POM; e.g.: + + **/script_templates/** +' } # Prepare new release without pushing (requires the release plugin >= 2.1). From c1f427194c0d5ec198335b1ddb2e36df8891f436 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 28 May 2020 08:23:46 -0500 Subject: [PATCH 168/429] release-version: improve --skip-version-check docs Document that it also skips the parent POM version check. And give a hint how to skip the check if you don't want it. --- release-version.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/release-version.sh b/release-version.sh index 7f27aaf..bfa1793 100755 --- a/release-version.sh +++ b/release-version.sh @@ -91,7 +91,7 @@ Where is the version to release. If omitted, it will prompt you. Options include: --dry-run - Simulate the release without actually doing it. - --skip-version-check - Violate SemVer version numbering intentionally. + --skip-version-check - Skips the SemVer and parent pom version checks. --skip-license-update - Skips update of the copyright blurbs. --skip-push - Do not push to the remote git repository. --dev-version= - Specify next development version explicitly; @@ -154,7 +154,9 @@ test "$SKIP_VERSION_CHECK" -o "$parentGAV" != "${parentGAV#$}" || { latestParentVersion=$(sh -$- "$MAVEN_HELPER" latest-version "$parentGAV") currentParentVersion=${parentGAV##*:} test "$currentParentVersion" = "$latestParentVersion" || - die "Newer version of parent '${parentGAV%:*}' is available: $latestParentVersion" + die "Newer version of parent '${parentGAV%:*}' is available: $latestParentVersion. +I recommend you update it before releasing. +Or if you know better, try again with --skip-version-check flag." } # Check that the working copy is clean. From d12215204a888f5f66472e13850ab833d8d0b7cb Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 29 May 2020 08:34:00 -0500 Subject: [PATCH 169/429] release-version: give advice in more failure cases --- release-version.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/release-version.sh b/release-version.sh index bfa1793..8b9e0e9 100755 --- a/release-version.sh +++ b/release-version.sh @@ -141,15 +141,18 @@ esac # Check that the release version number conforms to SemVer. VALID_SEMVER_BUMP="$(cd "$(dirname "$0")" && pwd)/valid-semver-bump.sh" test -f "$VALID_SEMVER_BUMP" || - die "Missing helper script at '$VALID_SEMVER_BUMP'" + die "Missing helper script at '$VALID_SEMVER_BUMP' +Do you have a full clone of git://github.com/scijava/scijava-scripts?" test "$SKIP_VERSION_CHECK" || { - sh -$- "$VALID_SEMVER_BUMP" "$pomVersion" "$VERSION" || die + sh -$- "$VALID_SEMVER_BUMP" "$pomVersion" "$VERSION" || + die "If you are sure, try again with --skip-version-check flag." } # Check that the project extends the latest version of pom-scijava. MAVEN_HELPER="$(cd "$(dirname "$0")" && pwd)/maven-helper.sh" test -f "$MAVEN_HELPER" || - die "Missing helper script at '$MAVEN_HELPER'" + die "Missing helper script at '$MAVEN_HELPER' +Do you have a full clone of git://github.com/scijava/scijava-scripts?" test "$SKIP_VERSION_CHECK" -o "$parentGAV" != "${parentGAV#$}" || { latestParentVersion=$(sh -$- "$MAVEN_HELPER" latest-version "$parentGAV") currentParentVersion=${parentGAV##*:} From cd736c70c08278b426bde01b6ac7e2ab63bbbdc7 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 5 Jun 2020 15:54:14 -0500 Subject: [PATCH 170/429] Fix bug in license string parsing We now use colon as token divider uniformly, never slash, because we sometimes use slash for the licenseName (specifically: "N/A"). I don't know of any case where colon appears in any of these properties. --- release-version.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/release-version.sh b/release-version.sh index 8b9e0e9..d2f0aef 100755 --- a/release-version.sh +++ b/release-version.sh @@ -105,11 +105,11 @@ Options include: # -- Extract project details -- -projectDetails=$(mvn -N -Dexec.executable='echo' -Dexec.args='${project.version}/${license.licenseName}/${project.parent.groupId}:${project.parent.artifactId}:${project.parent.version}' exec:exec -q) -currentVersion=${projectDetails%%/*} -projectDetails=${projectDetails#*/} -licenseName=${projectDetails%%/*} -parentGAV=${projectDetails#*/} +projectDetails=$(mvn -N -Dexec.executable='echo' -Dexec.args='${project.version}:${license.licenseName}:${project.parent.groupId}:${project.parent.artifactId}:${project.parent.version}' exec:exec -q) +currentVersion=${projectDetails%%:*} +projectDetails=${projectDetails#*:} +licenseName=${projectDetails%%:*} +parentGAV=${projectDetails#*:} # -- Sanity checks -- From 1461866d87c111a94a4dc75670e9752e262b4d06 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 12 Jun 2020 11:20:41 -0500 Subject: [PATCH 171/429] travis-build: disable TLSv1.3 Hopefully this will avoid intermittent errors of the form: javax.net.ssl.SSLException: No PSK available. Unable to resume. Credit for the workaround: https://stackoverflow.com/a/54280095/1207769 --- travis-build.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/travis-build.sh b/travis-build.sh index ded0dce..38da95f 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -136,22 +136,23 @@ EOL fi # Run the build. + BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' if [ "$deployOK" -a "$TRAVIS_BRANCH" = master ] then echo echo "== Building and deploying master SNAPSHOT ==" - mvn -B -Pdeploy-to-scijava deploy + mvn -Pdeploy-to-scijava $BUILD_ARGS deploy checkSuccess $? elif [ "$deployOK" -a -f release.properties ] then echo echo "== Cutting and deploying release version ==" - mvn -B release:perform + mvn -B $BUILD_ARGS release:perform checkSuccess $? else echo echo "== Building the artifact locally only ==" - mvn -B install javadoc:javadoc + mvn $BUILD_ARGS install javadoc:javadoc checkSuccess $? fi echo travis_fold:end:scijava-maven From 7edaa0420c8155436e9074bbc6fc57332bbcfb12 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 9 Jul 2020 22:14:07 -0500 Subject: [PATCH 172/429] travis-build: show how ciManagement command fails --- travis-build.sh | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/travis-build.sh b/travis-build.sh index 38da95f..cb076fe 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -89,24 +89,31 @@ EOL fi # Determine whether deploying will be possible. - ciURL=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${project.ciManagement.url}' --non-recursive validate exec:exec 2>/dev/null) - ciRepo=${ciURL##*/} - ciPrefix=${ciURL%/*} - ciOrg=${ciPrefix##*/} deployOK= - if [ "$TRAVIS_SECURE_ENV_VARS" != true ] + ciURL=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${project.ciManagement.url}' --non-recursive validate exec:exec 2>&1) + if [ $? -ne 0 ] then - echo "No deploy -- secure environment variables not available" - elif [ "$TRAVIS_PULL_REQUEST" != false ] - then - echo "No deploy -- pull request detected" - elif [ "$TRAVIS_REPO_SLUG" != "$ciOrg/$ciRepo" ] - then - echo "No deploy -- repository fork: $TRAVIS_REPO_SLUG != $ciOrg/$ciRepo" - # TODO: Detect travis-ci.org versus travis-ci.com? + echo "No deploy -- could not extract ciManagement URL" + echo "Output of failed attempt follows:" + echo "$ciURL" else - echo "All checks passed for artifact deployment" - deployOK=1 + ciRepo=${ciURL##*/} + ciPrefix=${ciURL%/*} + ciOrg=${ciPrefix##*/} + if [ "$TRAVIS_SECURE_ENV_VARS" != true ] + then + echo "No deploy -- secure environment variables not available" + elif [ "$TRAVIS_PULL_REQUEST" != false ] + then + echo "No deploy -- pull request detected" + elif [ "$TRAVIS_REPO_SLUG" != "$ciOrg/$ciRepo" ] + then + echo "No deploy -- repository fork: $TRAVIS_REPO_SLUG != $ciOrg/$ciRepo" + # TODO: Detect travis-ci.org versus travis-ci.com? + else + echo "All checks passed for artifact deployment" + deployOK=1 + fi fi # Install GPG on OSX/macOS From eb4048825e47c0e757bd6fd2e294bd31b4a933df Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 21 Jul 2020 15:09:51 -0500 Subject: [PATCH 173/429] release-version: fail when untracked files present See discussion at: https://gitter.im/scijava/scijava-common?at=5f16dbaecb20727cfd3bb7b7 --- release-version.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release-version.sh b/release-version.sh index d2f0aef..734ec1e 100755 --- a/release-version.sh +++ b/release-version.sh @@ -164,6 +164,8 @@ Or if you know better, try again with --skip-version-check flag." # Check that the working copy is clean. no_changes_pending || die 'There are uncommitted changes!' +test -z "$(git ls-files -o --exclude-standard)" || + die 'There are untracked files! Please stash them before releasing.' # Check that we are on the master branch. test refs/heads/master = "$(git rev-parse --symbolic-full-name HEAD)" || From 3c619494e3a66c2e2095109c7ac9dc6240040642 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sat, 1 Aug 2020 22:55:45 -0500 Subject: [PATCH 174/429] travis-build: invalidate Nexus cache after release Hopefully this fixes problems with artifacts deployed to OSS Sonatype being unavailable from the SciJava Maven repository's public group. --- travis-build.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/travis-build.sh b/travis-build.sh index cb076fe..0162c13 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -156,6 +156,15 @@ EOL echo "== Cutting and deploying release version ==" mvn -B $BUILD_ARGS release:perform checkSuccess $? + echo "== Invalidating SciJava Maven repository cache ==" + curl -fsLO https://raw.githubusercontent.com/scijava/scijava-scripts/master/maven-helper.sh && + gav=$(sh maven-helper.sh gav-from-pom pom.xml) && + ga=${gav%:*} && + echo "machine maven.scijava.org" > "$HOME/.netrc" && + echo " login travis" >> "$HOME/.netrc" && + echo " password $MAVEN_PASS" >> "$HOME/.netrc" && + sh maven-helper.sh invalidate-cache "$ga" + checkSuccess $? else echo echo "== Building the artifact locally only ==" From 6b25a0970cbc5eed9df7a4822f1dbc04a92ee0ac Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sat, 1 Aug 2020 23:17:05 -0500 Subject: [PATCH 175/429] travis-build: give more info when clearing cache --- travis-build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/travis-build.sh b/travis-build.sh index 0162c13..1a759e1 100644 --- a/travis-build.sh +++ b/travis-build.sh @@ -160,6 +160,7 @@ EOL curl -fsLO https://raw.githubusercontent.com/scijava/scijava-scripts/master/maven-helper.sh && gav=$(sh maven-helper.sh gav-from-pom pom.xml) && ga=${gav%:*} && + echo "--> Artifact to invalidate = $ga" && echo "machine maven.scijava.org" > "$HOME/.netrc" && echo " login travis" >> "$HOME/.netrc" && echo " password $MAVEN_PASS" >> "$HOME/.netrc" && From 38600626e13247ae7133f55c842150782f14ffa7 Mon Sep 17 00:00:00 2001 From: Jan Eglinger Date: Tue, 11 Aug 2020 16:54:01 +0200 Subject: [PATCH 176/429] travisify.sh: ensure correct build.sh permissions on Windows --- travisify.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/travisify.sh b/travisify.sh index 1cfd228..8c1c356 100755 --- a/travisify.sh +++ b/travisify.sh @@ -41,6 +41,7 @@ var() { update() { file=$1 msg=$2 + exe=$3 test "$msg" || msg="Travis: update $file" if [ -e "$file" ] then @@ -59,6 +60,11 @@ update() { fi rm -rf "$tmpFile" $EXEC git add "$file" + if [ -n "$exe" ] + then + info "Adding execute permission to $file" + $EXEC git update-index --chmod=+x "$file" + fi $EXEC git diff-index --quiet HEAD -- || $EXEC git commit -m "$msg" } @@ -135,7 +141,7 @@ curl -fsLO https://raw.githubusercontent.com/scijava/scijava-scripts/master/trav sh travis-build.sh EOL chmod +x "$tmpFile" - update "$travisBuildScript" + update "$travisBuildScript" "Travis: add executable script $travisBuildScript" "true" # Remove obsolete Travis-related files. if [ -f "$travisSettingsFile" ] From 1819e476d8695c6972a81a6bc98e61fa82227b3e Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 20 Aug 2020 15:16:04 -0500 Subject: [PATCH 177/429] Remove disk usage script It's not used in public SciJava dev-ops. --- disk-usage.sh | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100755 disk-usage.sh diff --git a/disk-usage.sh b/disk-usage.sh deleted file mode 100755 index bbc5180..0000000 --- a/disk-usage.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh - -# disk-usage.sh - report when disk space starts getting low - -# Credit to: -# http://www.cyberciti.biz/faq/mac-osx-unix-get-an-alert-when-my-disk-is-full/ - -check() { - FS=$1 - CURRENT=$(df -P "$FS" | tail -n1 | awk '{print $5}' | sed 's/%//') - test "$CURRENT" -gt "$threshold" && - echo "$FS: file system usage at $CURRENT%" && - return 1 - return 0 -} - -threshold=80 -code=0 -while test $# -gt 0 -do - case "$1" in - --threshold|-t) - shift - threshold="$1" - ;; - *) - check "$1" || code=$((code+1)) - ;; - esac - shift -done - -exit $code From 4eca748b4f0a6c8ab6166b7a0ebcd974af7ef6b7 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 20 Aug 2020 15:16:27 -0500 Subject: [PATCH 178/429] release-version: die if release prep fails --- release-version.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release-version.sh b/release-version.sh index 734ec1e..3766450 100755 --- a/release-version.sh +++ b/release-version.sh @@ -199,7 +199,9 @@ exclude them by setting license.excludes in your POM; e.g.: # Prepare new release without pushing (requires the release plugin >= 2.1). $DRY_RUN mvn $BATCH_MODE release:prepare -DpushChanges=false -Dresume=false $TAG \ $PROFILE $DEV_VERSION -DreleaseVersion="$VERSION" \ - "-Darguments=-Dgpg.skip=true ${EXTRA_ARGS# }" && + "-Darguments=-Dgpg.skip=true ${EXTRA_ARGS# }" || + die 'The release preparation step failed -- look above for errors and fix them. +Use "mvn javadoc:javadoc | grep error" to check for javadoc syntax errors.' # Squash the maven-release-plugin's two commits into one. if test -z "$DRY_RUN" From 6d8c49858bf923dda6bcc5c3b4a4fa455bfd7e5f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 8 Oct 2020 16:23:06 -0500 Subject: [PATCH 179/429] class-version.sh: analyze bare classes, too And refactor the code to be more DRY. --- class-version.sh | 44 +++++++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/class-version.sh b/class-version.sh index 54251d5..5d128c2 100755 --- a/class-version.sh +++ b/class-version.sh @@ -2,19 +2,9 @@ # class-version.sh - find the Java version which wrote a JAR file -for jar in "$@" -do - # find the first class of the JAR - class="$(jar tf "$jar" | grep '\.class' | head -n 1 | sed 's/\//./g' | sed 's/\.class$//')" - - if [ -z "$class" ] - then - echo "$jar: No classes" - continue - fi - +class_version() { # extract bytes 4-7 - info="$(unzip -p "$jar" "$(jar tf "$jar" | grep \.class$ | head -n 1)" | head -c 8 | hexdump -s 4 -e '4/1 "%d\n" "\n"')" + info=$(head -c 8 | hexdump -s 4 -e '4/1 "%d\n" "\n"') minor1="$(echo "$info" | sed -n 1p)" minor2="$(echo "$info" | sed -n 2p)" major1="$(echo "$info" | sed -n 3p)" @@ -55,5 +45,33 @@ do esac # report the results - echo "$jar: $version ($major.$minor)" + echo "$version ($major.$minor)" +} + +first_class() { + jar tf "$1" | grep '\.class' | head -n 1 +} + +for file in "$@" +do + case "$file" in + *.class) + version=$(cat "$file" | class_version) + ;; + *.jar) + class=$(first_class "$file") + if [ -z "$class" ] + then + echo "$file: No classes" + continue + fi + version=$(unzip -p "$file" "$class" | class_version) + ;; + *) + >&2 echo "Unsupported file: $file" + continue + esac + + # report the results + echo "$file: $version" done From 7d69169024eea07c814ed8d9303dbe100cabde6b Mon Sep 17 00:00:00 2001 From: Jan Eglinger Date: Tue, 8 Dec 2020 11:34:06 +0100 Subject: [PATCH 180/429] travisify.sh: respect default branch name --- travisify.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/travisify.sh b/travisify.sh index 8c1c356..d05ef3e 100755 --- a/travisify.sh +++ b/travisify.sh @@ -82,8 +82,9 @@ process() { esac git fetch >/dev/null git diff-index --quiet HEAD -- || die "Dirty working copy" - branch=$(git rev-parse --abbrev-ref HEAD) - test "$branch" = "master" || die "Non-master branch: $branch" + currentBranch=$(git rev-parse --abbrev-ref HEAD) + defaultBranch=$(git remote show origin | grep "HEAD branch" | sed 's/.*: //') + test "$currentBranch" = "$defaultBranch" || die "Non-default branch: $currentBranch" git merge --ff --ff-only 'HEAD@{u}' >/dev/null || die "Cannot fast forward (local diverging?)" # test "$(git rev-parse HEAD)" = "$(git rev-parse 'HEAD@{u}')" || @@ -124,7 +125,7 @@ language: java jdk: openjdk8 branches: only: - - master + - $defaultBranch - "/.*-[0-9]+\\\\..*/" install: true script: "$travisBuildScript" From b812833edc273932679bda8ce742853b51ba29cf Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 11 Dec 2020 21:08:40 +0100 Subject: [PATCH 181/429] travisify.sh: generalize name of remote Signed-off-by: Jan Eglinger --- travisify.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/travisify.sh b/travisify.sh index d05ef3e..64058d3 100755 --- a/travisify.sh +++ b/travisify.sh @@ -83,7 +83,9 @@ process() { git fetch >/dev/null git diff-index --quiet HEAD -- || die "Dirty working copy" currentBranch=$(git rev-parse --abbrev-ref HEAD) - defaultBranch=$(git remote show origin | grep "HEAD branch" | sed 's/.*: //') + upstreamBranch=$(git rev-parse --abbrev-ref --symbolic-full-name @{u}) + remote=${upstreamBranch%/*} + defaultBranch=$(git remote show "$remote" | grep "HEAD branch" | sed 's/.*: //') test "$currentBranch" = "$defaultBranch" || die "Non-default branch: $currentBranch" git merge --ff --ff-only 'HEAD@{u}' >/dev/null || die "Cannot fast forward (local diverging?)" From 483b2cf48297546d76161805b22b4c0dbd96812a Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 13 Dec 2020 23:22:08 -0600 Subject: [PATCH 182/429] Point Travis badge at the correct main branch --- travisify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travisify.sh b/travisify.sh index 64058d3..87e4ab2 100755 --- a/travisify.sh +++ b/travisify.sh @@ -195,7 +195,7 @@ EOL update README.md 'Travis: fix README.md badge link' else info "Adding Travis badge to README.md" - echo "[![](https://$domain/$repoSlug.svg?branch=master)](https://$domain/$repoSlug)" >"$tmpFile" + echo "[![](https://$domain/$repoSlug.svg?branch=$defaultBranch)](https://$domain/$repoSlug)" >"$tmpFile" echo >>"$tmpFile" test -f README.md && cat README.md >>"$tmpFile" update README.md 'Travis: add badge to README.md' From aacec2d09ac7c142e4fa01ac8ed49d03eb669d0a Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 13 Dec 2020 23:28:29 -0600 Subject: [PATCH 183/429] release-version.sh: add hints if remote is wrong --- release-version.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/release-version.sh b/release-version.sh index 3766450..f2e3e5f 100755 --- a/release-version.sh +++ b/release-version.sh @@ -117,8 +117,10 @@ parentGAV=${projectDetails#*:} if [ ! "$SKIP_PUSH" ] then push=$(git remote -v | grep origin | grep '(push)') - test "$push" || die 'No push URL found for remote origin' - echo "$push" | grep -q 'git:/' && die 'Remote origin is read-only' + test "$push" || die 'No push URL found for remote origin. +Please use "git remote -v" to double check your remote settings.' + echo "$push" | grep -q 'git:/' && die 'Remote origin is read-only. +Please use "git remote set-url origin ..." to change it.' fi # Discern the version to release. From e62940a7cca70fc62cf30f0697087f94b5b82f0f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 13 Dec 2020 23:29:13 -0600 Subject: [PATCH 184/429] release-version.sh: enable skipping of digit check If --skip-version-check is given, also skip the check enforcing that the release version starts with a number. And when that check fails, give a hint that this flag can skip it. --- release-version.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/release-version.sh b/release-version.sh index f2e3e5f..114cafc 100755 --- a/release-version.sh +++ b/release-version.sh @@ -133,12 +133,15 @@ test "$VERSION" || test ! -t 0 || { # Check that the release version number starts with a digit. test "$VERSION" || die 'Please specify the version to release!' -case "$VERSION" in -[0-9]*) - ;; -*) - die "Version '$VERSION' does not start with a digit!" -esac +test "$SKIP_VERSION_CHECK" || { + case "$VERSION" in + [0-9]*) + ;; + *) + die "Version '$VERSION' does not start with a digit! +If you are sure, try again with --skip-version-check flag." + esac +} # Check that the release version number conforms to SemVer. VALID_SEMVER_BUMP="$(cd "$(dirname "$0")" && pwd)/valid-semver-bump.sh" From a51f89417591fcd389e0f8ea095d6358567411b8 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 13 Dec 2020 23:31:34 -0600 Subject: [PATCH 185/429] release-version.sh: remove hardcoded master+origin We now discern the default/main branch, as well as its upstream remote, rather than hardcoding master and origin respectively. --- release-version.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/release-version.sh b/release-version.sh index 114cafc..96dc4e8 100755 --- a/release-version.sh +++ b/release-version.sh @@ -172,20 +172,25 @@ no_changes_pending || die 'There are uncommitted changes!' test -z "$(git ls-files -o --exclude-standard)" || die 'There are untracked files! Please stash them before releasing.' -# Check that we are on the master branch. -test refs/heads/master = "$(git rev-parse --symbolic-full-name HEAD)" || - die "Not on 'master' branch" +# Discern default branch. +currentBranch=$(git rev-parse --abbrev-ref --symbolic-full-name HEAD) +upstreamBranch=$(git rev-parse --abbrev-ref --symbolic-full-name @{u}) +remote=${upstreamBranch%/*} +defaultBranch=$(git remote show "$remote" | grep "HEAD branch" | sed 's/.*: //') -# If REMOTE is unset, use origin by default. -REMOTE="${REMOTE:-origin}" +# Check that we are on the main branch. +test "$currentBranch" = "$defaultBranch" || die "Non-default branch: $currentBranch" -# Check that the master branch isn't behind the upstream branch. +# If REMOTE is unset, use branch's upstream remote by default. +REMOTE="${REMOTE:-$remote}" + +# Check that the main branch isn't behind the upstream branch. HEAD="$(git rev-parse HEAD)" && -git fetch "$REMOTE" master && +git fetch "$REMOTE" "$defaultBranch" && FETCH_HEAD="$(git rev-parse FETCH_HEAD)" && test "$FETCH_HEAD" = HEAD || test "$FETCH_HEAD" = "$(git merge-base $FETCH_HEAD $HEAD)" || - die "'master' is not up-to-date" + die "'$defaultBranch' is not up-to-date" # Ensure license headers are up-to-date. test "$SKIP_LICENSE_UPDATE" -o -z "$licenseName" -o "$licenseName" = "N/A" || { From 576c6bab9611d284ccd6f30f1a1f2e1361b9661d Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 13 Dec 2020 23:34:31 -0600 Subject: [PATCH 186/429] release-version.sh: add flag to skip branch check The SciJava ecosystem doesn't have scenarios where we'd want to release from the non-default branch. But this flag makes it possible to try. --- release-version.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/release-version.sh b/release-version.sh index 96dc4e8..b6fc06d 100755 --- a/release-version.sh +++ b/release-version.sh @@ -30,6 +30,7 @@ SCIJAVA_THIRDPARTY_REPOSITORY=$SCIJAVA_BASE_REPOSITORY/thirdparty # Parse command line options. BATCH_MODE=--batch-mode SKIP_VERSION_CHECK= +SKIP_BRANCH_CHECK= SKIP_LICENSE_UPDATE= SKIP_PUSH= SKIP_GPG= @@ -47,6 +48,7 @@ do --dry-run) DRY_RUN=echo;; --no-batch-mode) BATCH_MODE=;; --skip-version-check) SKIP_VERSION_CHECK=t;; + --skip-branch-check) SKIP_BRANCH_CHECK=t;; --skip-license-update) SKIP_LICENSE_UPDATE=t;; --skip-push) SKIP_PUSH=t;; --tag=*) @@ -92,6 +94,7 @@ Where is the version to release. If omitted, it will prompt you. Options include: --dry-run - Simulate the release without actually doing it. --skip-version-check - Skips the SemVer and parent pom version checks. + --skip-branch-check - Skips the default branch check. --skip-license-update - Skips update of the copyright blurbs. --skip-push - Do not push to the remote git repository. --dev-version= - Specify next development version explicitly; @@ -179,7 +182,11 @@ remote=${upstreamBranch%/*} defaultBranch=$(git remote show "$remote" | grep "HEAD branch" | sed 's/.*: //') # Check that we are on the main branch. -test "$currentBranch" = "$defaultBranch" || die "Non-default branch: $currentBranch" +test "$SKIP_BRANCH_CHECK" || { + test "$currentBranch" = "$defaultBranch" || die "Non-default branch: $currentBranch. +If you are certain you want to release from this branch, +try again with --skip-branch-check flag." +} # If REMOTE is unset, use branch's upstream remote by default. REMOTE="${REMOTE:-$remote}" From 8379c1fe9de7105b900479d8890fe17930b5560f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 13 Dec 2020 23:58:10 -0600 Subject: [PATCH 187/429] release-version.sh: add --skip-license-update hint --- release-version.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/release-version.sh b/release-version.sh index b6fc06d..188617a 100755 --- a/release-version.sh +++ b/release-version.sh @@ -202,15 +202,16 @@ test "$FETCH_HEAD" = "$(git merge-base $FETCH_HEAD $HEAD)" || # Ensure license headers are up-to-date. test "$SKIP_LICENSE_UPDATE" -o -z "$licenseName" -o "$licenseName" = "N/A" || { mvn license:update-project-license license:update-file-header && - git add LICENSE.txt || - die 'Failed to update copyright blurbs' + git add LICENSE.txt || die 'Failed to update copyright blurbs. +You can skip the license update using the --skip-license-update flag.' no_changes_pending || die 'Copyright blurbs needed an update -- commit changes and try again. Or if the license headers are being added erroneously to certain files, exclude them by setting license.excludes in your POM; e.g.: **/script_templates/** -' + +Alternately, try again with the --skip-license-update flag.' } # Prepare new release without pushing (requires the release plugin >= 2.1). From da318ab82e5d0824b267abc8340b025cdc50d816 Mon Sep 17 00:00:00 2001 From: Jack Yuan <55564584+yongleyuan@users.noreply.github.com> Date: Sun, 14 Mar 2021 01:37:36 -0600 Subject: [PATCH 188/429] Create main.yml Create `main.yml` to adapt `travis-build.sh`, `travisify.sh`, and, `travis-javadoc.sh` to GitHub Actions. --- .github/workflows/main.yml | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..26c56f5 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,46 @@ +# This is a GitHub action that runs the following Travis builds every time a push/commit event occurs. +# +# travis-build.sh +# travisify.sh +# travis-javadoc.sh + +name: CI + +# Triggers the workflow on push for every branch +on: push + +jobs: + build: + # This workflow runs on Windows machines + runs-on: windows-latest + + steps: + # Checks-out the repository under $GITHUB_WORKSPACE, so the job can access it + - uses: actions/checkout@v2 + + # Set up the JDK version + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + + # Build by running travis-build.sh + - name: travis-build + run: | + echo Running travis-build.sh + sh ./travis-build.sh + echo Finished running travis-build.sh + + # Build by running travisify.sh + - name: travisify + run: | + echo Running travisify.sh + sh ./travisify.sh + echo Finished running travisify.sh + + # Build by running travis-javadoc.sh + - name: travis-javadoc + run: | + echo Running travis-javadoc.sh + sh ./travis-javadoc.sh + echo Finished running travis-javadoc.sh From b9b063e688efb436e71e884d4519386158786b5d Mon Sep 17 00:00:00 2001 From: Jack Yuan <55564584+yongleyuan@users.noreply.github.com> Date: Thu, 18 Mar 2021 02:37:29 -0500 Subject: [PATCH 189/429] Update main.yml Commented out `travisify.sh` and `travis-javadoc.sh' to test `travis-build.sh` first. --- .github/workflows/main.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 26c56f5..440ad37 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,16 +31,16 @@ jobs: sh ./travis-build.sh echo Finished running travis-build.sh - # Build by running travisify.sh - - name: travisify - run: | - echo Running travisify.sh - sh ./travisify.sh - echo Finished running travisify.sh +# # Build by running travisify.sh +# - name: travisify +# run: | +# echo Running travisify.sh +# sh ./travisify.sh +# echo Finished running travisify.sh - # Build by running travis-javadoc.sh - - name: travis-javadoc - run: | - echo Running travis-javadoc.sh - sh ./travis-javadoc.sh - echo Finished running travis-javadoc.sh +# # Build by running travis-javadoc.sh +# - name: travis-javadoc +# run: | +# echo Running travis-javadoc.sh +# sh ./travis-javadoc.sh +# echo Finished running travis-javadoc.sh From caf9e34162a1c9b266adfa163b3fb836c1fb4460 Mon Sep 17 00:00:00 2001 From: Jack Yuan <55564584+yongleyuan@users.noreply.github.com> Date: Thu, 18 Mar 2021 10:17:02 -0500 Subject: [PATCH 190/429] Delete .github/workflows directory This directory is unnecessary as the goal was not to create a GitHub Action. --- .github/workflows/main.yml | 46 -------------------------------------- 1 file changed, 46 deletions(-) delete mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 440ad37..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,46 +0,0 @@ -# This is a GitHub action that runs the following Travis builds every time a push/commit event occurs. -# -# travis-build.sh -# travisify.sh -# travis-javadoc.sh - -name: CI - -# Triggers the workflow on push for every branch -on: push - -jobs: - build: - # This workflow runs on Windows machines - runs-on: windows-latest - - steps: - # Checks-out the repository under $GITHUB_WORKSPACE, so the job can access it - - uses: actions/checkout@v2 - - # Set up the JDK version - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - # Build by running travis-build.sh - - name: travis-build - run: | - echo Running travis-build.sh - sh ./travis-build.sh - echo Finished running travis-build.sh - -# # Build by running travisify.sh -# - name: travisify -# run: | -# echo Running travisify.sh -# sh ./travisify.sh -# echo Finished running travisify.sh - -# # Build by running travis-javadoc.sh -# - name: travis-javadoc -# run: | -# echo Running travis-javadoc.sh -# sh ./travis-javadoc.sh -# echo Finished running travis-javadoc.sh From 115e13861ad40ee1a13d3f89cba2810ed35b8d4d Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Fri, 19 Mar 2021 18:10:26 -0500 Subject: [PATCH 191/429] Copied and pasted codes from `travis-build.sh` to `github-action-build.sh`. --- github-action-build.sh | 243 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 github-action-build.sh diff --git a/github-action-build.sh b/github-action-build.sh new file mode 100644 index 0000000..1a759e1 --- /dev/null +++ b/github-action-build.sh @@ -0,0 +1,243 @@ +#!/bin/bash + +# +# travis-build.sh - A script to build and/or release SciJava-based projects. +# + +dir="$(dirname "$0")" + +success=0 +checkSuccess() { + # Log non-zero exit code. + test $1 -eq 0 || echo "==> FAILED: EXIT CODE $1" 1>&2 + + # Record the first non-zero exit code. + test $success -eq 0 && success=$1 +} + +# Build Maven projects. +if [ -f pom.xml ] +then + echo travis_fold:start:scijava-maven + echo "= Maven build =" + echo + echo "== Configuring Maven ==" + + # NB: Suppress "Downloading/Downloaded" messages. + # See: https://stackoverflow.com/a/35653426/1207769 + export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" + + # Populate the settings.xml configuration. + mkdir -p "$HOME/.m2" + settingsFile="$HOME/.m2/settings.xml" + customSettings=.travis/settings.xml + if [ -f "$customSettings" ] + then + cp "$customSettings" "$settingsFile" + else + cat >"$settingsFile" < + + + scijava.releases + travis + \${env.MAVEN_PASS} + + + scijava.snapshots + travis + \${env.MAVEN_PASS} + + + sonatype-nexus-releases + scijava-ci + \${env.OSSRH_PASS} + + +EOL + # NB: Use maven.scijava.org instead of Central if defined in repositories. + # This hopefully avoids intermittent "ReasonPhrase:Forbidden" errors + # when the Travis build pings Maven Central; see travis-ci/travis-ci#6593. + grep -A 2 '' pom.xml | grep -q 'maven.scijava.org' && + cat >>"$settingsFile" < + + scijava-mirror + SciJava mirror + https://maven.scijava.org/content/groups/public/ + central + + +EOL + cat >>"$settingsFile" < + + gpg + + + \${env.HOME}/.gnupg + + + + \${env.GPG_KEY_NAME} + \${env.GPG_PASSPHRASE} + + + + +EOL + fi + + # Determine whether deploying will be possible. + deployOK= + ciURL=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${project.ciManagement.url}' --non-recursive validate exec:exec 2>&1) + if [ $? -ne 0 ] + then + echo "No deploy -- could not extract ciManagement URL" + echo "Output of failed attempt follows:" + echo "$ciURL" + else + ciRepo=${ciURL##*/} + ciPrefix=${ciURL%/*} + ciOrg=${ciPrefix##*/} + if [ "$TRAVIS_SECURE_ENV_VARS" != true ] + then + echo "No deploy -- secure environment variables not available" + elif [ "$TRAVIS_PULL_REQUEST" != false ] + then + echo "No deploy -- pull request detected" + elif [ "$TRAVIS_REPO_SLUG" != "$ciOrg/$ciRepo" ] + then + echo "No deploy -- repository fork: $TRAVIS_REPO_SLUG != $ciOrg/$ciRepo" + # TODO: Detect travis-ci.org versus travis-ci.com? + else + echo "All checks passed for artifact deployment" + deployOK=1 + fi + fi + + # Install GPG on OSX/macOS + if [ "$TRAVIS_OS_NAME" = osx ] + then + HOMEBREW_NO_AUTO_UPDATE=1 brew install gnupg2 + fi + + # Import the GPG signing key. + keyFile=.travis/signingkey.asc + key=$1 + iv=$2 + if [ "$key" -a "$iv" -a -f "$keyFile.enc" ] + then + # NB: Key and iv values were given as arguments. + echo + echo "== Decrypting GPG keypair ==" + openssl aes-256-cbc -K "$key" -iv "$iv" -in "$keyFile.enc" -out "$keyFile" -d + checkSuccess $? + fi + if [ "$deployOK" -a -f "$keyFile" ] + then + echo + echo "== Importing GPG keypair ==" + gpg --batch --fast-import "$keyFile" + checkSuccess $? + fi + + # Run the build. + BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' + if [ "$deployOK" -a "$TRAVIS_BRANCH" = master ] + then + echo + echo "== Building and deploying master SNAPSHOT ==" + mvn -Pdeploy-to-scijava $BUILD_ARGS deploy + checkSuccess $? + elif [ "$deployOK" -a -f release.properties ] + then + echo + echo "== Cutting and deploying release version ==" + mvn -B $BUILD_ARGS release:perform + checkSuccess $? + echo "== Invalidating SciJava Maven repository cache ==" + curl -fsLO https://raw.githubusercontent.com/scijava/scijava-scripts/master/maven-helper.sh && + gav=$(sh maven-helper.sh gav-from-pom pom.xml) && + ga=${gav%:*} && + echo "--> Artifact to invalidate = $ga" && + echo "machine maven.scijava.org" > "$HOME/.netrc" && + echo " login travis" >> "$HOME/.netrc" && + echo " password $MAVEN_PASS" >> "$HOME/.netrc" && + sh maven-helper.sh invalidate-cache "$ga" + checkSuccess $? + else + echo + echo "== Building the artifact locally only ==" + mvn $BUILD_ARGS install javadoc:javadoc + checkSuccess $? + fi + echo travis_fold:end:scijava-maven +fi + +# Configure conda environment, if one is needed. +if [ -f environment.yml ] +then + echo travis_fold:start:scijava-conda + echo "= Conda setup =" + + condaDir=$HOME/miniconda + condaSh=$condaDir/etc/profile.d/conda.sh + if [ ! -f "$condaSh" ]; then + echo + echo "== Installing conda ==" + if [ "$TRAVIS_PYTHON_VERSION" = "2.7" ]; then + wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh + else + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + fi + rm -rf "$condaDir" + bash miniconda.sh -b -p "$condaDir" + checkSuccess $? + fi + + echo + echo "== Updating conda ==" + . "$condaSh" && + conda config --set always_yes yes --set changeps1 no && + conda update -q conda && + conda info -a + checkSuccess $? + + echo + echo "== Configuring environment ==" + condaEnv=travis-scijava + test -d "$condaDir/envs/$condaEnv" && condaAction=update || condaAction=create + conda env "$condaAction" -n "$condaEnv" -f environment.yml && + conda activate "$condaEnv" + checkSuccess $? + + echo travis_fold:end:scijava-conda +fi + +# Execute Jupyter notebooks. +if which jupyter >/dev/null 2>/dev/null +then + echo travis_fold:start:scijava-jupyter + echo "= Jupyter notebooks =" + # NB: This part is fiddly. We want to loop over files even with spaces, + # so we use the "find ... -print0 | while read $'\0' ..." idiom. + # However, that runs the piped expression in a subshell, which means + # that any updates to the success variable will not persist outside + # the loop. So we suppress all stdout inside the loop, echoing only + # the final value of success upon completion, and then capture the + # echoed value back into the parent shell's success variable. + success=$(find . -name '*.ipynb' -print0 | { + while read -d $'\0' nbf + do + echo 1>&2 + echo "== $nbf ==" 1>&2 + jupyter nbconvert --execute --stdout "$nbf" >/dev/null + checkSuccess $? + done + echo $success + }) + echo travis_fold:end:scijava-jupyter +fi + +exit $success From 8bd74a0ba6a33c6ece6d6f1937edadc0f2f71447 Mon Sep 17 00:00:00 2001 From: Niko Ehrenfeuchter Date: Wed, 24 Mar 2021 11:02:59 +0100 Subject: [PATCH 192/429] Force locale setting for the script This avoids localized output that can confuse the script, e.g. when determining the $defaultBranch. --- release-version.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/release-version.sh b/release-version.sh index 188617a..591b218 100755 --- a/release-version.sh +++ b/release-version.sh @@ -8,6 +8,10 @@ # Authors: Johannes Schindelin & Curtis Rueden # ============================================================================ +# -- Avoid localized output that might confuse the script -- + +export LC_ALL=C + # -- Functions -- die () { From 1ce941365ddc1c3d22dffd4bf78a006abb62dee4 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Wed, 24 Mar 2021 17:58:59 -0500 Subject: [PATCH 193/429] added comments in for further changes to migrate to GitHub Actions --- github-action-build.sh | 77 +++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 43 deletions(-) diff --git a/github-action-build.sh b/github-action-build.sh index 1a759e1..2dcfd6e 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -16,8 +16,7 @@ checkSuccess() { } # Build Maven projects. -if [ -f pom.xml ] -then +if [ -f pom.xml ]; then echo travis_fold:start:scijava-maven echo "= Maven build =" echo @@ -25,16 +24,16 @@ then # NB: Suppress "Downloading/Downloaded" messages. # See: https://stackoverflow.com/a/35653426/1207769 - export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" + export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" # Populate the settings.xml configuration. mkdir -p "$HOME/.m2" settingsFile="$HOME/.m2/settings.xml" customSettings=.travis/settings.xml - if [ -f "$customSettings" ] - then + # Since Travis files will be deleted, just manually set $settingsFile + if [ -f "$customSettings" ]; then cp "$customSettings" "$settingsFile" - else + else # Might need to change travis cat >"$settingsFile" < @@ -59,7 +58,7 @@ EOL # This hopefully avoids intermittent "ReasonPhrase:Forbidden" errors # when the Travis build pings Maven Central; see travis-ci/travis-ci#6593. grep -A 2 '' pom.xml | grep -q 'maven.scijava.org' && - cat >>"$settingsFile" <>"$settingsFile" < scijava-mirror @@ -91,25 +90,25 @@ EOL # Determine whether deploying will be possible. deployOK= ciURL=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${project.ciManagement.url}' --non-recursive validate exec:exec 2>&1) - if [ $? -ne 0 ] - then + if [ $? -ne 0 ]; then echo "No deploy -- could not extract ciManagement URL" echo "Output of failed attempt follows:" echo "$ciURL" else + # https://developer.okta.com/blog/2020/05/18/travis-ci-to-github-actions + # This website specifies Travis environment variables with their + # equivalent GitHub Actions environment variables ciRepo=${ciURL##*/} ciPrefix=${ciURL%/*} ciOrg=${ciPrefix##*/} - if [ "$TRAVIS_SECURE_ENV_VARS" != true ] - then + if [ "$TRAVIS_SECURE_ENV_VARS" != true ]; then # haven't found equivalent env var echo "No deploy -- secure environment variables not available" - elif [ "$TRAVIS_PULL_REQUEST" != false ] - then + elif [ "$TRAVIS_PULL_REQUEST" != false ]; then # ${{ github.event.number }} echo "No deploy -- pull request detected" - elif [ "$TRAVIS_REPO_SLUG" != "$ciOrg/$ciRepo" ] - then + elif [ "$TRAVIS_REPO_SLUG" != "$ciOrg/$ciRepo" ]; then # ${{ github.repository }} echo "No deploy -- repository fork: $TRAVIS_REPO_SLUG != $ciOrg/$ciRepo" # TODO: Detect travis-ci.org versus travis-ci.com? + # Maybe above is not needed else echo "All checks passed for artifact deployment" deployOK=1 @@ -117,8 +116,7 @@ EOL fi # Install GPG on OSX/macOS - if [ "$TRAVIS_OS_NAME" = osx ] - then + if [ "$TRAVIS_OS_NAME" = osx ]; then # $RUNNER_OS == 'macOS' ? HOMEBREW_NO_AUTO_UPDATE=1 brew install gnupg2 fi @@ -126,16 +124,14 @@ EOL keyFile=.travis/signingkey.asc key=$1 iv=$2 - if [ "$key" -a "$iv" -a -f "$keyFile.enc" ] - then + if [ "$key" -a "$iv" -a -f "$keyFile.enc" ]; then # NB: Key and iv values were given as arguments. echo echo "== Decrypting GPG keypair ==" openssl aes-256-cbc -K "$key" -iv "$iv" -in "$keyFile.enc" -out "$keyFile" -d checkSuccess $? fi - if [ "$deployOK" -a -f "$keyFile" ] - then + if [ "$deployOK" -a -f "$keyFile" ]; then echo echo "== Importing GPG keypair ==" gpg --batch --fast-import "$keyFile" @@ -144,27 +140,25 @@ EOL # Run the build. BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' - if [ "$deployOK" -a "$TRAVIS_BRANCH" = master ] - then + if [ "$deployOK" -a "$TRAVIS_BRANCH" = master ]; then # on push event, so need further info echo echo "== Building and deploying master SNAPSHOT ==" mvn -Pdeploy-to-scijava $BUILD_ARGS deploy checkSuccess $? - elif [ "$deployOK" -a -f release.properties ] - then + elif [ "$deployOK" -a -f release.properties ]; then echo echo "== Cutting and deploying release version ==" mvn -B $BUILD_ARGS release:perform checkSuccess $? echo "== Invalidating SciJava Maven repository cache ==" curl -fsLO https://raw.githubusercontent.com/scijava/scijava-scripts/master/maven-helper.sh && - gav=$(sh maven-helper.sh gav-from-pom pom.xml) && - ga=${gav%:*} && - echo "--> Artifact to invalidate = $ga" && - echo "machine maven.scijava.org" > "$HOME/.netrc" && - echo " login travis" >> "$HOME/.netrc" && - echo " password $MAVEN_PASS" >> "$HOME/.netrc" && - sh maven-helper.sh invalidate-cache "$ga" + gav=$(sh maven-helper.sh gav-from-pom pom.xml) && + ga=${gav%:*} && + echo "--> Artifact to invalidate = $ga" && + echo "machine maven.scijava.org" >"$HOME/.netrc" && + echo " login travis" >>"$HOME/.netrc" && + echo " password $MAVEN_PASS" >>"$HOME/.netrc" && + sh maven-helper.sh invalidate-cache "$ga" checkSuccess $? else echo @@ -176,8 +170,7 @@ EOL fi # Configure conda environment, if one is needed. -if [ -f environment.yml ] -then +if [ -f environment.yml ]; then echo travis_fold:start:scijava-conda echo "= Conda setup =" @@ -186,7 +179,7 @@ then if [ ! -f "$condaSh" ]; then echo echo "== Installing conda ==" - if [ "$TRAVIS_PYTHON_VERSION" = "2.7" ]; then + if [ "$TRAVIS_PYTHON_VERSION" = "2.7" ]; then # ${{ matrix.python-version }} wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh else wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh @@ -199,9 +192,9 @@ then echo echo "== Updating conda ==" . "$condaSh" && - conda config --set always_yes yes --set changeps1 no && - conda update -q conda && - conda info -a + conda config --set always_yes yes --set changeps1 no && + conda update -q conda && + conda info -a checkSuccess $? echo @@ -209,15 +202,14 @@ then condaEnv=travis-scijava test -d "$condaDir/envs/$condaEnv" && condaAction=update || condaAction=create conda env "$condaAction" -n "$condaEnv" -f environment.yml && - conda activate "$condaEnv" + conda activate "$condaEnv" checkSuccess $? echo travis_fold:end:scijava-conda fi # Execute Jupyter notebooks. -if which jupyter >/dev/null 2>/dev/null -then +if which jupyter >/dev/null 2>/dev/null; then echo travis_fold:start:scijava-jupyter echo "= Jupyter notebooks =" # NB: This part is fiddly. We want to loop over files even with spaces, @@ -228,8 +220,7 @@ then # the final value of success upon completion, and then capture the # echoed value back into the parent shell's success variable. success=$(find . -name '*.ipynb' -print0 | { - while read -d $'\0' nbf - do + while read -d $'\0' nbf; do echo 1>&2 echo "== $nbf ==" 1>&2 jupyter nbconvert --execute --stdout "$nbf" >/dev/null From ff8de3d5e0f1a6f90f47d595972c30406a3efce9 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Wed, 31 Mar 2021 11:12:11 -0500 Subject: [PATCH 194/429] modified --- github-action-build.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/github-action-build.sh b/github-action-build.sh index 2dcfd6e..9f7fbab 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -29,11 +29,8 @@ if [ -f pom.xml ]; then # Populate the settings.xml configuration. mkdir -p "$HOME/.m2" settingsFile="$HOME/.m2/settings.xml" - customSettings=.travis/settings.xml # Since Travis files will be deleted, just manually set $settingsFile - if [ -f "$customSettings" ]; then - cp "$customSettings" "$settingsFile" - else # Might need to change travis + # Might need to change travis cat >"$settingsFile" < @@ -103,10 +100,10 @@ EOL ciOrg=${ciPrefix##*/} if [ "$TRAVIS_SECURE_ENV_VARS" != true ]; then # haven't found equivalent env var echo "No deploy -- secure environment variables not available" - elif [ "$TRAVIS_PULL_REQUEST" != false ]; then # ${{ github.event.number }} + elif [ ${github.event.number} != false ]; then echo "No deploy -- pull request detected" - elif [ "$TRAVIS_REPO_SLUG" != "$ciOrg/$ciRepo" ]; then # ${{ github.repository }} - echo "No deploy -- repository fork: $TRAVIS_REPO_SLUG != $ciOrg/$ciRepo" + elif [ ${ github.repository } != "$ciOrg/$ciRepo" ]; then + echo "No deploy -- repository fork: ${ github.repository } != $ciOrg/$ciRepo" # TODO: Detect travis-ci.org versus travis-ci.com? # Maybe above is not needed else @@ -116,11 +113,12 @@ EOL fi # Install GPG on OSX/macOS - if [ "$TRAVIS_OS_NAME" = osx ]; then # $RUNNER_OS == 'macOS' ? + if [ "$RUNNER_OS" == 'macOS' ]; then HOMEBREW_NO_AUTO_UPDATE=1 brew install gnupg2 fi # Import the GPG signing key. + # Do we still need this one? keyFile=.travis/signingkey.asc key=$1 iv=$2 @@ -140,7 +138,7 @@ EOL # Run the build. BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' - if [ "$deployOK" -a "$TRAVIS_BRANCH" = master ]; then # on push event, so need further info + if [ "$deployOK" -a "$TRAVIS_BRANCH" = master ]; then # TODO: on push event, so need further info echo echo "== Building and deploying master SNAPSHOT ==" mvn -Pdeploy-to-scijava $BUILD_ARGS deploy @@ -179,7 +177,7 @@ if [ -f environment.yml ]; then if [ ! -f "$condaSh" ]; then echo echo "== Installing conda ==" - if [ "$TRAVIS_PYTHON_VERSION" = "2.7" ]; then # ${{ matrix.python-version }} + if [ ${ matrix.python-version } = "2.7" ]; then wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh else wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh From cd3dc231d5773bcf989bfeba4cc97fb3cc3f4941 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Wed, 7 Apr 2021 23:19:50 -0500 Subject: [PATCH 195/429] changed commands to corresponding shell commands --- github-action-build.sh | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/github-action-build.sh b/github-action-build.sh index 9f7fbab..354d1a2 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -1,7 +1,8 @@ #!/bin/bash # -# travis-build.sh - A script to build and/or release SciJava-based projects. +# github-action-build.sh - A script to build and/or release SciJava-based projects +# automatically using GitHub Actions. # dir="$(dirname "$0")" @@ -17,8 +18,7 @@ checkSuccess() { # Build Maven projects. if [ -f pom.xml ]; then - echo travis_fold:start:scijava-maven - echo "= Maven build =" + echo ::group::"= Maven build =" # beginning of travis_fold (scijava-maven) echo echo "== Configuring Maven ==" @@ -29,8 +29,11 @@ if [ -f pom.xml ]; then # Populate the settings.xml configuration. mkdir -p "$HOME/.m2" settingsFile="$HOME/.m2/settings.xml" + customSettings=.travis/settings.xml # Since Travis files will be deleted, just manually set $settingsFile - # Might need to change travis + if [ -f "$customSettings" ]; then + cp "$customSettings" "$settingsFile" + else # Might need to change travis cat >"$settingsFile" < @@ -100,10 +103,10 @@ EOL ciOrg=${ciPrefix##*/} if [ "$TRAVIS_SECURE_ENV_VARS" != true ]; then # haven't found equivalent env var echo "No deploy -- secure environment variables not available" - elif [ ${github.event.number} != false ]; then + elif [ "$TRAVIS_PULL_REQUEST" != false ]; then # ${{ github.event.number }} echo "No deploy -- pull request detected" - elif [ ${ github.repository } != "$ciOrg/$ciRepo" ]; then - echo "No deploy -- repository fork: ${ github.repository } != $ciOrg/$ciRepo" + elif [ "$TRAVIS_REPO_SLUG" != "$ciOrg/$ciRepo" ]; then # ${{ github.repository }} + echo "No deploy -- repository fork: $TRAVIS_REPO_SLUG != $ciOrg/$ciRepo" # TODO: Detect travis-ci.org versus travis-ci.com? # Maybe above is not needed else @@ -113,12 +116,11 @@ EOL fi # Install GPG on OSX/macOS - if [ "$RUNNER_OS" == 'macOS' ]; then + if [ "$TRAVIS_OS_NAME" = osx ]; then # $RUNNER_OS == 'macOS' ? HOMEBREW_NO_AUTO_UPDATE=1 brew install gnupg2 fi # Import the GPG signing key. - # Do we still need this one? keyFile=.travis/signingkey.asc key=$1 iv=$2 @@ -138,7 +140,7 @@ EOL # Run the build. BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' - if [ "$deployOK" -a "$TRAVIS_BRANCH" = master ]; then # TODO: on push event, so need further info + if [ "$deployOK" -a "$TRAVIS_BRANCH" = master ]; then # on push event, so need further info echo echo "== Building and deploying master SNAPSHOT ==" mvn -Pdeploy-to-scijava $BUILD_ARGS deploy @@ -164,20 +166,19 @@ EOL mvn $BUILD_ARGS install javadoc:javadoc checkSuccess $? fi - echo travis_fold:end:scijava-maven + echo ::endgroup:: # end of travis_fold (scijava-maven) fi # Configure conda environment, if one is needed. if [ -f environment.yml ]; then - echo travis_fold:start:scijava-conda - echo "= Conda setup =" + echo ::group::"= Conda setup =" # beginning of travis_fold (scijava-conda) condaDir=$HOME/miniconda condaSh=$condaDir/etc/profile.d/conda.sh if [ ! -f "$condaSh" ]; then echo echo "== Installing conda ==" - if [ ${ matrix.python-version } = "2.7" ]; then + if [ "$TRAVIS_PYTHON_VERSION" = "2.7" ]; then # ${{ matrix.python-version }} wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh else wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh @@ -203,13 +204,12 @@ if [ -f environment.yml ]; then conda activate "$condaEnv" checkSuccess $? - echo travis_fold:end:scijava-conda + echo ::endgroup:: # end of scijava-conda fi # Execute Jupyter notebooks. if which jupyter >/dev/null 2>/dev/null; then - echo travis_fold:start:scijava-jupyter - echo "= Jupyter notebooks =" + echo ::group::"= Jupyter notebooks =" # beginning of travis_fold (scijava-jupyter) # NB: This part is fiddly. We want to loop over files even with spaces, # so we use the "find ... -print0 | while read $'\0' ..." idiom. # However, that runs the piped expression in a subshell, which means @@ -226,7 +226,7 @@ if which jupyter >/dev/null 2>/dev/null; then done echo $success }) - echo travis_fold:end:scijava-jupyter + echo ::endgroup:: # end of travis_fold (scijava-jupyter) fi exit $success From e5e54750ecf96312f4a905d003d93120832c15e4 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Thu, 8 Apr 2021 02:41:23 -0500 Subject: [PATCH 196/429] changed github-action-build.sh file --- github-action-build.sh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/github-action-build.sh b/github-action-build.sh index 354d1a2..bc1b96e 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -29,7 +29,7 @@ if [ -f pom.xml ]; then # Populate the settings.xml configuration. mkdir -p "$HOME/.m2" settingsFile="$HOME/.m2/settings.xml" - customSettings=.travis/settings.xml + customSettings=.gh-action/settings.xml # Since Travis files will be deleted, just manually set $settingsFile if [ -f "$customSettings" ]; then cp "$customSettings" "$settingsFile" @@ -101,14 +101,13 @@ EOL ciRepo=${ciURL##*/} ciPrefix=${ciURL%/*} ciOrg=${ciPrefix##*/} - if [ "$TRAVIS_SECURE_ENV_VARS" != true ]; then # haven't found equivalent env var + if [ "${github.secret.github_token}" != true ]; then echo "No deploy -- secure environment variables not available" - elif [ "$TRAVIS_PULL_REQUEST" != false ]; then # ${{ github.event.number }} + elif [ "${github.event.number}" != false ]; then echo "No deploy -- pull request detected" - elif [ "$TRAVIS_REPO_SLUG" != "$ciOrg/$ciRepo" ]; then # ${{ github.repository }} - echo "No deploy -- repository fork: $TRAVIS_REPO_SLUG != $ciOrg/$ciRepo" + elif [ "${github.repository}" != "$ciOrg/$ciRepo" ]; then + echo "No deploy -- repository fork: ${github.repository} != $ciOrg/$ciRepo" # TODO: Detect travis-ci.org versus travis-ci.com? - # Maybe above is not needed else echo "All checks passed for artifact deployment" deployOK=1 @@ -116,12 +115,12 @@ EOL fi # Install GPG on OSX/macOS - if [ "$TRAVIS_OS_NAME" = osx ]; then # $RUNNER_OS == 'macOS' ? + if [ "${runner.os}" = 'macOS' ]; then HOMEBREW_NO_AUTO_UPDATE=1 brew install gnupg2 fi # Import the GPG signing key. - keyFile=.travis/signingkey.asc + keyFile=.gh-action/signingkey.asc key=$1 iv=$2 if [ "$key" -a "$iv" -a -f "$keyFile.enc" ]; then @@ -140,7 +139,7 @@ EOL # Run the build. BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' - if [ "$deployOK" -a "$TRAVIS_BRANCH" = master ]; then # on push event, so need further info + if [ "$deployOK" -a "${github.head_ref}" = master ]; then # on push event, so need further info echo echo "== Building and deploying master SNAPSHOT ==" mvn -Pdeploy-to-scijava $BUILD_ARGS deploy @@ -178,7 +177,7 @@ if [ -f environment.yml ]; then if [ ! -f "$condaSh" ]; then echo echo "== Installing conda ==" - if [ "$TRAVIS_PYTHON_VERSION" = "2.7" ]; then # ${{ matrix.python-version }} + if [ "${matrix.python-version}" = "2.7" ]; then wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh else wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh From f685749c1f0e22991a50de52d1c95a6de467eccd Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 9 Apr 2021 11:59:23 -0500 Subject: [PATCH 197/429] Add script to find duplicate classes across JARs This commit is dedicated to Phillip Hanslovsky. --- find-duplicate-classes.py | 42 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 find-duplicate-classes.py diff --git a/find-duplicate-classes.py b/find-duplicate-classes.py new file mode 100755 index 0000000..d604c78 --- /dev/null +++ b/find-duplicate-classes.py @@ -0,0 +1,42 @@ +#!/bin/env python + +import os, subprocess + +def extract_classes(f): + lines = subprocess.check_output(['jar', 'tf', f]).split() + result = set() + for line in lines: + v = line.decode('utf-8').strip() + if v.endswith('.class'): + result.add(v) + return result + +print('Reading JAR files... ', end='', flush=True) +paths = [] +for root, dirs, files in os.walk("."): + for name in files: + if not name.lower().endswith('.jar'): continue + paths.append(os.path.join(root, name)) + +classes = {} +count = 0 +perc = '' +for path in paths: + count += 1 + classes[path] = extract_classes(path) + print('\b' * len(perc), end='') + perc = str(round(100 * count / len(paths))) + '% ' + print(perc, end='', flush=True) + +exceptions = ['module-info.class'] +print('Scanning for duplicate classes..') +for jar1 in classes: + for jar2 in classes: + if jar1 == jar2: continue + dups = classes[jar1].intersection(classes[jar2]) + for exc in exceptions: + if exc in dups: dups.remove(exc) + if len(dups) > 0: + print(f'==> {jar1} and {jar2} have duplicates! E.g. {next(iter(dups))}') + +print('Done!') From 191553c7a912b21816a1375b41898bde5189375b Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sat, 10 Apr 2021 16:37:39 -0500 Subject: [PATCH 198/429] find-duplicate-classes: improve the output --- find-duplicate-classes.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/find-duplicate-classes.py b/find-duplicate-classes.py index d604c78..6050d40 100755 --- a/find-duplicate-classes.py +++ b/find-duplicate-classes.py @@ -7,16 +7,17 @@ def extract_classes(f): result = set() for line in lines: v = line.decode('utf-8').strip() - if v.endswith('.class'): + if v.endswith('.class') and not v.endswith('module-info.class'): result.add(v) return result print('Reading JAR files... ', end='', flush=True) paths = [] -for root, dirs, files in os.walk("."): +for root, dirs, files in os.walk('.'): for name in files: if not name.lower().endswith('.jar'): continue paths.append(os.path.join(root, name)) +paths.sort() classes = {} count = 0 @@ -25,18 +26,22 @@ def extract_classes(f): count += 1 classes[path] = extract_classes(path) print('\b' * len(perc), end='') - perc = str(round(100 * count / len(paths))) + '% ' + perc = str(round(100 * count / len(paths))) + '% ' print(perc, end='', flush=True) -exceptions = ['module-info.class'] -print('Scanning for duplicate classes..') -for jar1 in classes: - for jar2 in classes: - if jar1 == jar2: continue - dups = classes[jar1].intersection(classes[jar2]) - for exc in exceptions: - if exc in dups: dups.remove(exc) +print() +print('Scanning for duplicate classes...') +for i1 in range(len(paths)): + p1 = paths[i1] + duplist = [] + for i2 in range(i1 + 1, len(paths)): + p2 = paths[i2] + dups = classes[p1].intersection(classes[p2]) if len(dups) > 0: - print(f'==> {jar1} and {jar2} have duplicates! E.g. {next(iter(dups))}') + duplist.append(f'==> {p2} (e.g. {next(iter(dups))})') + if len(duplist) > 0: + print(p1) + for line in duplist: + print(line) print('Done!') From 5796a488845f244abf831e6c0084e2fd9d028f6b Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Thu, 29 Apr 2021 02:26:57 -0500 Subject: [PATCH 199/429] deleted unnecessary comments --- github-action-build.sh | 25 ++-- github-actionify.sh | 280 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 292 insertions(+), 13 deletions(-) create mode 100644 github-actionify.sh diff --git a/github-action-build.sh b/github-action-build.sh index bc1b96e..ee36bd6 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -18,7 +18,7 @@ checkSuccess() { # Build Maven projects. if [ -f pom.xml ]; then - echo ::group::"= Maven build =" # beginning of travis_fold (scijava-maven) + echo ::group::"= Maven build =" echo echo "== Configuring Maven ==" @@ -30,21 +30,20 @@ if [ -f pom.xml ]; then mkdir -p "$HOME/.m2" settingsFile="$HOME/.m2/settings.xml" customSettings=.gh-action/settings.xml - # Since Travis files will be deleted, just manually set $settingsFile if [ -f "$customSettings" ]; then cp "$customSettings" "$settingsFile" - else # Might need to change travis + else cat >"$settingsFile" < scijava.releases - travis + scijava-ci \${env.MAVEN_PASS} scijava.snapshots - travis + scijava-ci \${env.MAVEN_PASS} @@ -139,7 +138,7 @@ EOL # Run the build. BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' - if [ "$deployOK" -a "${github.head_ref}" = master ]; then # on push event, so need further info + if [ "$deployOK" -a "${github.head_ref}" = master ]; then echo echo "== Building and deploying master SNAPSHOT ==" mvn -Pdeploy-to-scijava $BUILD_ARGS deploy @@ -155,7 +154,7 @@ EOL ga=${gav%:*} && echo "--> Artifact to invalidate = $ga" && echo "machine maven.scijava.org" >"$HOME/.netrc" && - echo " login travis" >>"$HOME/.netrc" && + echo " login scijava-ci" >>"$HOME/.netrc" && echo " password $MAVEN_PASS" >>"$HOME/.netrc" && sh maven-helper.sh invalidate-cache "$ga" checkSuccess $? @@ -165,12 +164,12 @@ EOL mvn $BUILD_ARGS install javadoc:javadoc checkSuccess $? fi - echo ::endgroup:: # end of travis_fold (scijava-maven) + echo ::endgroup:: fi # Configure conda environment, if one is needed. if [ -f environment.yml ]; then - echo ::group::"= Conda setup =" # beginning of travis_fold (scijava-conda) + echo ::group::"= Conda setup =" condaDir=$HOME/miniconda condaSh=$condaDir/etc/profile.d/conda.sh @@ -197,18 +196,18 @@ if [ -f environment.yml ]; then echo echo "== Configuring environment ==" - condaEnv=travis-scijava + condaEnv=github-scijava test -d "$condaDir/envs/$condaEnv" && condaAction=update || condaAction=create conda env "$condaAction" -n "$condaEnv" -f environment.yml && conda activate "$condaEnv" checkSuccess $? - echo ::endgroup:: # end of scijava-conda + echo ::endgroup:: fi # Execute Jupyter notebooks. if which jupyter >/dev/null 2>/dev/null; then - echo ::group::"= Jupyter notebooks =" # beginning of travis_fold (scijava-jupyter) + echo ::group::"= Jupyter notebooks =" # NB: This part is fiddly. We want to loop over files even with spaces, # so we use the "find ... -print0 | while read $'\0' ..." idiom. # However, that runs the piped expression in a subshell, which means @@ -225,7 +224,7 @@ if which jupyter >/dev/null 2>/dev/null; then done echo $success }) - echo ::endgroup:: # end of travis_fold (scijava-jupyter) + echo ::endgroup:: fi exit $success diff --git a/github-actionify.sh b/github-actionify.sh new file mode 100644 index 0000000..87e4ab2 --- /dev/null +++ b/github-actionify.sh @@ -0,0 +1,280 @@ +#!/bin/sh + +# travisify.sh +# +# Script for enabling or updating Travis CI builds for a given repository. + +#set -e + +dir="$(dirname "$0")" + +travisDir=.travis +travisConfig=.travis.yml +travisBuildScript=$travisDir/build.sh +travisSettingsFile=$travisDir/settings.xml +travisNotifyScript=$travisDir/notify.sh +credentialsDir=$HOME/.scijava/credentials +varsFile=$credentialsDir/vars +signingKeySourceFile=$credentialsDir/scijava-ci-signing.asc +signingKeyDestFile=$travisDir/signingkey.asc +pomMinVersion='17.1.1' +tmpFile=travisify.tmp + +info() { echo "- $@"; } +warn() { echo "[WARNING] $@" 1>&2; } +err() { echo "[ERROR] $@" 1>&2; } +die() { err "$@"; exit 1; } + +check() { + for tool in $@ + do + which "$tool" >/dev/null || + die "The '$tool' utility is required but not found" + done +} + +var() { + grep "^$1=" "$varsFile" || + die "$1 not found in $varsFile" +} + +update() { + file=$1 + msg=$2 + exe=$3 + test "$msg" || msg="Travis: update $file" + if [ -e "$file" ] + then + if diff -q "$file" "$tmpFile" >/dev/null + then + info "$file is already OK" + else + info "Updating $file" + $EXEC rm -rf "$file" + $EXEC mv -f "$tmpFile" "$file" + fi + else + info "Creating $file" + $EXEC mkdir -p "$(dirname "$file")" + $EXEC mv "$tmpFile" "$file" + fi + rm -rf "$tmpFile" + $EXEC git add "$file" + if [ -n "$exe" ] + then + info "Adding execute permission to $file" + $EXEC git update-index --chmod=+x "$file" + fi + $EXEC git diff-index --quiet HEAD -- || $EXEC git commit -m "$msg" +} + +process() { + cd "$1" + + # -- Git sanity checks -- + + repoSlug=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="scm"]/*[local-name()="connection"]' pom.xml|sed 's_.*github.com[:/]\(.*\)<.*_\1_') + test "$repoSlug" && info "Repository = $repoSlug" || die 'Could not determine GitHub repository slug' + case "$repoSlug" in + *.git) + die "GitHub repository slug ('$repoSlug') ends in '.git'; please fix the POM" + ;; + esac + git fetch >/dev/null + git diff-index --quiet HEAD -- || die "Dirty working copy" + currentBranch=$(git rev-parse --abbrev-ref HEAD) + upstreamBranch=$(git rev-parse --abbrev-ref --symbolic-full-name @{u}) + remote=${upstreamBranch%/*} + defaultBranch=$(git remote show "$remote" | grep "HEAD branch" | sed 's/.*: //') + test "$currentBranch" = "$defaultBranch" || die "Non-default branch: $currentBranch" + git merge --ff --ff-only 'HEAD@{u}' >/dev/null || + die "Cannot fast forward (local diverging?)" +# test "$(git rev-parse HEAD)" = "$(git rev-parse 'HEAD@{u}')" || +# die "Mismatch with upstream branch (local ahead?)" + + # -- POM sanity checks -- + + parent=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="parent"]/*[local-name()="artifactId"]' pom.xml|sed 's/[^>]*>//'|sed 's/<.*//') + if [ -z "$SKIP_PARENT_CHECK" ] + then + test "$parent" = "pom-scijava" || + die "Not pom-scijava parent: $parent. Run with -p flag to skip this check." + fi + + domain=$(grep "travis-ci\.[a-z]*/$repoSlug" pom.xml 2>/dev/null | sed 's/.*\(travis-ci\.[a-z]*\).*/\1/') + test "$domain" && + info "Detected domain from pom.xml: $domain" || + die "No valid ciManagement section in pom.xml. Please add one, then try again." + + case $domain in + travis-ci.com) mode=pro;; + travis-ci.org) mode=org;; + *) die "Unsupported domain: $domain";; + esac + + # -- Travis sanity checks -- + + test -e "$travisDir" -a ! -d "$travisDir" && die "$travisDir is not a directory" + test -e "$travisConfig" -a ! -f "$travisConfig" && die "$travisConfig is not a regular file" + test -e "$travisConfig" && warn "$travisConfig already exists" + test -e "$travisBuildScript" && warn "$travisBuildScript already exists" + + # -- Do things -- + + # Add/update the Travis configuration file. + cat >"$tmpFile" <"$tmpFile" <]*>//'|sed 's/<.*//') + # HACK: Using a lexicographic comparison here is imperfect. + if [ "$version" \< "$pomMinVersion" ] + then + info 'Upgrading pom-scijava version' + sed "s|^ $version$| $pomMinVersion|" pom.xml >"$tmpFile" + update pom.xml "POM: update pom-scijava parent to $pomMinVersion" + else + info "Version of pom-scijava ($version) is OK" + fi + fi + + # ensure section is present + releaseProfile=$(grep '' pom.xml 2>/dev/null | sed 's/[^>]*>//' | sed 's/<.*//') + if [ "$releaseProfile" ] + then + test "$releaseProfile" = 'deploy-to-scijava' || + warn "Unknown release profile: $releaseProfile" + else + info 'Adding property' + cp pom.xml "$tmpFile" + perl -0777 -i -pe 's/(\n\t<\/properties>\n)/\n\n\t\t\n\t\tdeploy-to-scijava<\/releaseProfiles>\1/igs' "$tmpFile" + update pom.xml 'POM: deploy releases to the SciJava repository' + fi + + # update the README + if grep -q "travis-ci\.[a-zA-Z0-9/_-]*\.svg" README.md >/dev/null 2>&1 + then + info "Updating README.md Travis badge" + sed "s|travis-ci\.[a-zA-Z0-9/_-]*|$domain/$repoSlug|g" README.md >"$tmpFile" + update README.md 'Travis: fix README.md badge link' + else + info "Adding Travis badge to README.md" + echo "[![](https://$domain/$repoSlug.svg?branch=$defaultBranch)](https://$domain/$repoSlug)" >"$tmpFile" + echo >>"$tmpFile" + test -f README.md && cat README.md >>"$tmpFile" + update README.md 'Travis: add badge to README.md' + fi + + # encrypt key/value pairs in variables file + if [ -f "$varsFile" ] + then + while read p; do + # Skip comments. (Cannot use ${p:0:1} because it's bash-specific.) + case "$p" in + '#'*) continue;; + esac + info "Encrypting ${p%%=*}" + yes | $EXEC travis encrypt --$mode "$p" --add env.global --repo "$repoSlug" + test $? -eq 0 || die "Failed to encrypt variable '$p'" + done <"$varsFile" + $EXEC git commit "$travisConfig" -m "Travis: add encrypted environment variables" + else + warn "No $varsFile found. Travis will not have any environment variables set!" + fi + + # encrypt GPG keypair + if [ -f "$signingKeySourceFile" ] + then + info "Encrypting $signingKeyDestFile" + if [ -z "$EXEC" ] + then + rm -f "$signingKeyDestFile.enc" + encryptOutput=$(travis encrypt-file --$mode "$signingKeySourceFile" "$signingKeyDestFile.enc" --repo "$repoSlug") + test $? -eq 0 || die "Failed to encrypt signing key." + encryptResult=$(echo "$encryptOutput" | grep openssl) + test "$encryptResult" || die "No openssl variables emitted." + key=$(echo "$encryptResult" | cut -d' ' -f4) + iv=$(echo "$encryptResult" | cut -d' ' -f6) + sed -i.bak "s/\(sh travis-build.sh\)/\1 $key $iv/" "$travisBuildScript" + rm -f "$travisBuildScript.bak" + git add "$travisBuildScript" "$signingKeyDestFile.enc" + git commit -m "Travis: add encrypted GPG signing keypair" + fi + else + warn "No $signingKeySourceFile found. Travis will not be able to do GPG signing!" + fi +} + +test -d "$credentialsDir" || + die "This script requires configuration stored in $credentialsDir,\n" \ + "including $varsFile for needed environment variables,\n" \ + "and $signingKeySourceFile for signing of artifacts.\n" \ + "Please contact a SciJava administrator to receive a copy of this content." + +# check prerequisites +check git sed cut perl xmllint travis + +# parse arguments +EXEC=: +SKIP_PARENT_CHECK= +while test $# -gt 0 +do + case "$1" in + -f) EXEC=;; + -p) SKIP_PARENT_CHECK=true;; + --) break;; + -*) echo "Ignoring unknown option: $1" >&2; break;; + *) break;; + esac + shift +done + +test "$EXEC" && warn "Simulation only. Run with -f flag to go for real." + +# process arguments +if [ $# -gt 0 ] +then + for d in $@ + do ( + echo "[$d]" + process "$d" + ) done +else + process . +fi From 2cf6a93015223f86b6b0e9c9d3dc882b20e44433 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Fri, 30 Apr 2021 09:07:18 -0500 Subject: [PATCH 200/429] test_var --- github-action-build.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/github-action-build.sh b/github-action-build.sh index ee36bd6..3b487dd 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -56,6 +56,7 @@ EOL # NB: Use maven.scijava.org instead of Central if defined in repositories. # This hopefully avoids intermittent "ReasonPhrase:Forbidden" errors # when the Travis build pings Maven Central; see travis-ci/travis-ci#6593. + # TODO: Try removing and see if error comes up grep -A 2 '' pom.xml | grep -q 'maven.scijava.org' && cat >>"$settingsFile" < @@ -100,7 +101,10 @@ EOL ciRepo=${ciURL##*/} ciPrefix=${ciURL%/*} ciOrg=${ciPrefix##*/} - if [ "${github.secret.github_token}" != true ]; then + if [ $test_var == true ]; then + echo "----TEST COMPLETE----" + # TODO: try ${ secret.GITHUB_TOKEN } / double braces? + if [ "${ github.secret.github_token}" != true ]; then echo "No deploy -- secure environment variables not available" elif [ "${github.event.number}" != false ]; then echo "No deploy -- pull request detected" From 3b7d8a9defd8bf38935aa1d717c9bd2ab6cc7783 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 02:40:03 -0500 Subject: [PATCH 201/429] double curly braces for github --- github-action-build.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/github-action-build.sh b/github-action-build.sh index ee36bd6..8468617 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -100,12 +100,12 @@ EOL ciRepo=${ciURL##*/} ciPrefix=${ciURL%/*} ciOrg=${ciPrefix##*/} - if [ "${github.secret.github_token}" != true ]; then + if [ ${{github.secret.github_token}} != true ]; then echo "No deploy -- secure environment variables not available" - elif [ "${github.event.number}" != false ]; then + elif [ ${{github.event.number}} != false ]; then echo "No deploy -- pull request detected" - elif [ "${github.repository}" != "$ciOrg/$ciRepo" ]; then - echo "No deploy -- repository fork: ${github.repository} != $ciOrg/$ciRepo" + elif [ ${{github.repository}} != "$ciOrg/$ciRepo" ]; then + echo "No deploy -- repository fork: ${{github.repository}} != $ciOrg/$ciRepo" # TODO: Detect travis-ci.org versus travis-ci.com? else echo "All checks passed for artifact deployment" @@ -138,7 +138,7 @@ EOL # Run the build. BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' - if [ "$deployOK" -a "${github.head_ref}" = master ]; then + if [ "$deployOK" -a ${{github.head_ref}} = master ]; then echo echo "== Building and deploying master SNAPSHOT ==" mvn -Pdeploy-to-scijava $BUILD_ARGS deploy From a037a5386728d6b8bf685773ad12840d83e6cb68 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 02:49:03 -0500 Subject: [PATCH 202/429] deleted unexpected eof --- github-action-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-action-build.sh b/github-action-build.sh index b7616f4..e44628a 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -231,4 +231,4 @@ if which jupyter >/dev/null 2>/dev/null; then echo ::endgroup:: fi -exit $success +exit $success \ No newline at end of file From 42e989e1afaa3fb2a46b01e8f94dee747e36505d Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 02:53:36 -0500 Subject: [PATCH 203/429] deleted double curly braces --- github-action-build.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/github-action-build.sh b/github-action-build.sh index e44628a..55eedaa 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -106,10 +106,10 @@ EOL # TODO: try ${ secret.GITHUB_TOKEN } / double braces? if [ "${ github.secret.github_token}" != true ]; then echo "No deploy -- secure environment variables not available" - elif [ ${{github.event.number}} != false ]; then + elif [ ${github.event.number} != false ]; then echo "No deploy -- pull request detected" - elif [ ${{github.repository}} != "$ciOrg/$ciRepo" ]; then - echo "No deploy -- repository fork: ${{github.repository}} != $ciOrg/$ciRepo" + elif [ ${github.repository} != "$ciOrg/$ciRepo" ]; then + echo "No deploy -- repository fork: ${github.repository} != $ciOrg/$ciRepo" # TODO: Detect travis-ci.org versus travis-ci.com? else echo "All checks passed for artifact deployment" @@ -142,7 +142,7 @@ EOL # Run the build. BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' - if [ "$deployOK" -a ${{github.head_ref}} = master ]; then + if [ "$deployOK" -a ${github.head_ref} = master ]; then echo echo "== Building and deploying master SNAPSHOT ==" mvn -Pdeploy-to-scijava $BUILD_ARGS deploy @@ -231,4 +231,4 @@ if which jupyter >/dev/null 2>/dev/null; then echo ::endgroup:: fi -exit $success \ No newline at end of file +exit $success From e8958e3c7cda6db10fc9ef8b557779de82b5261d Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 02:56:46 -0500 Subject: [PATCH 204/429] added after to avoid unexpected eof --- github-action-build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/github-action-build.sh b/github-action-build.sh index 55eedaa..9b90c24 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -103,6 +103,7 @@ EOL ciOrg=${ciPrefix##*/} if [ $test_var == true ]; then echo "----TEST COMPLETE----" + fi # TODO: try ${ secret.GITHUB_TOKEN } / double braces? if [ "${ github.secret.github_token}" != true ]; then echo "No deploy -- secure environment variables not available" From 89b227c0791b138ef665decbdf5b62792802e677 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 03:04:51 -0500 Subject: [PATCH 205/429] commented out test section to see what caused unexpected eof error --- github-action-build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/github-action-build.sh b/github-action-build.sh index 9b90c24..b4cbf36 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -101,9 +101,9 @@ EOL ciRepo=${ciURL##*/} ciPrefix=${ciURL%/*} ciOrg=${ciPrefix##*/} - if [ $test_var == true ]; then - echo "----TEST COMPLETE----" - fi + # if [ $test_var == true ]; then + # echo "----TEST COMPLETE----" + # fi # TODO: try ${ secret.GITHUB_TOKEN } / double braces? if [ "${ github.secret.github_token}" != true ]; then echo "No deploy -- secure environment variables not available" From 62c670f7884026cf1be780fdbf3d3872c2f13dac Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 04:00:05 -0500 Subject: [PATCH 206/429] changed to --- github-action-build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/github-action-build.sh b/github-action-build.sh index b4cbf36..e56c014 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -101,9 +101,9 @@ EOL ciRepo=${ciURL##*/} ciPrefix=${ciURL%/*} ciOrg=${ciPrefix##*/} - # if [ $test_var == true ]; then - # echo "----TEST COMPLETE----" - # fi + if [ ${test_var} != false ]; then + echo "----TEST COMPLETE----" + fi # TODO: try ${ secret.GITHUB_TOKEN } / double braces? if [ "${ github.secret.github_token}" != true ]; then echo "No deploy -- secure environment variables not available" From ee76878765f206f1f459a641aa71e769a6947f8b Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 04:07:56 -0500 Subject: [PATCH 207/429] added quotation marks --- github-action-build.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/github-action-build.sh b/github-action-build.sh index e56c014..d520245 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -101,16 +101,16 @@ EOL ciRepo=${ciURL##*/} ciPrefix=${ciURL%/*} ciOrg=${ciPrefix##*/} - if [ ${test_var} != false ]; then + if [ "${test_var}" != false ]; then echo "----TEST COMPLETE----" fi # TODO: try ${ secret.GITHUB_TOKEN } / double braces? - if [ "${ github.secret.github_token}" != true ]; then + if [ "${secret.GITHUB_TOKEN}" != true ]; then echo "No deploy -- secure environment variables not available" - elif [ ${github.event.number} != false ]; then + elif [ "${github.event.number}" != false ]; then echo "No deploy -- pull request detected" - elif [ ${github.repository} != "$ciOrg/$ciRepo" ]; then - echo "No deploy -- repository fork: ${github.repository} != $ciOrg/$ciRepo" + elif [ "${github.repository}" != "$ciOrg/$ciRepo" ]; then + echo "No deploy -- repository fork: "${github.repository}"" != $ciOrg/$ciRepo" # TODO: Detect travis-ci.org versus travis-ci.com? else echo "All checks passed for artifact deployment" @@ -143,7 +143,7 @@ EOL # Run the build. BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' - if [ "$deployOK" -a ${github.head_ref} = master ]; then + if [ "$deployOK" -a "${github.head_ref}"" = master ]; then echo echo "== Building and deploying master SNAPSHOT ==" mvn -Pdeploy-to-scijava $BUILD_ARGS deploy From cd5312e0369461df14cc4f48f2e6c465ab67aad0 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 04:10:27 -0500 Subject: [PATCH 208/429] deleted errornous quotation mark --- github-action-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-action-build.sh b/github-action-build.sh index d520245..c04ebb5 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -143,7 +143,7 @@ EOL # Run the build. BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' - if [ "$deployOK" -a "${github.head_ref}"" = master ]; then + if [ "$deployOK" -a "${github.head_ref}" = master ]; then echo echo "== Building and deploying master SNAPSHOT ==" mvn -Pdeploy-to-scijava $BUILD_ARGS deploy From 6ac10f53e62c80ff2cab0bf1b9b7df81c751d5c4 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 04:18:21 -0500 Subject: [PATCH 209/429] commit to refresh --- github-action-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-action-build.sh b/github-action-build.sh index c04ebb5..d33436d 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -143,7 +143,7 @@ EOL # Run the build. BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' - if [ "$deployOK" -a "${github.head_ref}" = master ]; then + if [ "$deployOK" -a ${github.head_ref} = master ]; then echo echo "== Building and deploying master SNAPSHOT ==" mvn -Pdeploy-to-scijava $BUILD_ARGS deploy From 6e5c05b6fb7118a7c36b1fe0c49c619aacad8954 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 04:18:40 -0500 Subject: [PATCH 210/429] commit to refresh --- github-action-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-action-build.sh b/github-action-build.sh index d33436d..c04ebb5 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -143,7 +143,7 @@ EOL # Run the build. BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' - if [ "$deployOK" -a ${github.head_ref} = master ]; then + if [ "$deployOK" -a "${github.head_ref}" = master ]; then echo echo "== Building and deploying master SNAPSHOT ==" mvn -Pdeploy-to-scijava $BUILD_ARGS deploy From 15d59463e2752f12382fa7464ed78a637817eb58 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 04:21:40 -0500 Subject: [PATCH 211/429] fixed addition quotation mark --- github-action-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-action-build.sh b/github-action-build.sh index c04ebb5..27c2d0b 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -110,7 +110,7 @@ EOL elif [ "${github.event.number}" != false ]; then echo "No deploy -- pull request detected" elif [ "${github.repository}" != "$ciOrg/$ciRepo" ]; then - echo "No deploy -- repository fork: "${github.repository}"" != $ciOrg/$ciRepo" + echo "No deploy -- repository fork: "${github.repository}" != $ciOrg/$ciRepo" # TODO: Detect travis-ci.org versus travis-ci.com? else echo "All checks passed for artifact deployment" From bda81c6f43deba8ef735c9582ba228e99f66c891 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 04:27:34 -0500 Subject: [PATCH 212/429] restore changes --- github-action-build.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/github-action-build.sh b/github-action-build.sh index 27c2d0b..ee36bd6 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -56,7 +56,6 @@ EOL # NB: Use maven.scijava.org instead of Central if defined in repositories. # This hopefully avoids intermittent "ReasonPhrase:Forbidden" errors # when the Travis build pings Maven Central; see travis-ci/travis-ci#6593. - # TODO: Try removing and see if error comes up grep -A 2 '' pom.xml | grep -q 'maven.scijava.org' && cat >>"$settingsFile" < @@ -101,16 +100,12 @@ EOL ciRepo=${ciURL##*/} ciPrefix=${ciURL%/*} ciOrg=${ciPrefix##*/} - if [ "${test_var}" != false ]; then - echo "----TEST COMPLETE----" - fi - # TODO: try ${ secret.GITHUB_TOKEN } / double braces? - if [ "${secret.GITHUB_TOKEN}" != true ]; then + if [ "${github.secret.github_token}" != true ]; then echo "No deploy -- secure environment variables not available" elif [ "${github.event.number}" != false ]; then echo "No deploy -- pull request detected" elif [ "${github.repository}" != "$ciOrg/$ciRepo" ]; then - echo "No deploy -- repository fork: "${github.repository}" != $ciOrg/$ciRepo" + echo "No deploy -- repository fork: ${github.repository} != $ciOrg/$ciRepo" # TODO: Detect travis-ci.org versus travis-ci.com? else echo "All checks passed for artifact deployment" From cc7e39d67ceb2749e906b28f916ce09fd4a3bc07 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 04:31:59 -0500 Subject: [PATCH 213/429] readded test codes --- github-action-build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/github-action-build.sh b/github-action-build.sh index ee36bd6..96e25f3 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -100,6 +100,9 @@ EOL ciRepo=${ciURL##*/} ciPrefix=${ciURL%/*} ciOrg=${ciPrefix##*/} + if [ "${test_var}" == true ]; then + echo "----TEST COMPLETE----" + # TODO: try ${ secret.GITHUB_TOKEN } / double braces? if [ "${github.secret.github_token}" != true ]; then echo "No deploy -- secure environment variables not available" elif [ "${github.event.number}" != false ]; then From de0782bc971febf501f6caa4a0265c56fdc34c11 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 04:34:17 -0500 Subject: [PATCH 214/429] added fi --- github-action-build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/github-action-build.sh b/github-action-build.sh index 96e25f3..6a8fd9f 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -102,6 +102,7 @@ EOL ciOrg=${ciPrefix##*/} if [ "${test_var}" == true ]; then echo "----TEST COMPLETE----" + fi # TODO: try ${ secret.GITHUB_TOKEN } / double braces? if [ "${github.secret.github_token}" != true ]; then echo "No deploy -- secure environment variables not available" From 954219874f3f7ccf4d39529a180368bc50e9b6f3 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 05:03:44 -0500 Subject: [PATCH 215/429] test 15 --- github-action-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-action-build.sh b/github-action-build.sh index 6a8fd9f..de3d6f9 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -100,7 +100,7 @@ EOL ciRepo=${ciURL##*/} ciPrefix=${ciURL%/*} ciOrg=${ciPrefix##*/} - if [ "${test_var}" == true ]; then + if [ "${test_var}" = true ]; then echo "----TEST COMPLETE----" fi # TODO: try ${ secret.GITHUB_TOKEN } / double braces? From c88691882914d9906ebc97c3975bd3c99ef0dc1b Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 05:25:36 -0500 Subject: [PATCH 216/429] test 16 --- github-action-build.sh | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/github-action-build.sh b/github-action-build.sh index de3d6f9..d3a3a51 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -100,16 +100,13 @@ EOL ciRepo=${ciURL##*/} ciPrefix=${ciURL%/*} ciOrg=${ciPrefix##*/} - if [ "${test_var}" = true ]; then - echo "----TEST COMPLETE----" - fi # TODO: try ${ secret.GITHUB_TOKEN } / double braces? - if [ "${github.secret.github_token}" != true ]; then + if [ "${secure_env}" != true ]; then echo "No deploy -- secure environment variables not available" - elif [ "${github.event.number}" != false ]; then + elif [ "${pull_request}" != false ]; then echo "No deploy -- pull request detected" - elif [ "${github.repository}" != "$ciOrg/$ciRepo" ]; then - echo "No deploy -- repository fork: ${github.repository} != $ciOrg/$ciRepo" + elif [ "${repo_fork}" != "$ciOrg/$ciRepo" ]; then + echo "No deploy -- repository fork: ${repo_fork}} != $ciOrg/$ciRepo" # TODO: Detect travis-ci.org versus travis-ci.com? else echo "All checks passed for artifact deployment" @@ -118,7 +115,7 @@ EOL fi # Install GPG on OSX/macOS - if [ "${runner.os}" = 'macOS' ]; then + if [ "${runner_os}" = 'macOS' ]; then HOMEBREW_NO_AUTO_UPDATE=1 brew install gnupg2 fi @@ -142,7 +139,7 @@ EOL # Run the build. BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' - if [ "$deployOK" -a "${github.head_ref}" = master ]; then + if [ "$deployOK" -a "${git_branch}" = master ]; then echo echo "== Building and deploying master SNAPSHOT ==" mvn -Pdeploy-to-scijava $BUILD_ARGS deploy @@ -180,7 +177,7 @@ if [ -f environment.yml ]; then if [ ! -f "$condaSh" ]; then echo echo "== Installing conda ==" - if [ "${matrix.python-version}" = "2.7" ]; then + if [ "${python_version}" = "2.7" ]; then wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh else wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh From a95e4de37dc0880aca33f4fdac47459df58bb793 Mon Sep 17 00:00:00 2001 From: Jack Yuan Date: Tue, 11 May 2021 08:42:46 -0500 Subject: [PATCH 217/429] test pr and repo fork --- github-action-build.sh | 34 +++++++++++---------- github-actionify.sh | 68 +++++++++++++++++++++--------------------- 2 files changed, 52 insertions(+), 50 deletions(-) diff --git a/github-action-build.sh b/github-action-build.sh index d3a3a51..08cddb6 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -53,20 +53,20 @@ if [ -f pom.xml ]; then EOL - # NB: Use maven.scijava.org instead of Central if defined in repositories. - # This hopefully avoids intermittent "ReasonPhrase:Forbidden" errors - # when the Travis build pings Maven Central; see travis-ci/travis-ci#6593. - grep -A 2 '' pom.xml | grep -q 'maven.scijava.org' && - cat >>"$settingsFile" < - - scijava-mirror - SciJava mirror - https://maven.scijava.org/content/groups/public/ - central - - -EOL +# # NB: Use maven.scijava.org instead of Central if defined in repositories. +# # This hopefully avoids intermittent "ReasonPhrase:Forbidden" errors +# # when the Travis build pings Maven Central; see travis-ci/travis-ci#6593. +# grep -A 2 '' pom.xml | grep -q 'maven.scijava.org' && +# cat >>"$settingsFile" < +# +# scijava-mirror +# SciJava mirror +# https://maven.scijava.org/content/groups/public/ +# central +# +# +# EOL cat >>"$settingsFile" < @@ -103,9 +103,11 @@ EOL # TODO: try ${ secret.GITHUB_TOKEN } / double braces? if [ "${secure_env}" != true ]; then echo "No deploy -- secure environment variables not available" - elif [ "${pull_request}" != false ]; then + fi + if [ "${pull_request}" != false ]; then echo "No deploy -- pull request detected" - elif [ "${repo_fork}" != "$ciOrg/$ciRepo" ]; then + fi + if [ "${repo_fork}" != "$ciOrg/$ciRepo" ]; then echo "No deploy -- repository fork: ${repo_fork}} != $ciOrg/$ciRepo" # TODO: Detect travis-ci.org versus travis-ci.com? else diff --git a/github-actionify.sh b/github-actionify.sh index 87e4ab2..33cd340 100644 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -2,23 +2,23 @@ # travisify.sh # -# Script for enabling or updating Travis CI builds for a given repository. +# Script for enabling or updating GitHub Action builds for a given repository. #set -e dir="$(dirname "$0")" -travisDir=.travis -travisConfig=.travis.yml -travisBuildScript=$travisDir/build.sh -travisSettingsFile=$travisDir/settings.xml -travisNotifyScript=$travisDir/notify.sh +gitactionDir=.github +gitactionConfig=.gitaction.yml +gitactionBuildScript=$gitactionDir/build.sh +gitactionSettingsFile=$gitactionDir/settings.xml +gitactionNotifyScript=$gitactionDir/notify.sh credentialsDir=$HOME/.scijava/credentials varsFile=$credentialsDir/vars signingKeySourceFile=$credentialsDir/scijava-ci-signing.asc -signingKeyDestFile=$travisDir/signingkey.asc +signingKeyDestFile=$gitactionDir/signingkey.asc pomMinVersion='17.1.1' -tmpFile=travisify.tmp +tmpFile=gitaction.tmp info() { echo "- $@"; } warn() { echo "[WARNING] $@" 1>&2; } @@ -42,7 +42,7 @@ update() { file=$1 msg=$2 exe=$3 - test "$msg" || msg="Travis: update $file" + test "$msg" || msg="GitHub Action: update $file" if [ -e "$file" ] then if diff -q "$file" "$tmpFile" >/dev/null @@ -71,7 +71,7 @@ update() { process() { cd "$1" - # -- Git sanity checks -- + # -- GitHub Action sanity checks -- repoSlug=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="scm"]/*[local-name()="connection"]' pom.xml|sed 's_.*github.com[:/]\(.*\)<.*_\1_') test "$repoSlug" && info "Repository = $repoSlug" || die 'Could not determine GitHub repository slug' @@ -112,16 +112,16 @@ process() { *) die "Unsupported domain: $domain";; esac - # -- Travis sanity checks -- + # -- GitHub Action sanity checks -- - test -e "$travisDir" -a ! -d "$travisDir" && die "$travisDir is not a directory" - test -e "$travisConfig" -a ! -f "$travisConfig" && die "$travisConfig is not a regular file" - test -e "$travisConfig" && warn "$travisConfig already exists" - test -e "$travisBuildScript" && warn "$travisBuildScript already exists" + test -e "$gitactionDir" -a ! -d "$gitactionDir" && die "$gitactionDir is not a directory" + test -e "$gitactionConfig" -a ! -f "$gitactionConfig" && die "$gitactionConfig is not a regular file" + test -e "$gitactionConfig" && warn "$gitactionConfig already exists" + test -e "$gitactionBuildScript" && warn "$gitactionBuildScript already exists" # -- Do things -- - # Add/update the Travis configuration file. + # Add/update the GitHun Actions configuration file. cat >"$tmpFile" <"$tmpFile" < Date: Mon, 17 May 2021 13:04:22 -0500 Subject: [PATCH 218/429] test --- github-action-build.sh | 9 ++- github-actionify.sh | 1 + github-javadoc.sh | 127 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 github-javadoc.sh diff --git a/github-action-build.sh b/github-action-build.sh index 08cddb6..505c608 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -100,16 +100,21 @@ EOL ciRepo=${ciURL##*/} ciPrefix=${ciURL%/*} ciOrg=${ciPrefix##*/} - # TODO: try ${ secret.GITHUB_TOKEN } / double braces? + # https://docs.github.com/en/actions/reference/encrypted-secrets if [ "${secure_env}" != true ]; then echo "No deploy -- secure environment variables not available" + echo ${secure_env} + echo ${pull_request} + echo ${repo_fork} + echo ${runner_os} + echo ${git_branch} + echo ${python_version} fi if [ "${pull_request}" != false ]; then echo "No deploy -- pull request detected" fi if [ "${repo_fork}" != "$ciOrg/$ciRepo" ]; then echo "No deploy -- repository fork: ${repo_fork}} != $ciOrg/$ciRepo" - # TODO: Detect travis-ci.org versus travis-ci.com? else echo "All checks passed for artifact deployment" deployOK=1 diff --git a/github-actionify.sh b/github-actionify.sh index 33cd340..38f5c0f 100644 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -101,6 +101,7 @@ process() { die "Not pom-scijava parent: $parent. Run with -p flag to skip this check." fi + # https://docs.github.com/en/actions/managing-workflow-runs/adding-a-workflow-status-badge domain=$(grep "travis-ci\.[a-z]*/$repoSlug" pom.xml 2>/dev/null | sed 's/.*\(travis-ci\.[a-z]*\).*/\1/') test "$domain" && info "Detected domain from pom.xml: $domain" || diff --git a/github-javadoc.sh b/github-javadoc.sh new file mode 100644 index 0000000..7badbcb --- /dev/null +++ b/github-javadoc.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +# +# travis-javadoc.sh - A script to build the javadocs of a SciJava-based project. +# + +# The following repositories are known to use this script: +# +# bonej-org/bonej-javadoc +# fiji/fiji-javadoc +# imagej/imagej-javadoc +# imglib/imglib2-javadoc +# scifio/scifio-javadoc +# scijava/java3d-javadoc +# scijava/scijava-javadoc +# slim-curve/slim-javadoc +# uw-loci/loci-javadoc + +# Wait for a launched background command to complete, emitting +# an occasional message to avoid long periods without output. +# Return the same exit code as the launched command. +keep_alive() { + pid="$1" + if [ "$pid" = "" ] + then + echo "[ERROR] No PID given" + return + fi + i=0 + while kill -0 "$pid" 2>/dev/null; do + i=$((i+1)) + m=$((i/60)) + s=$((i%60)) + test $s -eq 0 && echo "[$m minutes elapsed]" + sleep 1 + done + wait "$pid" +} + +if [ "$TRAVIS_SECURE_ENV_VARS" = true \ + -a "$TRAVIS_PULL_REQUEST" = false \ + -a "$TRAVIS_BRANCH" = master ] +then + project=$1 + openssl_key=$2 + openssl_iv=$3 + + # Populate the settings.xml configuration. + mkdir -p "$HOME/.m2" + settingsFile="$HOME/.m2/settings.xml" + customSettings=.travis/settings.xml + if [ -f "$customSettings" ] + then + cp "$customSettings" "$settingsFile" + else + # NB: Use maven.scijava.org as sole mirror if defined in . + # This hopefully avoids intermittent "ReasonPhrase:Forbidden" errors + # when the Travis build pings Maven Central; see travis-ci/travis-ci#6593. + test -f pom.xml && grep -A 2 '' pom.xml | grep -q 'maven.scijava.org' && + cat >"$settingsFile" < + + + scijava-mirror + SciJava mirror + https://maven.scijava.org/content/groups/public/ + * + + + +EOL + fi + + # Emit some details useful for debugging. + # NB: We run once with -q to suppress the download messages, + # then again without it to emit the desired dependency tree. + mvn -B -q dependency:tree && + mvn -B dependency:tree && + + echo && + echo "== Generating javadoc ==" && + + # Build the javadocs. + (mvn -B -q -Pbuild-javadoc) & + keep_alive $! && + test -d target/apidocs && + # Strip out date stamps, to avoid spurious changes being committed. + sed -i'' -e '/\( Artifact to invalidate = $ga" && echo "machine maven.scijava.org" >"$HOME/.netrc" && - echo " login scijava-ci" >>"$HOME/.netrc" && + echo " login $MAVEN_USER" >>"$HOME/.netrc" && echo " password $MAVEN_PASS" >>"$HOME/.netrc" && sh maven-helper.sh invalidate-cache "$ga" checkSuccess $? From 60e4371790b58ee6dd2e7e865601e0d91ce9de20 Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Thu, 24 Jun 2021 13:28:00 -0500 Subject: [PATCH 265/429] Fix settings file creation Need to put variable values, not references, in the settings.xml. --- github-action-build.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/github-action-build.sh b/github-action-build.sh index 658ce93..04fb604 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -38,18 +38,18 @@ if [ -f pom.xml ]; then scijava.releases - \$MAVEN_USER - \$MAVEN_PASS + $MAVEN_USER + $MAVEN_PASS scijava.snapshots - \$MAVEN_USER - \$MAVEN_PASS + $MAVEN_USER + $MAVEN_PASS sonatype-nexus-releases scijava-ci - \$OSSRH_PASS + $OSSRH_PASS EOL @@ -59,12 +59,12 @@ EOL gpg - \$HOME/.gnupg + $HOME/.gnupg - \$GPG_KEY_NAME - \$GPG_PASSPHRASE + $GPG_KEY_NAME + $GPG_PASSPHRASE From 7f218b42e939ea908bcdcb60649a48e691f7a2f8 Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Thu, 24 Jun 2021 14:09:56 -0500 Subject: [PATCH 266/429] Restore fork detection --- github-action-build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/github-action-build.sh b/github-action-build.sh index 04fb604..90c1ad7 100644 --- a/github-action-build.sh +++ b/github-action-build.sh @@ -81,8 +81,13 @@ EOL echo "Output of failed attempt follows:" echo "$ciURL" else + ciRepo=${ciURL##*/} + ciPrefix=${ciURL%/*} + ciOrg=${ciPrefix##*/} if [ ! "$SIGNING_ASC" ] || [ ! "$GPG_KEY_NAME" ] || [ ! "$GPG_PASSPHRASE" ] || [ ! "$MAVEN_PASS" ] || [ ! "$OSSRH_PASS" ]; then echo "No deploy -- secure environment variables not available" + elif [ "${GITHUB_REPOSITORY}" != "$ciOrg/$ciRepo" ]; then + echo "No deploy -- repository fork: ${GITHUB_REPOSITORY} != $ciOrg/$ciRepo" else echo "All checks passed for artifact deployment" deployOK=1 From fe4f3ed06a4cedc77139af63e3e10cc932b181d9 Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Mon, 28 Jun 2021 13:52:17 -0500 Subject: [PATCH 267/429] Split out CI-dependent build code --- github-action-build.sh => ci-build.sh | 21 +++++++++++++-------- github-action-ci.sh | 11 +++++++++++ github-actionify.sh | 19 +++++++++++++++++-- 3 files changed, 41 insertions(+), 10 deletions(-) rename github-action-build.sh => ci-build.sh (89%) mode change 100644 => 100755 create mode 100755 github-action-ci.sh mode change 100644 => 100755 github-actionify.sh diff --git a/github-action-build.sh b/ci-build.sh old mode 100644 new mode 100755 similarity index 89% rename from github-action-build.sh rename to ci-build.sh index 90c1ad7..93a5392 --- a/github-action-build.sh +++ b/ci-build.sh @@ -1,9 +1,13 @@ #!/bin/bash # -# github-action-build.sh - A script to build and/or release SciJava-based projects -# automatically using GitHub Actions. +# ci-build.sh - A script to build and/or release SciJava-based projects +# automatically using a continuous integration +# service. # +# Required environment variables: +# BUILD_OS - the operating system running the current build (e.g. macOS) +# BUILD_REPOSITORY - the repository slug (org/repo) running the current build dir="$(dirname "$0")" @@ -29,7 +33,7 @@ if [ -f pom.xml ]; then # Populate the settings.xml configuration. mkdir -p "$HOME/.m2" settingsFile="$HOME/.m2/settings.xml" - customSettings=.github/settings.xml + customSettings=.ci/settings.xml if [ -f "$customSettings" ]; then cp "$customSettings" "$settingsFile" else @@ -86,8 +90,8 @@ EOL ciOrg=${ciPrefix##*/} if [ ! "$SIGNING_ASC" ] || [ ! "$GPG_KEY_NAME" ] || [ ! "$GPG_PASSPHRASE" ] || [ ! "$MAVEN_PASS" ] || [ ! "$OSSRH_PASS" ]; then echo "No deploy -- secure environment variables not available" - elif [ "${GITHUB_REPOSITORY}" != "$ciOrg/$ciRepo" ]; then - echo "No deploy -- repository fork: ${GITHUB_REPOSITORY} != $ciOrg/$ciRepo" + elif [ "$BUILD_REPOSITORY" != "$ciOrg/$ciRepo" ]; then + echo "No deploy -- repository fork: $BUILD_REPOSITORY != $ciOrg/$ciRepo" else echo "All checks passed for artifact deployment" deployOK=1 @@ -95,14 +99,15 @@ EOL fi # Install GPG on OSX/macOS - if [ ${RUNNER_OS} = 'macOS' ]; then + if [ $BUILD_OS = 'macOS' ]; then HOMEBREW_NO_AUTO_UPDATE=1 brew install gnupg2 fi # Import the GPG signing key. - keyFile=.github/signingkey.asc + keyFile=.ci/signingkey.asc if [ "$deployOK" ]; then echo "== Importing GPG keypair ==" + mkdir -p .ci echo "$SIGNING_ASC" > "$keyFile" ls -la "$keyFile" gpg --batch --fast-import "$keyFile" @@ -171,7 +176,7 @@ if [ -f environment.yml ]; then echo echo "== Configuring environment ==" - condaEnv=github-scijava + condaEnv=ci-scijava test -d "$condaDir/envs/$condaEnv" && condaAction=update || condaAction=create conda env "$condaAction" -n "$condaEnv" -f environment.yml && conda activate "$condaEnv" diff --git a/github-action-ci.sh b/github-action-ci.sh new file mode 100755 index 0000000..e368959 --- /dev/null +++ b/github-action-ci.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# +# github-action-ci.sh - A script to set up ci-related environment variables from GitHub Actions +# + +echo "BUILD_REPOSITORY=${GITHUB_REPOSITORY}" +echo "BUILD_OS=${RUNNER_OS}" + +echo "BUILD_REPOSITORY=${GITHUB_REPOSITORY}" >> $GITHUB_ENV +echo "BUILD_OS=${RUNNER_OS}" >> $GITHUB_ENV diff --git a/github-actionify.sh b/github-actionify.sh old mode 100644 new mode 100755 index 626a4f3..37fc436 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -17,6 +17,7 @@ gitactionDir=.github gitactionConfigRoot=/workflows/.gitaction.yml gitactionConfig=$gitactionDir$gitactionConfigRoot gitactionPRConfig=$gitactionDir/workflows/.gitaction-pr.yml +gitactionSetupScript=$gitactionDir/setup.sh gitactionBuildScript=$gitactionDir/build.sh gitactionSettingsFile=$gitactionDir/settings.xml gitactionNotifyScript=$gitactionDir/notify.sh @@ -118,6 +119,7 @@ process() { test -e "$gitactionPRConfig" -a ! -f "$gitactionPRConfig" && die "$gitactionPRConfig is not a regular file" test -e "$gitactionConfig" && warn "$gitactionConfig already exists" test -e "$gitactionBuildScript" && warn "$gitactionBuildScript already exists" + test -e "$gitactionSetupScript" && warn "$gitactionSetupScript already exists" # -- Do things -- @@ -154,6 +156,8 @@ jobs: with: java-version: '8' distribution: 'zulu' + - name: Set up CI environment + run: ./$gitactionSetupScript - name: Build with Maven run: ./$gitactionBuildScript env: @@ -199,16 +203,27 @@ jobs: with: java-version: '8' distribution: 'zulu' + - name: Set up CI environment + run: ./$gitactionSetupScript - name: Build with Maven run: ./$gitactionBuildScript EOL update "$gitactionPRConfig" + # Add/update the GitHub Action setup script. + cat >"$tmpFile" <"$tmpFile" < Date: Tue, 29 Jun 2021 10:35:59 -0500 Subject: [PATCH 268/429] ci-build.sh: eliminate BUILD_OS variable We can just test for the existence of the brew program. --- ci-build.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index 93a5392..09a2ebf 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -6,7 +6,6 @@ # service. # # Required environment variables: -# BUILD_OS - the operating system running the current build (e.g. macOS) # BUILD_REPOSITORY - the repository slug (org/repo) running the current build dir="$(dirname "$0")" @@ -98,8 +97,8 @@ EOL fi fi - # Install GPG on OSX/macOS - if [ $BUILD_OS = 'macOS' ]; then + # Install GPG on macOS + if which brew >/dev/null 2>/dev/null; then HOMEBREW_NO_AUTO_UPDATE=1 brew install gnupg2 fi From 24c5403f2eca02edfed62d512bba31f67654d3d4 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 29 Jun 2021 10:36:50 -0500 Subject: [PATCH 269/429] github-actionify.sh: fix typo --- github-actionify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-actionify.sh b/github-actionify.sh index 37fc436..f0a3853 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -123,7 +123,7 @@ process() { # -- Do things -- - # Add/update the main GitHun Actions configuration file. + # Add/update the main GitHub Actions configuration file. cat >"$tmpFile" < Date: Wed, 30 Jun 2021 09:51:25 -0500 Subject: [PATCH 270/429] github-actionify.sh: fix sed expressions The repoSlug contains a slash, which will affect the expression if slash is used as the delimiter. So we change to semicolon. The command "sed -i" with no extension does not work with BSD sed (e.g. macOS); you need to write "sed -i ''" to avoid saving the previous file to a backup. But this doesn't work with GNU sed, which requires "sed -i''" or simply "sed -i" instead. The only way I know that works with both sed flavors is to give the flag as "-i.bak" and then delete the .bak file afterward. --- github-actionify.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index f0a3853..e7c79c8 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -109,8 +109,9 @@ process() { # Change pom.xml from Travis CI to GitHub Action domain="github.com" - sed -i 's/Travis CI/GitHub Actions/g' pom.xml - sed -i "s/travis-ci.*/github.com\/$repoSlug\/actions\/workflows\/\.gitaction\.yml<\/url>/g" pom.xml + sed -i.bak 's/Travis CI/GitHub Actions/g' pom.xml + sed -i.bak "s;travis-ci.*;github.com/$repoSlug/actions/workflows/.gitaction.yml;g" pom.xml + rm -f pom.xml.bak # -- GitHub Action sanity checks -- From 0ea7d5d766eeda155ea7ada4ed01d41c2894d996 Mon Sep 17 00:00:00 2001 From: Jan Eglinger Date: Wed, 30 Jun 2021 21:15:41 +0200 Subject: [PATCH 271/429] github-actionify: fix badge when newly added Signed-off-by: Jan Eglinger --- github-actionify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-actionify.sh b/github-actionify.sh index e7c79c8..489daa4 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -279,7 +279,7 @@ EOL update README.md 'GitHub Action: fix README.md badge link' else info "Adding GitHub Action badge to README.md" - echo "[![SciJava CI](https://$domain/$repoSlug/actions/$gitactionConfig/badge.svg)](https://$domain/$repoSlug/actions/$gitactionConfig)/g" README.md >"$tmpFile" + echo "[![SciJava CI](https://$domain/$repoSlug/actions/$gitactionConfigRoot/badge.svg)](https://$domain/$repoSlug/actions/$gitactionConfigRoot)" >"$tmpFile" echo >>"$tmpFile" test -f README.md && cat README.md >>"$tmpFile" update README.md 'GitHub Action: add badge to README.md' From f1557422db54fd8682d4e1687a7e185ba89b419e Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 30 Jun 2021 15:02:42 -0500 Subject: [PATCH 272/429] github-actionify: fix README.md badge update --- github-actionify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-actionify.sh b/github-actionify.sh index 489daa4..24600a1 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -275,7 +275,7 @@ EOL if grep -q "travis-ci.*svg" README.md >/dev/null 2>&1 then info "Updating README.md GitHub Action badge" - sed "s/travis-ci.*/${domain//\//\\/}\/${repoSlug//\//\\/}\/actions${gitactionConfigRoot//\//\\/}\/badge\.svg\)\]\(https:\/\/$domain\/${repoSlug//\//\\/}\/actions${gitactionConfigRoot//\//\\/}\)/g" README.md >"$tmpFile" + sed "s;travis-ci.*;$domain/$repoSlug/actions/$gitactionConfigRoot/badge.svg)](https://$domain/$repoSlug/actions$gitactionConfigRoot);g" README.md >"$tmpFile" update README.md 'GitHub Action: fix README.md badge link' else info "Adding GitHub Action badge to README.md" From aa2a07ff1e94d6c3ac4a67231db60516e614b9b7 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 30 Jun 2021 15:03:15 -0500 Subject: [PATCH 273/429] github-actionify: fix pom.xml simulated update It was actually changing the file even in simulation mode. --- github-actionify.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index 24600a1..ecd2777 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -107,11 +107,11 @@ process() { die "Not pom-scijava parent: $parent. Run with -p flag to skip this check." fi - # Change pom.xml from Travis CI to GitHub Action + # Change pom.xml from Travis CI to GitHub Actions domain="github.com" - sed -i.bak 's/Travis CI/GitHub Actions/g' pom.xml - sed -i.bak "s;travis-ci.*;github.com/$repoSlug/actions/workflows/.gitaction.yml;g" pom.xml - rm -f pom.xml.bak + sed 's/Travis CI/GitHub Actions/g' pom.xml >"$tmpFile" + sed "s;travis-ci.*;github.com/$repoSlug/actions/workflows/.gitaction.yml;g" pom.xml >>"$tmpFile" + update pom.xml # -- GitHub Action sanity checks -- From 57d44052aec7fb952c81200fe92cae3d93d8f9cd Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 30 Jun 2021 15:24:24 -0500 Subject: [PATCH 274/429] github-actionify: tune commit message prefix It is now "CI: " rather than "GitHub Action: ", and it is used consistently for all commits. --- github-actionify.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index ecd2777..fa1aba2 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -25,6 +25,7 @@ credentialsDir=$HOME/.scijava/credentials varsFile=$credentialsDir/vars pomMinVersion='17.1.1' tmpFile=gitaction.tmp +msgPrefix="CI: " info() { echo "- $@"; } warn() { echo "[WARNING] $@" 1>&2; } @@ -48,7 +49,7 @@ update() { file=$1 msg=$2 exe=$3 - test "$msg" || msg="GitHub Action: update $file" + test "$msg" || msg="update $file" if [ -e "$file" ] then if diff -q "$file" "$tmpFile" >/dev/null @@ -71,7 +72,7 @@ update() { info "Adding execute permission to $file" $EXEC git update-index --chmod=+x "$file" fi - $EXEC git diff-index --quiet HEAD -- || $EXEC git commit -m "$msg" + $EXEC git diff-index --quiet HEAD -- || $EXEC git commit -m "$msgPrefix$msg" } process() { @@ -218,7 +219,7 @@ curl -fsLO https://raw.githubusercontent.com/scijava/scijava-scripts/master/gith sh github-action-ci.sh EOL chmod +x "$tmpFile" - update "$gitactionSetupScript" "GitHub Action: add executable script $gitactionSetupScript" "true" + update "$gitactionSetupScript" "add executable script $gitactionSetupScript" "true" # Add/update the GitHub Action build script. cat >"$tmpFile" <$version$| $pomMinVersion|" pom.xml >"$tmpFile" - update pom.xml "POM: update pom-scijava parent to $pomMinVersion" + update pom.xml "update pom-scijava parent to $pomMinVersion" else info "Version of pom-scijava ($version) is OK" fi @@ -267,7 +268,7 @@ EOL info 'Adding property' cp pom.xml "$tmpFile" perl -0777 -i -pe 's/(\n\t<\/properties>\n)/\n\n\t\t\n\t\tdeploy-to-scijava<\/releaseProfiles>\1/igs' "$tmpFile" - update pom.xml 'POM: deploy releases to the SciJava repository' + update pom.xml 'deploy releases to the SciJava repository' fi # update the README @@ -276,13 +277,13 @@ EOL then info "Updating README.md GitHub Action badge" sed "s;travis-ci.*;$domain/$repoSlug/actions/$gitactionConfigRoot/badge.svg)](https://$domain/$repoSlug/actions$gitactionConfigRoot);g" README.md >"$tmpFile" - update README.md 'GitHub Action: fix README.md badge link' + update README.md 'update README.md badge link' else info "Adding GitHub Action badge to README.md" echo "[![SciJava CI](https://$domain/$repoSlug/actions/$gitactionConfigRoot/badge.svg)](https://$domain/$repoSlug/actions/$gitactionConfigRoot)" >"$tmpFile" echo >>"$tmpFile" test -f README.md && cat README.md >>"$tmpFile" - update README.md 'GitHub Action: add badge to README.md' + update README.md 'add README.md badge link' fi } From 922ed808524f3a8f9b2cb7355ae8543f650f96e0 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 10:07:13 -0500 Subject: [PATCH 275/429] github-actionify: fix duplicate pom content bug --- github-actionify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-actionify.sh b/github-actionify.sh index fa1aba2..5780826 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -111,7 +111,7 @@ process() { # Change pom.xml from Travis CI to GitHub Actions domain="github.com" sed 's/Travis CI/GitHub Actions/g' pom.xml >"$tmpFile" - sed "s;travis-ci.*;github.com/$repoSlug/actions/workflows/.gitaction.yml;g" pom.xml >>"$tmpFile" + sed "s;travis-ci.*;github.com/$repoSlug/actions/workflows/.gitaction.yml;g" pom.xml >"$tmpFile" update pom.xml # -- GitHub Action sanity checks -- From 930f26f57814db49897fa674f8d08e152aee4503 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 10:08:38 -0500 Subject: [PATCH 276/429] github-actionify: rename vars from gitaction to ci The name of the CI system is "GitHub Actions". So we could use the prefix "githubactions" or "gha". But "ci" is more general and concise. --- github-actionify.sh | 60 ++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index 5780826..bdf6444 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -13,14 +13,14 @@ dir="$(dirname "$0")" -gitactionDir=.github -gitactionConfigRoot=/workflows/.gitaction.yml -gitactionConfig=$gitactionDir$gitactionConfigRoot -gitactionPRConfig=$gitactionDir/workflows/.gitaction-pr.yml -gitactionSetupScript=$gitactionDir/setup.sh -gitactionBuildScript=$gitactionDir/build.sh -gitactionSettingsFile=$gitactionDir/settings.xml -gitactionNotifyScript=$gitactionDir/notify.sh +ciDir=.github +ciConfigRoot=/workflows/.gitaction.yml +ciConfig=$ciDir$ciConfigRoot +ciPRConfig=$ciDir/workflows/.gitaction-pr.yml +ciSetupScript=$ciDir/setup.sh +ciBuildScript=$ciDir/build.sh +ciSettingsFile=$ciDir/settings.xml +ciNotifyScript=$ciDir/notify.sh credentialsDir=$HOME/.scijava/credentials varsFile=$credentialsDir/vars pomMinVersion='17.1.1' @@ -116,12 +116,12 @@ process() { # -- GitHub Action sanity checks -- - test -e "$gitactionDir" -a ! -d "$gitactionDir" && die "$gitactionDir is not a directory" - test -e "$gitactionConfig" -a ! -f "$gitactionConfig" && die "$gitactionConfig is not a regular file" - test -e "$gitactionPRConfig" -a ! -f "$gitactionPRConfig" && die "$gitactionPRConfig is not a regular file" - test -e "$gitactionConfig" && warn "$gitactionConfig already exists" - test -e "$gitactionBuildScript" && warn "$gitactionBuildScript already exists" - test -e "$gitactionSetupScript" && warn "$gitactionSetupScript already exists" + test -e "$ciDir" -a ! -d "$ciDir" && die "$ciDir is not a directory" + test -e "$ciConfig" -a ! -f "$ciConfig" && die "$ciConfig is not a regular file" + test -e "$ciPRConfig" -a ! -f "$ciPRConfig" && die "$ciPRConfig is not a regular file" + test -e "$ciConfig" && warn "$ciConfig already exists" + test -e "$ciBuildScript" && warn "$ciBuildScript already exists" + test -e "$ciSetupScript" && warn "$ciSetupScript already exists" # -- Do things -- @@ -159,9 +159,9 @@ jobs: java-version: '8' distribution: 'zulu' - name: Set up CI environment - run: ./$gitactionSetupScript + run: ./$ciSetupScript - name: Build with Maven - run: ./$gitactionBuildScript + run: ./$ciBuildScript env: GPG_KEY_NAME: \${{ secrets.GPG_KEY_NAME }} GPG_PASSPHRASE: \${{ secrets.GPG_PASSPHRASE }} @@ -170,7 +170,7 @@ jobs: OSSRH_PASS: \${{ secrets.OSSRH_PASS }} SIGNING_ASC: \${{ secrets.SIGNING_ASC }} EOL - update "$gitactionConfig" + update "$ciConfig" # Add/update the GitHun Actions PR configuration file. cat >"$tmpFile" <"$tmpFile" <"$tmpFile" </dev/null 2>&1 then info "Updating README.md GitHub Action badge" - sed "s;travis-ci.*;$domain/$repoSlug/actions/$gitactionConfigRoot/badge.svg)](https://$domain/$repoSlug/actions$gitactionConfigRoot);g" README.md >"$tmpFile" + sed "s;travis-ci.*;$domain/$repoSlug/actions/$ciConfigRoot/badge.svg)](https://$domain/$repoSlug/actions$ciConfigRoot);g" README.md >"$tmpFile" update README.md 'update README.md badge link' else info "Adding GitHub Action badge to README.md" - echo "[![SciJava CI](https://$domain/$repoSlug/actions/$gitactionConfigRoot/badge.svg)](https://$domain/$repoSlug/actions/$gitactionConfigRoot)" >"$tmpFile" + echo "[![SciJava CI](https://$domain/$repoSlug/actions/$ciConfigRoot/badge.svg)](https://$domain/$repoSlug/actions/$ciConfigRoot)" >"$tmpFile" echo >>"$tmpFile" test -f README.md && cat README.md >>"$tmpFile" update README.md 'add README.md badge link' From f8392aad66f9f01d86dd44e4ed257a91c2f7383e Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 13:21:03 -0500 Subject: [PATCH 277/429] github-actionify: use a clearer temp file name --- github-actionify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-actionify.sh b/github-actionify.sh index bdf6444..8067637 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -24,7 +24,7 @@ ciNotifyScript=$ciDir/notify.sh credentialsDir=$HOME/.scijava/credentials varsFile=$credentialsDir/vars pomMinVersion='17.1.1' -tmpFile=gitaction.tmp +tmpFile=github-actionify.tmp msgPrefix="CI: " info() { echo "- $@"; } From b3d87ea4d42861ad1fc2bbbd517a729fe04c190e Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 13:25:15 -0500 Subject: [PATCH 278/429] github-actionify: improve Travis-to-GHA logic * Link to toplevel .../actions path. * Give a better commit message. --- github-actionify.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index 8067637..deac236 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -111,8 +111,8 @@ process() { # Change pom.xml from Travis CI to GitHub Actions domain="github.com" sed 's/Travis CI/GitHub Actions/g' pom.xml >"$tmpFile" - sed "s;travis-ci.*;github.com/$repoSlug/actions/workflows/.gitaction.yml;g" pom.xml >"$tmpFile" - update pom.xml + sed "s;travis-ci.*;github.com/$repoSlug/actions;g" pom.xml >"$tmpFile" + update pom.xml "switch from Travis CI to GitHub Actions" # -- GitHub Action sanity checks -- From 7aa7c8e2be9c25393fcc5cde3eb068ed3696ad8d Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 13:26:11 -0500 Subject: [PATCH 279/429] github-actionify: improve badge labels We want the main build badge to just say "build" like other projects do. --- github-actionify.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index deac236..add04b1 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -127,7 +127,7 @@ process() { # Add/update the main GitHub Actions configuration file. cat >"$tmpFile" <"$tmpFile" < Date: Thu, 1 Jul 2021 13:27:11 -0500 Subject: [PATCH 280/429] github-actionify: improve commit messages --- github-actionify.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index add04b1..e9c7368 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -170,7 +170,7 @@ jobs: OSSRH_PASS: \${{ secrets.OSSRH_PASS }} SIGNING_ASC: \${{ secrets.SIGNING_ASC }} EOL - update "$ciConfig" + update "$ciConfig" "add/update main build action" # Add/update the GitHun Actions PR configuration file. cat >"$tmpFile" <"$tmpFile" < Date: Thu, 1 Jul 2021 13:27:25 -0500 Subject: [PATCH 281/429] github-actionify: fix typo --- github-actionify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-actionify.sh b/github-actionify.sh index e9c7368..cee519a 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -172,7 +172,7 @@ jobs: EOL update "$ciConfig" "add/update main build action" - # Add/update the GitHun Actions PR configuration file. + # Add/update the GitHub Actions PR configuration file. cat >"$tmpFile" < Date: Thu, 1 Jul 2021 13:27:49 -0500 Subject: [PATCH 282/429] github-actionify: improve var names and filenames There is no reason for the YAML files to begin with a dot character. These are not hidden files; they are just build configuration files. The workflows in question are the main build, and the PR build, so we now name them as such. While a project's CI may have other workflows besides these, in the standard case, these two workflows are the two used by the vast majority of SciJava projects. Projects needing customization beyond this are of course free to rename everything however they see fit, hack the build scripts, or do whatever. --- github-actionify.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index cee519a..52d066f 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -14,9 +14,10 @@ dir="$(dirname "$0")" ciDir=.github -ciConfigRoot=/workflows/.gitaction.yml -ciConfig=$ciDir$ciConfigRoot -ciPRConfig=$ciDir/workflows/.gitaction-pr.yml +ciSlugBuildMain=workflows/build-main.yml +ciSlugBuildPR=workflows/build-pr.yml +ciConfigBuildMain=$ciDir/$ciSlugBuildMain +ciConfigBuildPR=$ciDir/$ciSlugBuildPR ciSetupScript=$ciDir/setup.sh ciBuildScript=$ciDir/build.sh ciSettingsFile=$ciDir/settings.xml @@ -117,9 +118,9 @@ process() { # -- GitHub Action sanity checks -- test -e "$ciDir" -a ! -d "$ciDir" && die "$ciDir is not a directory" - test -e "$ciConfig" -a ! -f "$ciConfig" && die "$ciConfig is not a regular file" - test -e "$ciPRConfig" -a ! -f "$ciPRConfig" && die "$ciPRConfig is not a regular file" - test -e "$ciConfig" && warn "$ciConfig already exists" + test -e "$ciConfigBuildMain" -a ! -f "$ciConfigBuildMain" && die "$ciConfigBuildMain is not a regular file" + test -e "$ciConfigBuildPR" -a ! -f "$ciConfigBuildPR" && die "$ciConfigBuildPR is not a regular file" + test -e "$ciConfigBuildMain" && warn "$ciConfigBuildMain already exists" test -e "$ciBuildScript" && warn "$ciBuildScript already exists" test -e "$ciSetupScript" && warn "$ciSetupScript already exists" @@ -170,7 +171,7 @@ jobs: OSSRH_PASS: \${{ secrets.OSSRH_PASS }} SIGNING_ASC: \${{ secrets.SIGNING_ASC }} EOL - update "$ciConfig" "add/update main build action" + update "$ciConfigBuildMain" "add/update main build action" # Add/update the GitHub Actions PR configuration file. cat >"$tmpFile" <"$tmpFile" </dev/null 2>&1 then info "Updating README.md GitHub Action badge" - sed "s;travis-ci.*;$domain/$repoSlug/actions/$ciConfigRoot/badge.svg)](https://$domain/$repoSlug/actions$ciConfigRoot);g" README.md >"$tmpFile" + sed "s;travis-ci.*;$domain/$repoSlug/actions/$ciSlugBuildMain/badge.svg)](https://$domain/$repoSlug/actions/$ciSlugBuildMain);g" README.md >"$tmpFile" update README.md 'update README.md badge link' else info "Adding GitHub Action badge to README.md" - echo "[![SciJava CI](https://$domain/$repoSlug/actions/$ciConfigRoot/badge.svg)](https://$domain/$repoSlug/actions/$ciConfigRoot)" >"$tmpFile" + echo "[![](https://$domain/$repoSlug/actions/$ciSlugBuildMain/badge.svg)](https://$domain/$repoSlug/actions/$ciSlugBuildMain)" >"$tmpFile" echo >>"$tmpFile" test -f README.md && cat README.md >>"$tmpFile" update README.md 'add README.md badge link' From 507ccfffac243a9230db83c6f97154229069547d Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 13:41:25 -0500 Subject: [PATCH 283/429] github-actionify: improve explanatory notice --- github-actionify.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/github-actionify.sh b/github-actionify.sh index 52d066f..0a2d75b 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -288,7 +288,17 @@ EOL fi } -echo "Note that CI deployment requires additional configuration. Please contact a SciJava administrator for more information." +cat < Date: Thu, 1 Jul 2021 13:42:41 -0500 Subject: [PATCH 284/429] github-actionify: remove trailing whitespace --- github-actionify.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index 0a2d75b..fb664a7 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -153,7 +153,7 @@ jobs: \${{ runner.os }}-build-\${{ env.cache-name }}- \${{ runner.os }}-build- \${{ runner.os }}- - + - name: Set up JDK 8 uses: actions/setup-java@v2 with: @@ -200,7 +200,7 @@ jobs: \${{ runner.os }}-build-\${{ env.cache-name }}- \${{ runner.os }}-build- \${{ runner.os }}- - + - name: Set up JDK 8 uses: actions/setup-java@v2 with: From 2f913c25c2ba793e5d66777b0ca3d94ac551110b Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 13:43:41 -0500 Subject: [PATCH 285/429] github-actionify: remove redundant leading ./ The ./ prefix is only needed when referencing a file in the current directory -- not subdirectories such as .github/... --- github-actionify.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index fb664a7..d696ebf 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -160,9 +160,9 @@ jobs: java-version: '8' distribution: 'zulu' - name: Set up CI environment - run: ./$ciSetupScript + run: $ciSetupScript - name: Build with Maven - run: ./$ciBuildScript + run: $ciBuildScript env: GPG_KEY_NAME: \${{ secrets.GPG_KEY_NAME }} GPG_PASSPHRASE: \${{ secrets.GPG_PASSPHRASE }} @@ -207,9 +207,9 @@ jobs: java-version: '8' distribution: 'zulu' - name: Set up CI environment - run: ./$ciSetupScript + run: $ciSetupScript - name: Build with Maven - run: ./$ciBuildScript + run: $ciBuildScript EOL update "$ciConfigBuildPR" "add/update PR build action" From 14d714c446d3c7a8b64155949eaab4ca0f80f9a6 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 13:47:09 -0500 Subject: [PATCH 286/429] github-actionify: remove obsolete file logic These files will never exist inside the .github directory -- only inside .travis. And we are already removing the entire .travis folder. --- github-actionify.sh | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index d696ebf..73d3b8f 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -20,8 +20,6 @@ ciConfigBuildMain=$ciDir/$ciSlugBuildMain ciConfigBuildPR=$ciDir/$ciSlugBuildPR ciSetupScript=$ciDir/setup.sh ciBuildScript=$ciDir/build.sh -ciSettingsFile=$ciDir/settings.xml -ciNotifyScript=$ciDir/notify.sh credentialsDir=$HOME/.scijava/credentials varsFile=$credentialsDir/vars pomMinVersion='17.1.1' @@ -231,19 +229,6 @@ EOL chmod +x "$tmpFile" update "$ciBuildScript" "add executable script $ciBuildScript" "true" - # Remove obsolete GitHub-Actions-related files. - if [ -f "$ciSettingsFile" ] - then - info "Removing obsolete $ciSettingsFile (github-action-build.sh generates it now)" - $EXEC git rm -f "$ciSettingsFile" - fi - if [ -f "$ciNotifyScript" ] - then - info "Removing obsolete $ciNotifyScript (ImageJ Jenkins is dead)" - $EXEC git rm -f "$ciNotifyScript" - fi - $EXEC git diff-index --quiet HEAD -- || $EXEC git commit -m "${msgPrefix}remove obsolete files" - # Upgrade version of pom-scijava. if [ -z "$SKIP_PARENT_CHECK" ] then From 31fb8809fcb7983a5d2902f4d5c3b512b3545258 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 14:00:34 -0500 Subject: [PATCH 287/429] github-actionify: use a better cache name It's not caching the node_modules folder, so the name cache-node-modules isn't right. It's caching the .m2 folder, so the name cache-m2 is OK. --- github-actionify.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index 73d3b8f..6e5d6d6 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -140,10 +140,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Cache m2 modules + - name: Cache m2 folder uses: actions/cache@v2 env: - cache-name: cache-node-modules + cache-name: cache-m2 with: path: ~/.m2 key: \${{ runner.os }}-build-\${{ env.cache-name }} @@ -187,10 +187,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Cache m2 modules + - name: Cache m2 folder uses: actions/cache@v2 env: - cache-name: cache-node-modules + cache-name: cache-m2 with: path: ~/.m2 key: \${{ runner.os }}-build-\${{ env.cache-name }} From 58ecce66a53868c1f9cb057b3a84bf5b54316c3e Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 14:05:44 -0500 Subject: [PATCH 288/429] github-actionify: generalize build exec step name There may be other build steps performed beyond only Maven. --- github-actionify.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index 6e5d6d6..b45293a 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -159,7 +159,7 @@ jobs: distribution: 'zulu' - name: Set up CI environment run: $ciSetupScript - - name: Build with Maven + - name: Execute the build run: $ciBuildScript env: GPG_KEY_NAME: \${{ secrets.GPG_KEY_NAME }} @@ -206,7 +206,7 @@ jobs: distribution: 'zulu' - name: Set up CI environment run: $ciSetupScript - - name: Build with Maven + - name: Execute the build run: $ciBuildScript EOL update "$ciConfigBuildPR" "add/update PR build action" From d48f81b1b7bdded8c5f5f7133c4ccd3ce0ed9586 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 14:15:44 -0500 Subject: [PATCH 289/429] github-actionify: sign all release JARs Even when releasing to maven.scijava.org, we should sign the JARs, because we can. This will be very useful eventually for validating that JARs in a local environment are secure. --- github-actionify.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index b45293a..1864191 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -248,12 +248,23 @@ EOL releaseProfile=$(grep '' pom.xml 2>/dev/null | sed 's/[^>]*>//' | sed 's/<.*//') if [ "$releaseProfile" ] then - test "$releaseProfile" = 'deploy-to-scijava' || - warn "Unknown release profile: $releaseProfile" + case "$releaseProfile" in + sign,deploy-to-scijava) + info 'No changes needed to property' + ;; + deploy-to-scijava) + info 'Updating property' + sed 's;\(\).*\(\);\1sign,deploy-to-scijava\2;' pom.xml >"$tmpFile" + update pom.xml 'sign JARs when deploying releases' + ;; + *) + warn "Unknown release profile: $releaseProfile" + ;; + esac else info 'Adding property' cp pom.xml "$tmpFile" - perl -0777 -i -pe 's/(\n\t<\/properties>\n)/\n\n\t\t\n\t\tdeploy-to-scijava<\/releaseProfiles>\1/igs' "$tmpFile" + perl -0777 -i -pe 's/(\n\t<\/properties>\n)/\n\n\t\t\n\t\tsign,deploy-to-scijava<\/releaseProfiles>\1/igs' "$tmpFile" update pom.xml 'deploy releases to the SciJava repository' fi From b0a2e8c12c06922a2ed74d09501cef8cde214fd8 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 14:20:20 -0500 Subject: [PATCH 290/429] Rename github-action-ci to ci-setup-github-actions This clarifies that the ci-setup-* scripts go with the ci-build script. --- github-action-ci.sh => ci-setup-github-actions.sh | 2 +- github-actionify.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename github-action-ci.sh => ci-setup-github-actions.sh (67%) diff --git a/github-action-ci.sh b/ci-setup-github-actions.sh similarity index 67% rename from github-action-ci.sh rename to ci-setup-github-actions.sh index e368959..5f4b92a 100755 --- a/github-action-ci.sh +++ b/ci-setup-github-actions.sh @@ -1,7 +1,7 @@ #!/bin/bash # -# github-action-ci.sh - A script to set up ci-related environment variables from GitHub Actions +# ci-setup-github-actions.sh - Set CI-related environment variables from GitHub Actions. # echo "BUILD_REPOSITORY=${GITHUB_REPOSITORY}" diff --git a/github-actionify.sh b/github-actionify.sh index 1864191..94e51a3 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -214,8 +214,8 @@ EOL # Add/update the GitHub Action setup script. cat >"$tmpFile" < Date: Thu, 1 Jul 2021 14:24:59 -0500 Subject: [PATCH 291/429] github-actionify: remove unneeded env var notes There are many other environment variables besides only these. Their purpose can be gleaned from context within the script. --- github-actionify.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index 94e51a3..ad9008d 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -4,11 +4,6 @@ # # Script for enabling or updating GitHub Action builds for a given repository. -# Environment variables: -# $EXEC - an optional prefix for bash commands (for example, if $EXEC=sudo, then the commands will be run as super user access) -# $@ - all positional parameters -# $0, $1, ... - specific positional parameters for each method - #set -e dir="$(dirname "$0")" From 4ca65b98ec221158d6cd6de82e55412d57efa8f4 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 14:26:11 -0500 Subject: [PATCH 292/429] github-actionify: remove unused logic All secrets are now added on a per-org basis directly to GitHub. They do not need to be present on the local machine running the script. --- github-actionify.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index ad9008d..d8e507c 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -15,8 +15,6 @@ ciConfigBuildMain=$ciDir/$ciSlugBuildMain ciConfigBuildPR=$ciDir/$ciSlugBuildPR ciSetupScript=$ciDir/setup.sh ciBuildScript=$ciDir/build.sh -credentialsDir=$HOME/.scijava/credentials -varsFile=$credentialsDir/vars pomMinVersion='17.1.1' tmpFile=github-actionify.tmp msgPrefix="CI: " @@ -34,11 +32,6 @@ check() { done } -var() { - grep "^$1=" "$varsFile" || - die "$1 not found in $varsFile" -} - update() { file=$1 msg=$2 From 26dc9d43cb7e2e67e754b593404ac213b7f7e128 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 14:31:05 -0500 Subject: [PATCH 293/429] github-actionify: tweak whitespace --- github-actionify.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index d8e507c..fc5146b 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -67,7 +67,7 @@ process() { # -- Git sanity checks -- - repoSlug=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="scm"]/*[local-name()="connection"]' pom.xml|sed 's_.*github.com[:/]\(.*\)<.*_\1_') + repoSlug=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="scm"]/*[local-name()="connection"]' pom.xml | sed 's_.*github.com[:/]\(.*\)<.*_\1_') test "$repoSlug" && info "Repository = $repoSlug" || die 'Could not determine GitHub repository slug' case "$repoSlug" in *.git) @@ -88,7 +88,7 @@ process() { # -- POM sanity checks -- - parent=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="parent"]/*[local-name()="artifactId"]' pom.xml|sed 's/[^>]*>//'|sed 's/<.*//') + parent=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="parent"]/*[local-name()="artifactId"]' pom.xml | sed 's/[^>]*>//' | sed 's/<.*//') if [ -z "$SKIP_PARENT_CHECK" ] then test "$parent" = "pom-scijava" || @@ -220,7 +220,7 @@ EOL # Upgrade version of pom-scijava. if [ -z "$SKIP_PARENT_CHECK" ] then - version=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="parent"]/*[local-name()="version"]' pom.xml|sed 's/[^>]*>//'|sed 's/<.*//') + version=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="parent"]/*[local-name()="version"]' pom.xml | sed 's/[^>]*>//' | sed 's/<.*//') # HACK: Using a lexicographic comparison here is imperfect. if [ "$version" \< "$pomMinVersion" ] then From bf08216522d1fad6e8c4cf8e6b514186a081c481 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 14:31:14 -0500 Subject: [PATCH 294/429] github-actionify: remove unused prerequisite --- github-actionify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-actionify.sh b/github-actionify.sh index fc5146b..08aef22 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -285,7 +285,7 @@ secrets to your GitHub organization if they aren't already present. EOL # call check method to verify prerequisites -check git sed cut perl xmllint +check git sed perl xmllint # parse arguments EXEC=: From d9dc88916a4a5adb2058e12a5db2df6816009a58 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 14:32:32 -0500 Subject: [PATCH 295/429] github-actionify: fix "Travis CI" replacement --- github-actionify.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index 08aef22..b5eff40 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -97,8 +97,8 @@ process() { # Change pom.xml from Travis CI to GitHub Actions domain="github.com" - sed 's/Travis CI/GitHub Actions/g' pom.xml >"$tmpFile" - sed "s;travis-ci.*;github.com/$repoSlug/actions;g" pom.xml >"$tmpFile" + sed 's/Travis CI/GitHub Actions/g' pom.xml | + sed "s;travis-ci.*;github.com/$repoSlug/actions;g" >"$tmpFile" update pom.xml "switch from Travis CI to GitHub Actions" # -- GitHub Action sanity checks -- From 9e104376c0edfd8fa80fa929ba9cce0e9d9eae0b Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 14:37:38 -0500 Subject: [PATCH 296/429] github-actionify: cache only .m2/repository Not the entire .m2 structure. Because .m2/settings.xml is generated by the build script only when it's not already present, and if we cache it, then CI builds may get stuck with old versions of settings.xml that aren't regenerated when they should be. --- github-actionify.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index b5eff40..cfd6540 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -133,7 +133,7 @@ jobs: env: cache-name: cache-m2 with: - path: ~/.m2 + path: ~/.m2/repository key: \${{ runner.os }}-build-\${{ env.cache-name }} restore-keys: | \${{ runner.os }}-build-\${{ env.cache-name }}- @@ -180,7 +180,7 @@ jobs: env: cache-name: cache-m2 with: - path: ~/.m2 + path: ~/.m2/repository key: \${{ runner.os }}-build-\${{ env.cache-name }} restore-keys: | \${{ runner.os }}-build-\${{ env.cache-name }}- From d26711d5f8346fbba471e61e7f5b63e4e7b882bc Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 14:38:29 -0500 Subject: [PATCH 297/429] github-actionify: remove old CI configuration --- github-actionify.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/github-actionify.sh b/github-actionify.sh index cfd6540..59ab6e8 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -270,6 +270,12 @@ EOL test -f README.md && cat README.md >>"$tmpFile" update README.md 'add README.md badge link' fi + + # remove old Travis CI configuration + git rm -rf .travis.yml .travis + $EXEC git diff-index --quiet HEAD -- && + info "No old CI configuration to remove." || + $EXEC git commit -m "${msgPrefix}remove Travis CI configuration" } cat < Date: Thu, 1 Jul 2021 14:51:21 -0500 Subject: [PATCH 298/429] ci-build: fix project fork detection Let's make BUILD_REPOSITORY be the complete SCM URL as specified in the pom.xml. That way, other ci-setup- scripts can set the URL according to their platform as well, and we compare the whole string. --- ci-build.sh | 19 ++++++++----------- ci-setup-github-actions.sh | 4 ++-- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index 09a2ebf..750dfb3 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -2,11 +2,10 @@ # # ci-build.sh - A script to build and/or release SciJava-based projects -# automatically using a continuous integration -# service. +# automatically using a continuous integration service. # -# Required environment variables: -# BUILD_REPOSITORY - the repository slug (org/repo) running the current build +# Optional environment variables: +# BUILD_REPOSITORY - the repository URL running the current build dir="$(dirname "$0")" @@ -77,20 +76,18 @@ EOL # Determine whether deploying will be possible. deployOK= - ciURL=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${project.ciManagement.url}' --non-recursive validate exec:exec 2>&1) + scmURL=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${project.scm.url}' --non-recursive validate exec:exec 2>&1) if [ $? -ne 0 ]; then echo "No deploy -- could not extract ciManagement URL" echo "Output of failed attempt follows:" - echo "$ciURL" + echo "$scmURL" else - ciRepo=${ciURL##*/} - ciPrefix=${ciURL%/*} - ciOrg=${ciPrefix##*/} + scmRepo=${scmURL##*/} if [ ! "$SIGNING_ASC" ] || [ ! "$GPG_KEY_NAME" ] || [ ! "$GPG_PASSPHRASE" ] || [ ! "$MAVEN_PASS" ] || [ ! "$OSSRH_PASS" ]; then echo "No deploy -- secure environment variables not available" - elif [ "$BUILD_REPOSITORY" != "$ciOrg/$ciRepo" ]; then - echo "No deploy -- repository fork: $BUILD_REPOSITORY != $ciOrg/$ciRepo" + elif [ "$BUILD_REPOSITORY" -a "$BUILD_REPOSITORY" != "$scmRepo" ]; then + echo "No deploy -- repository fork: $BUILD_REPOSITORY != $scmRepo" else echo "All checks passed for artifact deployment" deployOK=1 diff --git a/ci-setup-github-actions.sh b/ci-setup-github-actions.sh index 5f4b92a..000f6d3 100755 --- a/ci-setup-github-actions.sh +++ b/ci-setup-github-actions.sh @@ -4,8 +4,8 @@ # ci-setup-github-actions.sh - Set CI-related environment variables from GitHub Actions. # -echo "BUILD_REPOSITORY=${GITHUB_REPOSITORY}" +echo "BUILD_REPOSITORY=https://github.com/${GITHUB_REPOSITORY}" echo "BUILD_OS=${RUNNER_OS}" -echo "BUILD_REPOSITORY=${GITHUB_REPOSITORY}" >> $GITHUB_ENV +echo "BUILD_REPOSITORY=https://github.com/${GITHUB_REPOSITORY}" >> $GITHUB_ENV echo "BUILD_OS=${RUNNER_OS}" >> $GITHUB_ENV From 1fe2d53b4882431ecdd045670b36b6ece152c1d9 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 14:57:44 -0500 Subject: [PATCH 299/429] ci-build: fix macOS detection The GitHub Actions Linux container has Linuxbrew installed! So we cannot merely test for the existence of the brew command. --- ci-build.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index 750dfb3..83ef615 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -9,6 +9,11 @@ dir="$(dirname "$0")" +MACOS= +case "$(uname -s)" + Darwin) MACOS=1;; +esac + success=0 checkSuccess() { # Log non-zero exit code. @@ -95,7 +100,7 @@ EOL fi # Install GPG on macOS - if which brew >/dev/null 2>/dev/null; then + if "$MACOS"; then HOMEBREW_NO_AUTO_UPDATE=1 brew install gnupg2 fi From 39cdd2baeee8a8d068b70a3fc94b1850e79a7b59 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 15:00:43 -0500 Subject: [PATCH 300/429] ci-build: fix SCM URL validation -- for real --- ci-build.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index 83ef615..9c24085 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -88,11 +88,10 @@ EOL echo "Output of failed attempt follows:" echo "$scmURL" else - scmRepo=${scmURL##*/} if [ ! "$SIGNING_ASC" ] || [ ! "$GPG_KEY_NAME" ] || [ ! "$GPG_PASSPHRASE" ] || [ ! "$MAVEN_PASS" ] || [ ! "$OSSRH_PASS" ]; then echo "No deploy -- secure environment variables not available" - elif [ "$BUILD_REPOSITORY" -a "$BUILD_REPOSITORY" != "$scmRepo" ]; then - echo "No deploy -- repository fork: $BUILD_REPOSITORY != $scmRepo" + elif [ "$BUILD_REPOSITORY" -a "$BUILD_REPOSITORY" != "$scmURL" ]; then + echo "No deploy -- repository fork: $BUILD_REPOSITORY != $scmURL" else echo "All checks passed for artifact deployment" deployOK=1 From 641035020b3fdd1a40ee57eb9d4f04ce815ccae4 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 15:02:16 -0500 Subject: [PATCH 301/429] ci-build: fix embarrassing syntax error I need to remember that you *can* simply run ci-build.sh locally now, which is a fine way to test changes to the script, without needing to push them and rerun CI jobs. --- ci-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index 9c24085..b229609 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -10,7 +10,7 @@ dir="$(dirname "$0")" MACOS= -case "$(uname -s)" +case "$(uname -s)" in Darwin) MACOS=1;; esac From cd4812805098427125bd63b9d448fb8a0536f6c9 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 15:18:42 -0500 Subject: [PATCH 302/429] ci-build: fix another embarrassing syntax error --- ci-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index b229609..9f7f0ba 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -99,7 +99,7 @@ EOL fi # Install GPG on macOS - if "$MACOS"; then + if [ "$MACOS" ]; then HOMEBREW_NO_AUTO_UPDATE=1 brew install gnupg2 fi From dab3c39aee6e99abaf623865b2e76e626b4078ed Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 15:21:58 -0500 Subject: [PATCH 303/429] ci-build: master -> main branch --- ci-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index 9f7f0ba..4bb3c47 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -118,7 +118,7 @@ EOL BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' if [ "$deployOK" ]; then echo - echo "== Building and deploying master SNAPSHOT ==" + echo "== Building and deploying main branch SNAPSHOT ==" mvn -Pdeploy-to-scijava $BUILD_ARGS deploy checkSuccess $? elif [ "$deployOK" -a -f release.properties ]; then From efb9e73fe1175e78e9f26e73d25506c4044b1999 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Jul 2021 15:24:38 -0500 Subject: [PATCH 304/429] github-actionify: simulate Travis removal w/o -f --- github-actionify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-actionify.sh b/github-actionify.sh index 59ab6e8..aba27fa 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -272,7 +272,7 @@ EOL fi # remove old Travis CI configuration - git rm -rf .travis.yml .travis + $EXEC git rm -rf .travis.yml .travis $EXEC git diff-index --quiet HEAD -- && info "No old CI configuration to remove." || $EXEC git commit -m "${msgPrefix}remove Travis CI configuration" From 14ae2eb804164e8781b0a6a450f85ee800e0681f Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Fri, 2 Jul 2021 10:37:30 -0500 Subject: [PATCH 305/429] github-actionify: build on release tags See https://github.com/scijava/scijava-common/commit/d24acd6c686227a3ff9a875b3c4024e6f94ae20f --- github-actionify.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/github-actionify.sh b/github-actionify.sh index aba27fa..70807ba 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -120,6 +120,8 @@ on: push: branches: - $defaultBranch + tags: + - "*-[0-9]+.*" jobs: build: From dd4a2e096a0cb8516b74dff29da16b6b078aea2d Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 27 Jul 2021 15:11:34 -0500 Subject: [PATCH 306/429] ci-build.sh: fix deployment of release versions As written, the release:perform branch never triggered, because the branch in question was never being checked. Let's first check for a release.properties, and only assume it's supposed to deploy from the main branch when one isn't present. --- ci-build.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index 4bb3c47..9321f2e 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -116,12 +116,7 @@ EOL # Run the build. BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' - if [ "$deployOK" ]; then - echo - echo "== Building and deploying main branch SNAPSHOT ==" - mvn -Pdeploy-to-scijava $BUILD_ARGS deploy - checkSuccess $? - elif [ "$deployOK" -a -f release.properties ]; then + if [ "$deployOK" -a -f release.properties ]; then echo echo "== Cutting and deploying release version ==" mvn -B $BUILD_ARGS release:perform @@ -136,6 +131,11 @@ EOL echo " password $MAVEN_PASS" >>"$HOME/.netrc" && sh maven-helper.sh invalidate-cache "$ga" checkSuccess $? + elif [ "$deployOK" ]; then + echo + echo "== Building and deploying main branch SNAPSHOT ==" + mvn -Pdeploy-to-scijava $BUILD_ARGS deploy + checkSuccess $? else echo echo "== Building the artifact locally only ==" From 9db97dd1c65ff79faca75aa69c8fbdc79380b4d3 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 27 Jul 2021 15:31:48 -0500 Subject: [PATCH 307/429] Avoid error when GPG signing --- ci-build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci-build.sh b/ci-build.sh index 9321f2e..d638e1b 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -103,6 +103,9 @@ EOL HOMEBREW_NO_AUTO_UPDATE=1 brew install gnupg2 fi + # Avoid GPG error: "signing failed: Inappropriate ioctl for device" + export GPG_TTY=$(tty) + # Import the GPG signing key. keyFile=.ci/signingkey.asc if [ "$deployOK" ]; then From f1d41291b0e39dc9a74b9ac6394b8c7152fbc033 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 27 Jul 2021 16:20:38 -0500 Subject: [PATCH 308/429] ci-build.sh: output version of gpg tool --- ci-build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-build.sh b/ci-build.sh index d638e1b..3e09047 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -113,6 +113,7 @@ EOL mkdir -p .ci echo "$SIGNING_ASC" > "$keyFile" ls -la "$keyFile" + gpg --version gpg --batch --fast-import "$keyFile" checkSuccess $? fi From c96fc0eed4b38e0dfadb378ba2c528c1faac48b2 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 27 Jul 2021 16:30:17 -0500 Subject: [PATCH 309/429] ci-build.sh: do not stomp existing Maven settings It's nice to be able to run ci-build.sh locally, to test the build. But not if it overwrites precious existing Maven settings with blanks. --- ci-build.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index 3e09047..450267c 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -36,11 +36,15 @@ if [ -f pom.xml ]; then # Populate the settings.xml configuration. mkdir -p "$HOME/.m2" settingsFile="$HOME/.m2/settings.xml" - customSettings=.ci/settings.xml - if [ -f "$customSettings" ]; then - cp "$customSettings" "$settingsFile" + if [ -e "$settingsFile" ] + then + echo "[WARNING] $settingsFile already exists; skipping generation." else - cat >"$settingsFile" <"$settingsFile" < @@ -60,7 +64,7 @@ if [ -f pom.xml ]; then EOL - cat >>"$settingsFile" <>"$settingsFile" < gpg @@ -77,6 +81,7 @@ EOL EOL + fi fi # Determine whether deploying will be possible. From ebff6f77c6e17335882b0eda45365566d158e84f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 27 Jul 2021 17:12:29 -0500 Subject: [PATCH 310/429] ci-build.sh: force maven-gpg-plugin to 3.0.1+ Otherwise, newer versions of gpg won't sign successfully. --- ci-build.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ci-build.sh b/ci-build.sh index 450267c..758f528 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -128,6 +128,19 @@ EOL if [ "$deployOK" -a -f release.properties ]; then echo echo "== Cutting and deploying release version ==" + + # HACK: Use a newer version of maven-gpg-plugin, if a too-old version is declared. + maven_gpg_plugin_version=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${maven-gpg-plugin.version}' --non-recursive validate exec:exec 2>&1) + case "$maven_gpg_plugin_version" in + 0.*|1.*|2.*|3.0.0) + echo "--> Forcing maven-gpg-plugin version from $maven_gpg_plugin_version to 3.0.1" + BUILD_ARGS="$BUILD_ARGS -Dmaven-gpg-plugin.version=3.0.1" + ;; + *) + echo "--> maven-gpg-plugin version OK: $maven_gpg_plugin_version" + ;; + esac + mvn -B $BUILD_ARGS release:perform checkSuccess $? echo "== Invalidating SciJava Maven repository cache ==" From 49de079f064de146e0171fe37a163a845885ef5e Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 27 Jul 2021 17:29:03 -0500 Subject: [PATCH 311/429] ci-build.sh: pass maven-gpg-plugin.version along The release:perform goal forks the Maven execution, so we need to wrap the property assignment in "-Darguments=..." also. --- ci-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index 758f528..c10e28e 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -134,7 +134,7 @@ EOL case "$maven_gpg_plugin_version" in 0.*|1.*|2.*|3.0.0) echo "--> Forcing maven-gpg-plugin version from $maven_gpg_plugin_version to 3.0.1" - BUILD_ARGS="$BUILD_ARGS -Dmaven-gpg-plugin.version=3.0.1" + BUILD_ARGS="$BUILD_ARGS -Dmaven-gpg-plugin.version=3.0.1 -Darguments=-Dmaven-gpg-plugin.version=3.0.1" ;; *) echo "--> maven-gpg-plugin version OK: $maven_gpg_plugin_version" From 8a2cf7be8789598f8d6748bb3cef337d16be8504 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 27 Jul 2021 21:37:37 -0500 Subject: [PATCH 312/429] ci-build.sh: use consistent formatting and syntax --- ci-build.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index c10e28e..b0b30d8 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -36,8 +36,7 @@ if [ -f pom.xml ]; then # Populate the settings.xml configuration. mkdir -p "$HOME/.m2" settingsFile="$HOME/.m2/settings.xml" - if [ -e "$settingsFile" ] - then + if [ -e "$settingsFile" ]; then echo "[WARNING] $settingsFile already exists; skipping generation." else customSettings=.ci/settings.xml @@ -208,7 +207,7 @@ if [ -f environment.yml ]; then fi # Execute Jupyter notebooks. -if which jupyter >/dev/null 2>/dev/null; then +if which jupyter >/dev/null 2>&1; then echo ::group::"= Jupyter notebooks =" # NB: This part is fiddly. We want to loop over files even with spaces, # so we use the "find ... -print0 | while read $'\0' ..." idiom. From 786802e90949c95d4e0b8d1bb5f3bc450d614d3f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 27 Jul 2021 21:38:25 -0500 Subject: [PATCH 313/429] ci-build.sh: reload gpg-agent before using gpg --- ci-build.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index b0b30d8..3f58921 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -107,7 +107,7 @@ EOL HOMEBREW_NO_AUTO_UPDATE=1 brew install gnupg2 fi - # Avoid GPG error: "signing failed: Inappropriate ioctl for device" + # Avoid "signing failed: Inappropriate ioctl for device" error. export GPG_TTY=$(tty) # Import the GPG signing key. @@ -128,7 +128,7 @@ EOL echo echo "== Cutting and deploying release version ==" - # HACK: Use a newer version of maven-gpg-plugin, if a too-old version is declared. + # HACK: Use maven-gpg-plugin 3.0.1+. Avoids "signing failed: No such file or directory" error. maven_gpg_plugin_version=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${maven-gpg-plugin.version}' --non-recursive validate exec:exec 2>&1) case "$maven_gpg_plugin_version" in 0.*|1.*|2.*|3.0.0) @@ -140,8 +140,16 @@ EOL ;; esac + # HACK: Try restarting the gpg agent. Avoids "signing failed: No pinentry" error. + if { pgrep gpg-agent >/dev/null && which gpgconf >/dev/null 2>&1; } then + echo '--> Restarting gpg-agent' + gpgconf --reload gpg-agent + checkSuccess $? + fi + mvn -B $BUILD_ARGS release:perform checkSuccess $? + echo "== Invalidating SciJava Maven repository cache ==" curl -fsLO https://raw.githubusercontent.com/scijava/scijava-scripts/master/maven-helper.sh && gav=$(sh maven-helper.sh gav-from-pom pom.xml) && From 0d6914173e18ab3f9048a9f0405cd3c8b3d32da6 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 3 Aug 2021 11:21:09 -0500 Subject: [PATCH 314/429] ci-build.sh: change how settings.xml is generated It seems that settings.xml sometimes(?) already exists on GitHub Actions infrastructure. Either that, or it is/was getting cached between runs (although that is not intended). Regardless: builds are now failing to deploy, because the already-existing settings.xml does not contain the proper credentials, and a correct one is no longer being written out. This commit changes the script to overwrite settings.xml when either MAVEN_PASS or OSSRH_PASS is set, even if ~/.m2/settings.xml already exists. On a typical developer workstation, these variables won't be set, so running ci-build.sh locally will not stomp ~/.m2/settings.xml. --- ci-build.sh | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index 3f58921..761cdf7 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -36,14 +36,13 @@ if [ -f pom.xml ]; then # Populate the settings.xml configuration. mkdir -p "$HOME/.m2" settingsFile="$HOME/.m2/settings.xml" - if [ -e "$settingsFile" ]; then - echo "[WARNING] $settingsFile already exists; skipping generation." + customSettings=.ci/settings.xml + if [ -z "$MAVEN_PASS" -a -z "$OSSRH_PASS" ]; then + echo "[WARNING] Skipping settings.xml generation (no deployment credentials)." + elif [ -f "$customSettings" ]; then + cp "$customSettings" "$settingsFile" else - customSettings=.ci/settings.xml - if [ -f "$customSettings" ]; then - cp "$customSettings" "$settingsFile" - else - cat >"$settingsFile" <"$settingsFile" < @@ -62,8 +61,6 @@ if [ -f pom.xml ]; then $OSSRH_PASS -EOL - cat >>"$settingsFile" < gpg @@ -80,7 +77,6 @@ EOL EOL - fi fi # Determine whether deploying will be possible. From bdd932af4c4816f88cb6a52cdd7449f175934634 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 5 Aug 2021 20:26:07 -0500 Subject: [PATCH 315/429] ci-build.sh: escape certain password characters In XML, the following characters must be escaped: * Ampersand * Less than * Greater than And it's probably wise also to escape: * Double quote * Single quote So now we do that. --- ci-build.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index 761cdf7..bb5c33d 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -23,6 +23,11 @@ checkSuccess() { test $success -eq 0 && success=$1 } +# Credit: https://stackoverflow.com/a/12873723/1207769 +escapeXML() { + echo "$1" | sed 's/&/\&/g; s//\>/g; s/"/\"/g; s/'"'"'/\'/g' +} + # Build Maven projects. if [ -f pom.xml ]; then echo ::group::"= Maven build =" @@ -48,17 +53,17 @@ if [ -f pom.xml ]; then scijava.releases $MAVEN_USER - $MAVEN_PASS + $(escapeXML "$MAVEN_PASS") scijava.snapshots $MAVEN_USER - $MAVEN_PASS + $(escapeXML "$MAVEN_PASS") sonatype-nexus-releases scijava-ci - $OSSRH_PASS + $(escapeXML "$OSSRH_PASS") @@ -71,7 +76,7 @@ if [ -f pom.xml ]; then $GPG_KEY_NAME - $GPG_PASSPHRASE + $(escapeXML "$GPG_PASSPHRASE") From 45cc7f512de62b9b9e285264eb628ce4b5e516a7 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 23 Aug 2021 19:19:51 -0500 Subject: [PATCH 316/429] ci-build: use fixed version of maven-helper.sh This is more secure than curling the master branch. We can update as needed, if maven-helper.sh ever changes again. --- ci-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index bb5c33d..a8b3714 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -152,7 +152,7 @@ EOL checkSuccess $? echo "== Invalidating SciJava Maven repository cache ==" - curl -fsLO https://raw.githubusercontent.com/scijava/scijava-scripts/master/maven-helper.sh && + curl -fsLO https://raw.githubusercontent.com/scijava/scijava-scripts/bdd932af4c4816f88cb6a52cdd7449f175934634/maven-helper.sh && gav=$(sh maven-helper.sh gav-from-pom pom.xml) && ga=${gav%:*} && echo "--> Artifact to invalidate = $ga" && From 7e30f01ba0cdbcaed35755aa8b085bc19417ef4a Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 3 Sep 2021 12:16:10 -0500 Subject: [PATCH 317/429] Fix error with local variable statement The error is: melting-pot.sh: 428: local: 3.4.2: bad variable name I don't fully understand it (am unable to reproduce outside the melting pot script itself), but quoting the expression squelches the message. --- melting-pot.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 92b1d4c..2819b20 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -425,8 +425,8 @@ scmTag() { local a=$(artifactId "$1") local v=$(version "$1") local scmURL="$(scmURL "$1")" - local allTags=$(git ls-remote --tags "$scmURL" | sed 's/.*refs\/tags\///' || - error "$1: Invalid scm url: $scmURL") + local allTags="$(git ls-remote --tags "$scmURL" | sed 's/.*refs\/tags\///' || + error "$1: Invalid scm url: $scmURL")" for tag in "$a-$v" "$v" "v$v" do echo "$allTags" | grep -q "^$tag$" && { From 5bbed2569ce5eab6b5c698913d34d40fc55cbedd Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 9 Sep 2021 09:49:33 -0500 Subject: [PATCH 318/429] ci-build.sh: install pinentry if needed + missing Maybe this will fix the "No pinentry" error on GitHub Actions. --- ci-build.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index a8b3714..815b269 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -141,11 +141,16 @@ EOL ;; esac - # HACK: Try restarting the gpg agent. Avoids "signing failed: No pinentry" error. - if { pgrep gpg-agent >/dev/null && which gpgconf >/dev/null 2>&1; } then - echo '--> Restarting gpg-agent' - gpgconf --reload gpg-agent - checkSuccess $? + # HACK: Install pinentry helper program if missing. Avoids "signing failed: No pinentry" error. + if which pinentry >/dev/null 2>&1; then + echo '--> Installing missing pinentry helper for GPG' + sudo apt-get install -y pinentry-tty + # HACK: Restart the gpg agent, to notice the newly installed pinentry. + if { pgrep gpg-agent >/dev/null && which gpgconf >/dev/null 2>&1; } then + echo '--> Restarting gpg-agent' + gpgconf --reload gpg-agent + checkSuccess $? + fi fi mvn -B $BUILD_ARGS release:perform From 028f23940f68a75ca567272aa6a550502fa43ff6 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 16 Sep 2021 12:20:43 -0500 Subject: [PATCH 319/429] github-actionify.sh: improve DRYness The two workflows share a lot of configuration. This commit moves each step to its own block, for reuse across the two, instead of copy/pasting. --- github-actionify.sh | 95 ++++++++++++++++++++------------------------- 1 file changed, 42 insertions(+), 53 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index 70807ba..84a39e9 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -110,27 +110,10 @@ process() { test -e "$ciBuildScript" && warn "$ciBuildScript already exists" test -e "$ciSetupScript" && warn "$ciSetupScript already exists" - # -- Do things -- - - # Add/update the main GitHub Actions configuration file. - cat >"$tmpFile" <"$tmpFile" < Date: Thu, 16 Sep 2021 12:22:27 -0500 Subject: [PATCH 320/429] github-actionify.sh: set up conda if needed We check whether conda is needed by the presence of an environment.yml file. --- github-actionify.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/github-actionify.sh b/github-actionify.sh index 84a39e9..1acc5f7 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -129,6 +129,8 @@ process() { with: java-version: '8' distribution: 'zulu'" + actionSetupConda="name: Set up conda + uses: s-weigand/setup-conda@v1" actionSetupCI="name: Set up CI environment run: $ciSetupScript" actionExecuteBuild="name: Execute the build @@ -162,6 +164,9 @@ jobs: - $actionCheckout - $actionCacheLocalRepo - $actionSetupJava +EOL + test -f environment.yml && echo " - $actionSetupConda" >>"$tmpFile" + cat >>"$tmpFile" <>"$tmpFile" + cat >>"$tmpFile" < Date: Thu, 16 Sep 2021 12:40:33 -0500 Subject: [PATCH 321/429] ci-build.sh: remove conda installation This needs to happen during setup phase, not build phase. --- ci-build.sh | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index 815b269..e0712d1 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -180,46 +180,6 @@ EOL echo ::endgroup:: fi -# Configure conda environment, if one is needed. -if [ -f environment.yml ]; then - echo ::group::"= Conda setup =" - - condaDir=$HOME/miniconda - condaSh=$condaDir/etc/profile.d/conda.sh - if [ ! -f "$condaSh" ]; then - echo - echo "== Installing conda ==" - python_version=${python -V} - python_version=${python_version:7:3} # get python version in the format like '2.7' - if [ "${python_version}" = "2.7" ]; then - wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh - else - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - fi - rm -rf "$condaDir" - bash miniconda.sh -b -p "$condaDir" - checkSuccess $? - fi - - echo - echo "== Updating conda ==" - . "$condaSh" && - conda config --set always_yes yes --set changeps1 no && - conda update -q conda && - conda info -a - checkSuccess $? - - echo - echo "== Configuring environment ==" - condaEnv=ci-scijava - test -d "$condaDir/envs/$condaEnv" && condaAction=update || condaAction=create - conda env "$condaAction" -n "$condaEnv" -f environment.yml && - conda activate "$condaEnv" - checkSuccess $? - - echo ::endgroup:: -fi - # Execute Jupyter notebooks. if which jupyter >/dev/null 2>&1; then echo ::group::"= Jupyter notebooks =" From 2a6b8206ca44b0441383e0c4b23a37d1a72002fc Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 16 Sep 2021 14:41:15 -0500 Subject: [PATCH 322/429] github-actionify.sh: install conda packages We install them into the base environment, which is already active thanks to setup-conda, to avoid problems activating a new environment. --- github-actionify.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/github-actionify.sh b/github-actionify.sh index 1acc5f7..faaec8d 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -130,7 +130,9 @@ process() { java-version: '8' distribution: 'zulu'" actionSetupConda="name: Set up conda - uses: s-weigand/setup-conda@v1" + uses: s-weigand/setup-conda@v1 + - name: Install conda packages + run: conda env update -f environment.yml -n base" actionSetupCI="name: Set up CI environment run: $ciSetupScript" actionExecuteBuild="name: Execute the build From aac002a236332dfad9fc132da0d8da617ad529cb Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 16 Sep 2021 15:16:38 -0500 Subject: [PATCH 323/429] ci-build.sh: try to fix the nbconvert loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The error on GitHub Actions was: ci-build.sh: 194: read: Illegal option -d find: ‘standard output’: Broken pipe find: write error Hopefully this updated version will work better. --- ci-build.sh | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index e0712d1..0e697a0 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -184,21 +184,20 @@ fi if which jupyter >/dev/null 2>&1; then echo ::group::"= Jupyter notebooks =" # NB: This part is fiddly. We want to loop over files even with spaces, - # so we use the "find ... -print0 | while read $'\0' ..." idiom. + # so we use the "find ... | while read ..." idiom. # However, that runs the piped expression in a subshell, which means # that any updates to the success variable will not persist outside - # the loop. So we suppress all stdout inside the loop, echoing only - # the final value of success upon completion, and then capture the - # echoed value back into the parent shell's success variable. - success=$(find . -name '*.ipynb' -print0 | { - while read -d $'\0' nbf; do - echo 1>&2 - echo "== $nbf ==" 1>&2 - jupyter nbconvert --execute --stdout "$nbf" >/dev/null - checkSuccess $? - done - echo $success - }) + # the loop. So we store non-zero success values into a temporary file, + # then capture the value back into the parent shell's success variable. + find . -name '*.ipynb' | while read f + do + echo + echo "== $nbf ==" + jupyter nbconvert --execute --stdout "$nbf" + checkSuccess $? + test "$success" -eq 0 || echo "$success" > success.tmp + done + test -f success.tmp && success=$(cat success.tmp) && rm success.tmp echo ::endgroup:: fi From 4cb5c13a56736bfccdd6980a395a84761da96c89 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 16 Sep 2021 15:31:26 -0500 Subject: [PATCH 324/429] ci-build.sh: fix variable name typo --- ci-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index 0e697a0..3f8e6d0 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -189,7 +189,7 @@ if which jupyter >/dev/null 2>&1; then # that any updates to the success variable will not persist outside # the loop. So we store non-zero success values into a temporary file, # then capture the value back into the parent shell's success variable. - find . -name '*.ipynb' | while read f + find . -name '*.ipynb' | while read nbf do echo echo "== $nbf ==" From 52bc25e98f5045362519d0649c4c474a54de530f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 16 Sep 2021 15:47:51 -0500 Subject: [PATCH 325/429] ci-build.sh: fix jupyter notebook execution --- ci-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index 3f8e6d0..0622fad 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -193,7 +193,7 @@ if which jupyter >/dev/null 2>&1; then do echo echo "== $nbf ==" - jupyter nbconvert --execute --stdout "$nbf" + jupyter nbconvert --to python --stdout --execute "$nbf" checkSuccess $? test "$success" -eq 0 || echo "$success" > success.tmp done From 2586f09341fdae861319c47cc286f8a2afca2719 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 17 Sep 2021 08:28:59 -0500 Subject: [PATCH 326/429] ci-build.sh: allow SCM URL .git suffix The .git suffix is not wrong, but it is optional. If the POM has included it, we don't want to consider the project a fork. --- ci-build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-build.sh b/ci-build.sh index 0622fad..fad1b39 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -87,6 +87,7 @@ EOL # Determine whether deploying will be possible. deployOK= scmURL=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${project.scm.url}' --non-recursive validate exec:exec 2>&1) + scmURL=${scmURL%.git} if [ $? -ne 0 ]; then echo "No deploy -- could not extract ciManagement URL" From 2a324eb4de6ad205dc56b15f91bb039ba3396541 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 26 Sep 2021 10:45:22 -0500 Subject: [PATCH 327/429] ci-build.sh: strip trailing slash from scmURL --- ci-build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-build.sh b/ci-build.sh index fad1b39..5b5c1ab 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -88,6 +88,7 @@ EOL deployOK= scmURL=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${project.scm.url}' --non-recursive validate exec:exec 2>&1) scmURL=${scmURL%.git} + scmURL=${scmURL%/} if [ $? -ne 0 ]; then echo "No deploy -- could not extract ciManagement URL" From abc3725f832445835c8eedad76202e2c0674bc81 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 4 Oct 2021 16:17:44 -0500 Subject: [PATCH 328/429] Generalize CI javadoc build script It should have no explicit dependence on GitHub Actions. This change removes the SSH configuration, in favor of assuming it has been set up externally. While we're at it: update slim-curve -> flimlib. --- github-javadoc.sh => ci-javadoc.sh | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) rename github-javadoc.sh => ci-javadoc.sh (75%) diff --git a/github-javadoc.sh b/ci-javadoc.sh similarity index 75% rename from github-javadoc.sh rename to ci-javadoc.sh index b680924..0a571af 100644 --- a/github-javadoc.sh +++ b/ci-javadoc.sh @@ -1,19 +1,20 @@ #!/bin/bash # -# github-javadoc.sh - A script to build the javadocs of a SciJava-based project. +# ci-build.sh - A script to build the javadocs of SciJava-based projects +# automatically using a continuous integration service. # # The following repositories are known to use this script: # # bonej-org/bonej-javadoc # fiji/fiji-javadoc +# flimlib/flimlib-javadoc # imagej/imagej-javadoc # imglib/imglib2-javadoc # scifio/scifio-javadoc # scijava/java3d-javadoc # scijava/scijava-javadoc -# slim-curve/slim-javadoc # uw-loci/loci-javadoc # Wait for a launched background command to complete, emitting @@ -42,15 +43,9 @@ ciRepo=${ciURL##*/} ciPrefix=${ciURL%/*} ciOrg=${ciPrefix##*/} gitBranch=$(git branch --show-current) # get current branch name -curl -o pull-request.txt https://api.github.com/repos/$ciOrg/$ciRepo/pulls >/dev/null 2>&1 # Check for pull requests -curl -o secure-env.txt https://api.github.com/orgs/$ciOrg/$ciRepo/secrets >/dev/null 2>&1 # Check for secure env var -if [ grep -q "documentation_url" secure-env.txt \ - -a ! grep -q "url" pull-request.txt \ - -a "$gitBranch" = master ] +if [ "$gitBranch" = main -o "$gitBranch" = master ] then project=$1 - openssl_key=$2 - openssl_iv=$3 # Populate the settings.xml configuration. mkdir -p "$HOME/.m2" @@ -95,12 +90,6 @@ EOL echo && echo "== Configuring environment ==" && - # Configure SSH. The file .github/javadoc.scijava.org.enc must contain - # an encrypted private RSA key for communicating with the git remote. - mkdir -p "$HOME/.ssh" && - openssl aes-256-cbc -K "$openssl_key" -iv "$openssl_iv" -in '.github/javadoc.scijava.org.enc' -out "$HOME/.ssh/id_rsa" -d && - chmod 400 "$HOME/.ssh/id_rsa" && - # Configure git settings. git config --global user.email "ci@scijava.org" && git config --global user.name "SciJava CI" && @@ -122,7 +111,7 @@ EOL test "$success" || exit 1 - git commit -m "Update $project javadocs (GitHub Actions build $WORKFLOW_RUN_NUMBER)" # Env var defined in GitHub Actions yml file + git commit -m "Update $project javadocs (via $ciOrg/$ciRepo)" git pull --rebase && git push -q origin gh-pages > /dev/null || exit 2 From 45ceb221479816314c6735889a202af539e5384e Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 4 Oct 2021 16:19:55 -0500 Subject: [PATCH 329/429] Make ci-javadoc.sh script executable --- ci-javadoc.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci-javadoc.sh diff --git a/ci-javadoc.sh b/ci-javadoc.sh old mode 100644 new mode 100755 From cebd158825169848272f6124c03d3f84bc7f07e1 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 5 Oct 2021 14:35:53 -0500 Subject: [PATCH 330/429] ci-javadoc.sh: fix header comment --- ci-javadoc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-javadoc.sh b/ci-javadoc.sh index 0a571af..c54cb15 100755 --- a/ci-javadoc.sh +++ b/ci-javadoc.sh @@ -1,8 +1,8 @@ #!/bin/bash # -# ci-build.sh - A script to build the javadocs of SciJava-based projects -# automatically using a continuous integration service. +# ci-javadoc.sh - A script to build the javadocs of SciJava-based projects +# automatically using a continuous integration service. # # The following repositories are known to use this script: From ffac804e4836f19a440d1ff36e07c121da6e6ad6 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 5 Oct 2021 16:10:11 -0500 Subject: [PATCH 331/429] Update list of associated projects --- ci-javadoc.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ci-javadoc.sh b/ci-javadoc.sh index c54cb15..dd4cec2 100755 --- a/ci-javadoc.sh +++ b/ci-javadoc.sh @@ -7,15 +7,16 @@ # The following repositories are known to use this script: # -# bonej-org/bonej-javadoc -# fiji/fiji-javadoc -# flimlib/flimlib-javadoc -# imagej/imagej-javadoc -# imglib/imglib2-javadoc -# scifio/scifio-javadoc -# scijava/java3d-javadoc -# scijava/scijava-javadoc -# uw-loci/loci-javadoc +# - bonej-org/bonej-javadoc +# - fiji/fiji-javadoc +# - flimlib/flimlib-javadoc +# - imagej/imagej-javadoc +# - imglib/imglib2-javadoc +# - scifio/scifio-javadoc +# - scenerygraphics/sciview-javadoc +# - scijava/java3d-javadoc +# - scijava/scijava-javadoc +# - uw-loci/loci-javadoc # Wait for a launched background command to complete, emitting # an occasional message to avoid long periods without output. From 55c59e0a6d26754446db2b47b528d223904b0e46 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 5 Oct 2021 16:10:33 -0500 Subject: [PATCH 332/429] Fix folder path --- ci-javadoc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-javadoc.sh b/ci-javadoc.sh index dd4cec2..7621e9b 100755 --- a/ci-javadoc.sh +++ b/ci-javadoc.sh @@ -104,7 +104,7 @@ EOL # Update the relevant javadocs. cd javadoc.scijava.org && rm -rf "$project" && - mv ../target/apidocs "$project" && + mv target/apidocs "$project" && # Commit and push the changes. git add "$project" && From 0b48bdbded9c47077dd1cd7830db285d1b9dfb50 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 5 Oct 2021 16:21:40 -0500 Subject: [PATCH 333/429] Fix folder path again It was right the first time. This reverts commit 55c59e0a6d26754446db2b47b528d223904b0e46. --- ci-javadoc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-javadoc.sh b/ci-javadoc.sh index 7621e9b..dd4cec2 100755 --- a/ci-javadoc.sh +++ b/ci-javadoc.sh @@ -104,7 +104,7 @@ EOL # Update the relevant javadocs. cd javadoc.scijava.org && rm -rf "$project" && - mv target/apidocs "$project" && + mv ../target/apidocs "$project" && # Commit and push the changes. git add "$project" && From 8c862b4bbe8b089ab27cf35813768a01b2c66a73 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 5 Oct 2021 16:37:54 -0500 Subject: [PATCH 334/429] ci-javadoc.sh: strip /actions suffix from ciURL --- ci-javadoc.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-javadoc.sh b/ci-javadoc.sh index dd4cec2..3390402 100755 --- a/ci-javadoc.sh +++ b/ci-javadoc.sh @@ -40,6 +40,7 @@ keep_alive() { } ciURL=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${project.ciManagement.url}' --non-recursive validate exec:exec 2>&1) +ciURL=${ciURL%/actions} ciRepo=${ciURL##*/} ciPrefix=${ciURL%/*} ciOrg=${ciPrefix##*/} From f61fe16aefb91161193430877678d0e6e5341689 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 11 Oct 2021 16:03:27 -0500 Subject: [PATCH 335/429] Generalize main branch logic --- check-branch.sh | 8 +++++++- github-actionify.sh | 4 ++-- remote-branch-info.sh | 13 +++++++------ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/check-branch.sh b/check-branch.sh index 8496e55..53744ec 100755 --- a/check-branch.sh +++ b/check-branch.sh @@ -4,7 +4,13 @@ # recording whether the build passes or fails for each. commits=$@ -test "$commits" || commits=$(git rev-list HEAD ^master | sed '1!G;h;$!d') + +remote=$(git rev-parse --symbolic-full-name HEAD@{u}) +remote=${remote%/*} +remote=${remote#refs/remotes/} +headBranch=$(git remote show "$remote" | grep HEAD | sed 's/ *HEAD branch: //') + +test "$commits" || commits=$(git rev-list HEAD "^$headBranch" | sed '1!G;h;$!d') # NB: The sed line above reverses the order of the commits. # See: http://stackoverflow.com/a/744093 diff --git a/github-actionify.sh b/github-actionify.sh index faaec8d..f8836f7 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -203,7 +203,7 @@ EOL # Add/update the GitHub Action setup script. cat >"$tmpFile" <"$tmpFile" < Date: Tue, 12 Oct 2021 10:24:32 -0500 Subject: [PATCH 336/429] dep-versions.pl: fix bug due to colorized output --- dep-versions.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dep-versions.pl b/dep-versions.pl index 3698568..96bf595 100755 --- a/dep-versions.pl +++ b/dep-versions.pl @@ -5,7 +5,7 @@ use strict; -my @deps = `mvn dependency:list`; +my @deps = `mvn -B dependency:list`; my $active = 0; for my $dep (@deps) { From 352a9e95c5495ed3fa70526cdedab8ea9945821a Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sat, 6 Nov 2021 21:34:35 -0500 Subject: [PATCH 337/429] github-actionify.sh: eliminate xmllint dependency Let's just use our old friend grep instead. It's good enough, it's smart enough, and people like it. --- github-actionify.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index f8836f7..2b9be2b 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -67,7 +67,7 @@ process() { # -- Git sanity checks -- - repoSlug=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="scm"]/*[local-name()="connection"]' pom.xml | sed 's_.*github.com[:/]\(.*\)<.*_\1_') + repoSlug=$(grep '' pom.xml | sed 's;.*github.com[/:]\(.*/.*\).*;\1;') test "$repoSlug" && info "Repository = $repoSlug" || die 'Could not determine GitHub repository slug' case "$repoSlug" in *.git) @@ -88,7 +88,7 @@ process() { # -- POM sanity checks -- - parent=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="parent"]/*[local-name()="artifactId"]' pom.xml | sed 's/[^>]*>//' | sed 's/<.*//') + parent=$(grep -A4 '' pom.xml | grep '' | sed 's;.*\(.*\).*;\1;') if [ -z "$SKIP_PARENT_CHECK" ] then test "$parent" = "pom-scijava" || @@ -221,7 +221,7 @@ EOL # Upgrade version of pom-scijava. if [ -z "$SKIP_PARENT_CHECK" ] then - version=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="parent"]/*[local-name()="version"]' pom.xml | sed 's/[^>]*>//' | sed 's/<.*//') + version=$(grep -A4 '' pom.xml | grep '' | sed 's;.*\(.*\).*;\1;') # HACK: Using a lexicographic comparison here is imperfect. if [ "$version" \< "$pomMinVersion" ] then @@ -292,7 +292,7 @@ secrets to your GitHub organization if they aren't already present. EOL # call check method to verify prerequisites -check git sed perl xmllint +check git grep sed perl # parse arguments EXEC=: From 9f53c7f281df656bf0e8d5b24db2f8fc268feb7d Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 14 Nov 2021 09:58:58 -0600 Subject: [PATCH 338/429] github-actionify.sh: make badge logic smarter --- github-actionify.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/github-actionify.sh b/github-actionify.sh index 2b9be2b..104d662 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -264,6 +264,9 @@ EOL info "Updating README.md GitHub Action badge" sed "s;travis-ci.*;$domain/$repoSlug/actions/$ciSlugBuildMain/badge.svg)](https://$domain/$repoSlug/actions/$ciSlugBuildMain);g" README.md >"$tmpFile" update README.md 'update README.md badge link' + elif grep -qF "$domain/$repoSlug/actions/$ciSlugBuildMain/badge.svg" README.md >/dev/null 2>&1 + then + info "GitHub Action badge already present in README.md" else info "Adding GitHub Action badge to README.md" echo "[![](https://$domain/$repoSlug/actions/$ciSlugBuildMain/badge.svg)](https://$domain/$repoSlug/actions/$ciSlugBuildMain)" >"$tmpFile" From d9bb526f5a1c8b095c45f18dce5c83c9fd5cd8d5 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 19 Nov 2021 20:55:50 -0600 Subject: [PATCH 339/429] check-branch.sh: fix bug when branch name has a / --- check-branch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check-branch.sh b/check-branch.sh index 53744ec..5dc57fe 100755 --- a/check-branch.sh +++ b/check-branch.sh @@ -6,8 +6,8 @@ commits=$@ remote=$(git rev-parse --symbolic-full-name HEAD@{u}) -remote=${remote%/*} remote=${remote#refs/remotes/} +remote=${remote%%/*} headBranch=$(git remote show "$remote" | grep HEAD | sed 's/ *HEAD branch: //') test "$commits" || commits=$(git rev-list HEAD "^$headBranch" | sed '1!G;h;$!d') From 0311fb43c99e0da03ac00367150c5d1bd689ebab Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 6 Jan 2022 12:40:20 -0600 Subject: [PATCH 340/429] Remove javadoc building scripts All you have to do now is deploy your javadoc classifier JAR. See: https://github.com/scijava/javadoc-wrangler --- ci-javadoc.sh | 123 -------------------------------------------- travis-javadoc.sh | 127 ---------------------------------------------- 2 files changed, 250 deletions(-) delete mode 100755 ci-javadoc.sh delete mode 100644 travis-javadoc.sh diff --git a/ci-javadoc.sh b/ci-javadoc.sh deleted file mode 100755 index 3390402..0000000 --- a/ci-javadoc.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/bash - -# -# ci-javadoc.sh - A script to build the javadocs of SciJava-based projects -# automatically using a continuous integration service. -# - -# The following repositories are known to use this script: -# -# - bonej-org/bonej-javadoc -# - fiji/fiji-javadoc -# - flimlib/flimlib-javadoc -# - imagej/imagej-javadoc -# - imglib/imglib2-javadoc -# - scifio/scifio-javadoc -# - scenerygraphics/sciview-javadoc -# - scijava/java3d-javadoc -# - scijava/scijava-javadoc -# - uw-loci/loci-javadoc - -# Wait for a launched background command to complete, emitting -# an occasional message to avoid long periods without output. -# Return the same exit code as the launched command. -keep_alive() { - pid="$1" - if [ "$pid" = "" ] - then - echo "[ERROR] No PID given" - return - fi - i=0 - while kill -0 "$pid" 2>/dev/null; do - i=$((i+1)) - m=$((i/60)) - s=$((i%60)) - test $s -eq 0 && echo "[$m minutes elapsed]" - sleep 1 - done - wait "$pid" -} - -ciURL=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${project.ciManagement.url}' --non-recursive validate exec:exec 2>&1) -ciURL=${ciURL%/actions} -ciRepo=${ciURL##*/} -ciPrefix=${ciURL%/*} -ciOrg=${ciPrefix##*/} -gitBranch=$(git branch --show-current) # get current branch name -if [ "$gitBranch" = main -o "$gitBranch" = master ] -then - project=$1 - - # Populate the settings.xml configuration. - mkdir -p "$HOME/.m2" - settingsFile="$HOME/.m2/settings.xml" - customSettings=.github/settings.xml - if [ -f "$customSettings" ] - then - cp "$customSettings" "$settingsFile" - else - # NB: Use maven.scijava.org as sole mirror if defined in . - test -f pom.xml && grep -A 2 '' pom.xml | grep -q 'maven.scijava.org' && - cat >"$settingsFile" < - - - scijava-mirror - SciJava mirror - https://maven.scijava.org/content/groups/public/ - * - - - -EOL - fi - - # Emit some details useful for debugging. - # NB: We run once with -q to suppress the download messages, - # then again without it to emit the desired dependency tree. - mvn -B -q dependency:tree && - mvn -B dependency:tree && - - echo && - echo "== Generating javadoc ==" && - - # Build the javadocs. - (mvn -B -q -Pbuild-javadoc) & - keep_alive $! && - test -d target/apidocs && - # Strip out date stamps, to avoid spurious changes being committed. - sed -i'' -e '/\( Forcing maven-gpg-plugin version from $maven_gpg_plugin_version to 3.0.1" + BUILD_ARGS="$BUILD_ARGS -Dmaven-gpg-plugin.version=3.0.1 -Darguments=-Dmaven-gpg-plugin.version=3.0.1" + ;; + *) + echo "--> maven-gpg-plugin version OK: $maven_gpg_plugin_version" + ;; + esac + + # HACK: Install pinentry helper program if missing. Avoids "signing failed: No pinentry" error. + if which pinentry >/dev/null 2>&1; then + echo '--> Installing missing pinentry helper for GPG' + sudo apt-get install -y pinentry-tty + # HACK: Restart the gpg agent, to notice the newly installed pinentry. + if { pgrep gpg-agent >/dev/null && which gpgconf >/dev/null 2>&1; } then + echo '--> Restarting gpg-agent' + gpgconf --reload gpg-agent + checkSuccess $? + fi + fi + # --== BUILD EXECUTION ==-- # Run the build. @@ -172,31 +196,6 @@ EOL if [ "$deployOK" -a -f release.properties ]; then echo echo "== Cutting and deploying release version ==" - - # HACK: Use maven-gpg-plugin 3.0.1+. Avoids "signing failed: No such file or directory" error. - maven_gpg_plugin_version=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${maven-gpg-plugin.version}' --non-recursive validate exec:exec 2>&1) - case "$maven_gpg_plugin_version" in - 0.*|1.*|2.*|3.0.0) - echo "--> Forcing maven-gpg-plugin version from $maven_gpg_plugin_version to 3.0.1" - BUILD_ARGS="$BUILD_ARGS -Dmaven-gpg-plugin.version=3.0.1 -Darguments=-Dmaven-gpg-plugin.version=3.0.1" - ;; - *) - echo "--> maven-gpg-plugin version OK: $maven_gpg_plugin_version" - ;; - esac - - # HACK: Install pinentry helper program if missing. Avoids "signing failed: No pinentry" error. - if which pinentry >/dev/null 2>&1; then - echo '--> Installing missing pinentry helper for GPG' - sudo apt-get install -y pinentry-tty - # HACK: Restart the gpg agent, to notice the newly installed pinentry. - if { pgrep gpg-agent >/dev/null && which gpgconf >/dev/null 2>&1; } then - echo '--> Restarting gpg-agent' - gpgconf --reload gpg-agent - checkSuccess $? - fi - fi - mvn -B $BUILD_ARGS release:perform checkSuccess $? From 3ffc4a6f04666fc763427557a6b6446ee28ac669 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 24 Mar 2022 23:48:02 -0500 Subject: [PATCH 359/429] ci-build.sh: fix inverted pinentry test We want to install it when it's *not* present... -_- --- ci-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index d6c351e..3e3b0f0 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -178,7 +178,7 @@ EOL esac # HACK: Install pinentry helper program if missing. Avoids "signing failed: No pinentry" error. - if which pinentry >/dev/null 2>&1; then + if ! which pinentry >/dev/null 2>&1; then echo '--> Installing missing pinentry helper for GPG' sudo apt-get install -y pinentry-tty # HACK: Restart the gpg agent, to notice the newly installed pinentry. From a08f4115523b2ed3a9ec6867290a923ca519c6b4 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 24 Mar 2022 23:39:44 -0500 Subject: [PATCH 360/429] ci-build.sh: remove redundant flag --- ci-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index 3e3b0f0..a27fdf9 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -196,7 +196,7 @@ EOL if [ "$deployOK" -a -f release.properties ]; then echo echo "== Cutting and deploying release version ==" - mvn -B $BUILD_ARGS release:perform + mvn $BUILD_ARGS release:perform checkSuccess $? echo "== Invalidating SciJava Maven repository cache ==" From f9d592c558214c5fe2fc75a3584a5ceb53badd81 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 24 Mar 2022 23:43:33 -0500 Subject: [PATCH 361/429] ci-build.sh: move cache invalidation to post-build Let's do it after any deploy, even a snapshot. --- ci-build.sh | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index a27fdf9..eaebc37 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -198,17 +198,6 @@ EOL echo "== Cutting and deploying release version ==" mvn $BUILD_ARGS release:perform checkSuccess $? - - echo "== Invalidating SciJava Maven repository cache ==" - curl -fsLO https://raw.githubusercontent.com/scijava/scijava-scripts/bdd932af4c4816f88cb6a52cdd7449f175934634/maven-helper.sh && - gav=$(sh maven-helper.sh gav-from-pom pom.xml) && - ga=${gav%:*} && - echo "--> Artifact to invalidate = $ga" && - echo "machine maven.scijava.org" >"$HOME/.netrc" && - echo " login $MAVEN_USER" >>"$HOME/.netrc" && - echo " password $MAVEN_PASS" >>"$HOME/.netrc" && - sh maven-helper.sh invalidate-cache "$ga" - checkSuccess $? elif [ "$deployOK" ]; then echo echo "== Building and deploying main branch SNAPSHOT ==" @@ -223,6 +212,20 @@ EOL # --== POST-BUILD ACTIONS ==-- + if [ "$deployOK" -a "$success" -eq 0 ]; then + echo + echo "== Invalidating SciJava Maven repository cache ==" + curl -fsLO https://raw.githubusercontent.com/scijava/scijava-scripts/bdd932af4c4816f88cb6a52cdd7449f175934634/maven-helper.sh && + gav=$(sh maven-helper.sh gav-from-pom pom.xml) && + ga=${gav%:*} && + echo "--> Artifact to invalidate = $ga" && + echo "machine maven.scijava.org" >"$HOME/.netrc" && + echo " login $MAVEN_USER" >>"$HOME/.netrc" && + echo " password $MAVEN_PASS" >>"$HOME/.netrc" && + sh maven-helper.sh invalidate-cache "$ga" + checkSuccess $? + fi + echo ::endgroup:: fi From b1ecadffbe59e886982c995ee02d3767e8008466 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sat, 9 Apr 2022 11:09:49 -0500 Subject: [PATCH 362/429] Split common mvn exec code to function And add the -B -U flags to make it work better: * Batch mode avoids any coloration (though probably not an issue here). * Force checking for updates in case new releases are now present and/or any snapshot versions have changed. Avoids cached failures. --- ci-build.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index eaebc37..bebf466 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -28,6 +28,10 @@ escapeXML() { echo "$1" | sed 's/&/\&/g; s//\>/g; s/"/\"/g; s/'"'"'/\'/g' } +mavenEvaluate() { + mvn -B -U -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args="$1" --non-recursive validate exec:exec 2>&1 +} + # Build Maven projects. if [ -f pom.xml ]; then echo ::group::"= Maven build =" @@ -93,7 +97,7 @@ EOL echo "Performing deployment checks" deployOK= - scmURL=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${project.scm.url}' --non-recursive validate exec:exec 2>&1) + scmURL=$(mavenEvaluate '${project.scm.url}') result=$? checkSuccess $result if [ $result -ne 0 ]; then @@ -109,7 +113,7 @@ EOL echo "No deploy -- repository fork: $BUILD_REPOSITORY != $scmURL" else # Are we building a snapshot version, or a release version? - version=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive validate exec:exec 2>&1) + version=$(mavenEvaluate '${project.version}') result=$? checkSuccess $result if [ $result -ne 0 ]; then @@ -166,7 +170,7 @@ EOL fi # HACK: Use maven-gpg-plugin 3.0.1+. Avoids "signing failed: No such file or directory" error. - maven_gpg_plugin_version=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${maven-gpg-plugin.version}' --non-recursive validate exec:exec 2>&1) + maven_gpg_plugin_version=$(mavenEvaluate '${maven-gpg-plugin.version}') case "$maven_gpg_plugin_version" in 0.*|1.*|2.*|3.0.0) echo "--> Forcing maven-gpg-plugin version from $maven_gpg_plugin_version to 3.0.1" From 7fc271dc8b21e98e78670aa69cb33412f3e5208f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 14 Apr 2022 12:28:18 -0500 Subject: [PATCH 363/429] ci-build.sh: fix build argument construction bug We need to initialize the BUILD_ARGS first, _then_ the GPG version hackery (and other things potentially later as needed) can modify it. --- ci-build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index bebf466..ea27f46 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -147,6 +147,10 @@ EOL echo "All checks passed for artifact deployment" fi + # --== Maven build arguments ==-- + + BUILD_ARGS="-B -Djdk.tls.client.protocols=TLSv1,TLSv1.1,TLSv1.2" + # --== GPG SETUP ==-- # Install GPG on macOS @@ -196,7 +200,6 @@ EOL # --== BUILD EXECUTION ==-- # Run the build. - BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' if [ "$deployOK" -a -f release.properties ]; then echo echo "== Cutting and deploying release version ==" From 4cfa93ef1159e814e2879067ede03628f234a421 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 26 Apr 2022 08:31:38 -0500 Subject: [PATCH 364/429] melting-pot: change git://github.com/ to https:// Because GitHub discontinued support for the git: protocol. --- melting-pot.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index 4c412da..7ce2ab5 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -391,7 +391,8 @@ pomValue() { # Gets the SCM URL for the given GAV. scmURL() { - pomValue "$1" project scm connection | sed -E 's/^scm:git://' + pomValue "$1" project scm connection | sed 's/^scm:git://' | + sed 's_git:\(//github.com/\)_https:\1_' } # Gets the SCM tag for the given GAV. From e18b22fbc1083cc38de8a6e2b00a51c40e365649 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 28 Apr 2022 21:24:32 -0500 Subject: [PATCH 365/429] melting-pot.sh: don't use subshells for builds We need the failCount to remember its increments. --- melting-pot.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 7ce2ab5..a170f67 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -602,14 +602,15 @@ generateScript() { fi done echo >> melt.sh - echo 'do (' >> melt.sh + echo 'do' >> melt.sh echo ' cd "$f"' >> melt.sh - echo ' sh "$dir/build.sh" > build.log 2>&1 &&' >> melt.sh + echo ' sh "$dir/build.sh" >build.log 2>&1 &&' >> melt.sh echo ' echo "[SUCCESS] $f" || {' >> melt.sh echo ' echo "[FAILURE] $f"' >> melt.sh echo ' failCount=$((failCount+1))' >> melt.sh echo ' }' >> melt.sh - echo ') done' >> melt.sh + echo ' cd - >/dev/null' >> melt.sh + echo 'done' >> melt.sh echo 'test "$failCount" -gt 255 && failCount=255' >> melt.sh echo 'exit "$failCount"' >> melt.sh } From 8b3e50a791acc23b9aa25943cb2cbeab8f6a7949 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 24 May 2022 14:12:50 -0500 Subject: [PATCH 366/429] Remove obsolete Travisification script SciJava projects migrated to GitHub Actions; use github-actionify.sh. --- travisify.sh | 280 --------------------------------------------------- 1 file changed, 280 deletions(-) delete mode 100755 travisify.sh diff --git a/travisify.sh b/travisify.sh deleted file mode 100755 index 87e4ab2..0000000 --- a/travisify.sh +++ /dev/null @@ -1,280 +0,0 @@ -#!/bin/sh - -# travisify.sh -# -# Script for enabling or updating Travis CI builds for a given repository. - -#set -e - -dir="$(dirname "$0")" - -travisDir=.travis -travisConfig=.travis.yml -travisBuildScript=$travisDir/build.sh -travisSettingsFile=$travisDir/settings.xml -travisNotifyScript=$travisDir/notify.sh -credentialsDir=$HOME/.scijava/credentials -varsFile=$credentialsDir/vars -signingKeySourceFile=$credentialsDir/scijava-ci-signing.asc -signingKeyDestFile=$travisDir/signingkey.asc -pomMinVersion='17.1.1' -tmpFile=travisify.tmp - -info() { echo "- $@"; } -warn() { echo "[WARNING] $@" 1>&2; } -err() { echo "[ERROR] $@" 1>&2; } -die() { err "$@"; exit 1; } - -check() { - for tool in $@ - do - which "$tool" >/dev/null || - die "The '$tool' utility is required but not found" - done -} - -var() { - grep "^$1=" "$varsFile" || - die "$1 not found in $varsFile" -} - -update() { - file=$1 - msg=$2 - exe=$3 - test "$msg" || msg="Travis: update $file" - if [ -e "$file" ] - then - if diff -q "$file" "$tmpFile" >/dev/null - then - info "$file is already OK" - else - info "Updating $file" - $EXEC rm -rf "$file" - $EXEC mv -f "$tmpFile" "$file" - fi - else - info "Creating $file" - $EXEC mkdir -p "$(dirname "$file")" - $EXEC mv "$tmpFile" "$file" - fi - rm -rf "$tmpFile" - $EXEC git add "$file" - if [ -n "$exe" ] - then - info "Adding execute permission to $file" - $EXEC git update-index --chmod=+x "$file" - fi - $EXEC git diff-index --quiet HEAD -- || $EXEC git commit -m "$msg" -} - -process() { - cd "$1" - - # -- Git sanity checks -- - - repoSlug=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="scm"]/*[local-name()="connection"]' pom.xml|sed 's_.*github.com[:/]\(.*\)<.*_\1_') - test "$repoSlug" && info "Repository = $repoSlug" || die 'Could not determine GitHub repository slug' - case "$repoSlug" in - *.git) - die "GitHub repository slug ('$repoSlug') ends in '.git'; please fix the POM" - ;; - esac - git fetch >/dev/null - git diff-index --quiet HEAD -- || die "Dirty working copy" - currentBranch=$(git rev-parse --abbrev-ref HEAD) - upstreamBranch=$(git rev-parse --abbrev-ref --symbolic-full-name @{u}) - remote=${upstreamBranch%/*} - defaultBranch=$(git remote show "$remote" | grep "HEAD branch" | sed 's/.*: //') - test "$currentBranch" = "$defaultBranch" || die "Non-default branch: $currentBranch" - git merge --ff --ff-only 'HEAD@{u}' >/dev/null || - die "Cannot fast forward (local diverging?)" -# test "$(git rev-parse HEAD)" = "$(git rev-parse 'HEAD@{u}')" || -# die "Mismatch with upstream branch (local ahead?)" - - # -- POM sanity checks -- - - parent=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="parent"]/*[local-name()="artifactId"]' pom.xml|sed 's/[^>]*>//'|sed 's/<.*//') - if [ -z "$SKIP_PARENT_CHECK" ] - then - test "$parent" = "pom-scijava" || - die "Not pom-scijava parent: $parent. Run with -p flag to skip this check." - fi - - domain=$(grep "travis-ci\.[a-z]*/$repoSlug" pom.xml 2>/dev/null | sed 's/.*\(travis-ci\.[a-z]*\).*/\1/') - test "$domain" && - info "Detected domain from pom.xml: $domain" || - die "No valid ciManagement section in pom.xml. Please add one, then try again." - - case $domain in - travis-ci.com) mode=pro;; - travis-ci.org) mode=org;; - *) die "Unsupported domain: $domain";; - esac - - # -- Travis sanity checks -- - - test -e "$travisDir" -a ! -d "$travisDir" && die "$travisDir is not a directory" - test -e "$travisConfig" -a ! -f "$travisConfig" && die "$travisConfig is not a regular file" - test -e "$travisConfig" && warn "$travisConfig already exists" - test -e "$travisBuildScript" && warn "$travisBuildScript already exists" - - # -- Do things -- - - # Add/update the Travis configuration file. - cat >"$tmpFile" <"$tmpFile" <]*>//'|sed 's/<.*//') - # HACK: Using a lexicographic comparison here is imperfect. - if [ "$version" \< "$pomMinVersion" ] - then - info 'Upgrading pom-scijava version' - sed "s|^ $version$| $pomMinVersion|" pom.xml >"$tmpFile" - update pom.xml "POM: update pom-scijava parent to $pomMinVersion" - else - info "Version of pom-scijava ($version) is OK" - fi - fi - - # ensure section is present - releaseProfile=$(grep '' pom.xml 2>/dev/null | sed 's/[^>]*>//' | sed 's/<.*//') - if [ "$releaseProfile" ] - then - test "$releaseProfile" = 'deploy-to-scijava' || - warn "Unknown release profile: $releaseProfile" - else - info 'Adding property' - cp pom.xml "$tmpFile" - perl -0777 -i -pe 's/(\n\t<\/properties>\n)/\n\n\t\t\n\t\tdeploy-to-scijava<\/releaseProfiles>\1/igs' "$tmpFile" - update pom.xml 'POM: deploy releases to the SciJava repository' - fi - - # update the README - if grep -q "travis-ci\.[a-zA-Z0-9/_-]*\.svg" README.md >/dev/null 2>&1 - then - info "Updating README.md Travis badge" - sed "s|travis-ci\.[a-zA-Z0-9/_-]*|$domain/$repoSlug|g" README.md >"$tmpFile" - update README.md 'Travis: fix README.md badge link' - else - info "Adding Travis badge to README.md" - echo "[![](https://$domain/$repoSlug.svg?branch=$defaultBranch)](https://$domain/$repoSlug)" >"$tmpFile" - echo >>"$tmpFile" - test -f README.md && cat README.md >>"$tmpFile" - update README.md 'Travis: add badge to README.md' - fi - - # encrypt key/value pairs in variables file - if [ -f "$varsFile" ] - then - while read p; do - # Skip comments. (Cannot use ${p:0:1} because it's bash-specific.) - case "$p" in - '#'*) continue;; - esac - info "Encrypting ${p%%=*}" - yes | $EXEC travis encrypt --$mode "$p" --add env.global --repo "$repoSlug" - test $? -eq 0 || die "Failed to encrypt variable '$p'" - done <"$varsFile" - $EXEC git commit "$travisConfig" -m "Travis: add encrypted environment variables" - else - warn "No $varsFile found. Travis will not have any environment variables set!" - fi - - # encrypt GPG keypair - if [ -f "$signingKeySourceFile" ] - then - info "Encrypting $signingKeyDestFile" - if [ -z "$EXEC" ] - then - rm -f "$signingKeyDestFile.enc" - encryptOutput=$(travis encrypt-file --$mode "$signingKeySourceFile" "$signingKeyDestFile.enc" --repo "$repoSlug") - test $? -eq 0 || die "Failed to encrypt signing key." - encryptResult=$(echo "$encryptOutput" | grep openssl) - test "$encryptResult" || die "No openssl variables emitted." - key=$(echo "$encryptResult" | cut -d' ' -f4) - iv=$(echo "$encryptResult" | cut -d' ' -f6) - sed -i.bak "s/\(sh travis-build.sh\)/\1 $key $iv/" "$travisBuildScript" - rm -f "$travisBuildScript.bak" - git add "$travisBuildScript" "$signingKeyDestFile.enc" - git commit -m "Travis: add encrypted GPG signing keypair" - fi - else - warn "No $signingKeySourceFile found. Travis will not be able to do GPG signing!" - fi -} - -test -d "$credentialsDir" || - die "This script requires configuration stored in $credentialsDir,\n" \ - "including $varsFile for needed environment variables,\n" \ - "and $signingKeySourceFile for signing of artifacts.\n" \ - "Please contact a SciJava administrator to receive a copy of this content." - -# check prerequisites -check git sed cut perl xmllint travis - -# parse arguments -EXEC=: -SKIP_PARENT_CHECK= -while test $# -gt 0 -do - case "$1" in - -f) EXEC=;; - -p) SKIP_PARENT_CHECK=true;; - --) break;; - -*) echo "Ignoring unknown option: $1" >&2; break;; - *) break;; - esac - shift -done - -test "$EXEC" && warn "Simulation only. Run with -f flag to go for real." - -# process arguments -if [ $# -gt 0 ] -then - for d in $@ - do ( - echo "[$d]" - process "$d" - ) done -else - process . -fi From fce414ceb34d9d556c79cfb95789b1845b8e9095 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 14 Jun 2022 09:23:51 -0500 Subject: [PATCH 367/429] github-actionify: use setup-java v3 No particular reason -- it's just the newest now. --- github-actionify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-actionify.sh b/github-actionify.sh index 24ea7ea..5bb0a00 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -114,7 +114,7 @@ process() { actionCheckout="uses: actions/checkout@v2" actionSetupJava="name: Set up Java - uses: actions/setup-java@v2 + uses: actions/setup-java@v3 with: java-version: '8' distribution: 'zulu' From 65f99cb01717cdffd077dd2204f405c351710c3c Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 15 Aug 2022 09:49:16 -0500 Subject: [PATCH 368/429] Change cache folder to match XDG directory spec https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html --- melting-pot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index a170f67..046ed6e 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -72,7 +72,7 @@ # -- Constants -- -meltingPotCache="$HOME/.scijava/melting-pot" +meltingPotCache="$HOME/.cache/scijava/melting-pot" # -- Functions -- From bff241cae711cdf3850facb959318a81855e689d Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 15 Aug 2022 13:23:11 -0500 Subject: [PATCH 369/429] melting-pot: improve leading documentation We give a real-world example now, rather than a contrived foo/bar one. And we link to a helpful "diamond dependency" explanation. --- melting-pot.sh | 63 ++++++++++++++++++++++---------------------------- 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 046ed6e..17b1aa1 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -5,50 +5,43 @@ # ============================================================================ # Tests all components of a project affected by changes in its dependencies. # -# First, an anecdote illustrating the problem this script solves: +# In particular, this script detects problems caused by diamond dependency +# structures: # -# Suppose you have a large application, org:app:1.0.0, with many dependencies: -# org:foo:1.2.3, org:bar:3.4.5, and many others. +# https://jlbp.dev/what-is-a-diamond-dependency-conflict # -# Now suppose you make some changes to foo, and want to know whether deploying -# them (i.e., releasing a new foo and updating app to depend on that release) -# will break the app. So you manually update your local copy of app to depend -# on org:foo:1.3.0-SNAPSHOT, and run the build (including tests, of course). +# This "melting pot" build rebuilds every dependency of a project, but at +# unified dependency versions matching those of the toplevel project. # -# The build passes, but this alone is insufficient: org:bar:3.4.5 also depends -# on org:foo:1.2.3, so you manually update bar to use org:foo:1.3.0-SNAPSHOT, -# then build bar to verify that it also is not broken by the update. +# For example, net.imagej:imagej:2.5.0 depends on many components including +# org.scijava:scijava-common:2.88.1 and net.imagej:imagej-ops:0.46.1, +# both of which depend on org.scijava:parsington. But: # -# This process quickly becomes very tedious when there are dozens of -# components of app which all depend on foo. +# - org.scijava:scijava-common:2.88.1 depends on org.scijava:parsington:3.0.0 +# - net.imagej:imagej-ops:0.46.1 depends on org.scijava:parsington:2.0.0 # -# And more importantly, testing each component individually in this manner is -# still insufficient to determine whether all of them will truly work together -# at runtime, where only a single version of each component is deployed. +# ImageJ2 can only depend on one of these versions at runtime. The newer one, +# ideally. SciJava projects use the pom-scijava parent POM as a Bill of +# Materials (BOM) to declare these winning versions, which works great... +# EXCEPT for when newer versions break backwards compatibility, as happened +# here: it's a SemVer-versioned project at different major version numbers. # -# For example: suppose org:bar:3.4.5 depends on org:lib:8.0.0, while -# org:foo:1.2.3 depends on org:lib:7.0.0. The relevant facts are: +# Enter this melting-pot script. It rebuilds each project dependency from +# source and runs the unit tests, but with all dependency versions pinned to +# match those of the toplevel project. # -# * Your new foo (org:foo:1.3.0-SNAPSHOT) builds against lib 7, and portions -# of it rely on lib-7-specific API. +# So in the example above, this script: # -# * The bar component pinned to foo 1.3.0-SNAPSHOT builds against lib 8; it -# compiles with passing tests because bar only invokes portions of the foo -# API which do not require lib-7-specific API. +# 1. gathers the dependencies of net.imagej:imagej:2.5.0; +# 2. clones each dependency from SCM at the correct release tag; +# 3. rebuilds each dependency, but with dependency versions overridden to +# those of net.imagej:imagej:2.5.0 rather than those originally used for +# that dependency at that release. # -# In this scenario, it is lib 8 that is actually deployed at runtime with the -# app, so parts of foo will be broken, even though both foo and bar build with -# passing tests individually. -# -# This "melting pot" build seeks to overcome many of these issues by unifying -# all components of the app into a single multi-module build, with all -# versions uniformly pinned to the ones that will actually be deployed at -# runtime. -# -# This goal is achieved by synthesizing a multi-module build including all -# affected components (or optionally, all components period) of the specific -# project, and then executing a Maven build with uniformly overridden versions -# of all components to the ones resolved for the project itself. +# So e.g. in the above scenario, net.imagej:imagej-ops:0.46.1 will be rebuilt +# against org.scijava:parsington:3.0.0, and we will discover whether any of +# parsington's breaking API changes from 2.0.0 to 3.0.0 actually impact the +# compilation or (tested) runtime behavior of imagej-ops. # # IMPORTANT IMPLEMENTATION DETAIL! The override works by setting a version # property for each component of the form "artifactId.version"; it is assumed From f084780e2d890b1ec58d61f4829500d40c1348a5 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 16 Aug 2022 12:10:10 -0500 Subject: [PATCH 370/429] melting-pot: improve local git repository caching This cuts down on excessive network use. The first time a particular G:A is requested at any version, it gets fully (not shallowly) cloned into the user's melting-pot cache folder. From there, the tag for the given version is cloned shallowly from the locally cached repository. In this way, complex melting pots can be repeatedly generated without hitting the network in most cases. This commit also improves debug and info messages in a few places. --- melting-pot.sh | 69 +++++++++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 17b1aa1..76beb90 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -347,6 +347,7 @@ pom() { xpath() { local xmlFile="$1" shift + local expression="$@" local xpath="/" while [ $# -gt 0 ] do @@ -354,9 +355,10 @@ xpath() { xpath="$xpath/*[local-name()='$1']" shift done - debug "xmllint --xpath \"$xpath\" \"$xmlFile\"" - xmllint --xpath "$xpath" "$xmlFile" 2> /dev/null | - sed -E 's/^[^>]*>(.*)<[^<]*$/\1/' + local value=$(xmllint --xpath "$xpath" "$xmlFile" 2> /dev/null | + sed -E 's/^[^>]*>(.*)<[^<]*$/\1/') + debug "xpath $xmlFile $expression -> $value" + echo "$value" } # For the given GAV ($1), recursively gets the value of the @@ -399,6 +401,10 @@ scmTag() { local a=$(artifactId "$1") local v=$(version "$1") local scmURL="$(scmURL "$1")" + # TODO: Avoid network use. We can scan the locally cached repo. + # But this gets complicated when the locally cached repo is + # out of date, and the needed tag is not there yet... + debug "git ls-remote --tags \"$scmURL\" | sed 's/.*refs\/tags\///'" local allTags="$(git ls-remote --tags "$scmURL" | sed 's/.*refs\/tags\///' || error "$1: Invalid scm url: $scmURL")" for tag in "$a-$v" "$v" "v$v" @@ -415,35 +421,46 @@ scmTag() { fi } -# Fetches the source code for the given GAV. Returns the directory. -retrieveSource() { - local scmURL="$(scmURL "$1")" - test "$scmURL" || die "Cannot glean SCM URL for $1" 10 - local scmBranch - test "$2" && scmBranch="$2" || scmBranch="$(scmTag "$1")" +# Ensures the source code for the given GAV exists in the melting-pot +# structure, and is up-to-date with the remote. Returns the directory. +resolveSource() { local g=$(groupId "$1") local a=$(artifactId "$1") - local v=$(version "$1") - local sourceDir="$meltingPotCache/$g/$a/$v" - if [ -d "$sourceDir" ] + local cachedRepoDir="$meltingPotCache/$g/$a" + if [ ! -d "$cachedRepoDir" ] then - debug "Using previously fetched project source at $sourceDir" - else - # Source has never been fetched before. Save it into ~/.scijava/melting-pot. - debug "git clone \"$scmURL\" --branch \"$scmBranch\" --depth 1 \"$sourceDir\"" - git clone "$scmURL" --branch "$scmBranch" --depth 1 "$sourceDir" 2> /dev/null || - die "Could not fetch project source for $1" 3 + # Source does not exist locally. Clone it into the melting pot cache. + local scmURL="$(scmURL "$1")" + test "$scmURL" || die "$1: cannot glean SCM URL" 10 + info "$1: cached repository not found; cloning from remote: $scmURL" + debug "git clone --bare \"$scmURL\" \"$cachedRepoDir\"" + git clone --bare "$scmURL" "$cachedRepoDir" 2> /dev/null || + die "$1: could not clone project source from $scmURL" 3 fi - # Copy the pristine cached source directory into the melting-pot structure. + + # Check whether the needed branch/tag exists. + local scmBranch + test "$2" && scmBranch="$2" || scmBranch="$(scmTag "$1")" + debug "git ls-remote \"file://$cachedRepoDir\" | grep -q \"\trefs/tags/$scmBranch$\"" + git ls-remote "file://$cachedRepoDir" | grep -q "\trefs/tags/$scmBranch$" || { + # Couldn't find the scmBranch as a tag in the cached repo. Either the + # tag is new, or it's not a tag ref at all (e.g. it's a branch). + # So let's update from the original remote repository. + info "$1: local tag not found for ref '$scmBranch'; updating cached repository: $cachedRepoDir" + (cd "$cachedRepoDir" && debug "git fetch --tags" && git fetch --tags) + } + + # Shallow clone the source at the given version into melting-pot structure. local destDir="$g/$a" - mkdir -p "$(dirname "$destDir")" - cp -rp "$sourceDir" "$destDir" + debug "git clone \"file://$cachedRepoDir\" --branch \"$scmBranch\" --depth 1 \"$destDir\"" + git clone "file://$cachedRepoDir" --branch "$scmBranch" --depth 1 "$destDir" 2> /dev/null || + die "$1: could not clone branch '$scmBranch' from local cache" 15 # Now verify that the cloned pom.xml contains the expected version! local expectedVersion=$(version "$1") local actualVersion=$(xpath "$destDir/pom.xml" project version) test "$expectedVersion" = "$actualVersion" || - die "POM for $1 contains wrong version: $actualVersion" 14 + warn "$1: POM contains wrong version: $actualVersion" echo "$destDir" } @@ -451,7 +468,7 @@ retrieveSource() { # Gets the list of dependencies for the project in the CWD. deps() { cd "$1" - debug "mvn dependency:list" + debug "mvn -DincludeScope=runtime -B dependency:list" local depList="$(mvn -DincludeScope=runtime -B dependency:list)" || die "Problem fetching dependencies!" 5 echo "$depList" | grep '^\[INFO\] [^ ]' | @@ -626,7 +643,7 @@ meltDown() { else # Treat specified project as a GAV. info "Fetching project source" - retrieveSource "$1" "$branch" + resolveSource "$1" "$branch" fi # Get the project dependencies. @@ -653,8 +670,8 @@ meltDown() { if [ "$(isIncluded "$gav")" ] then - info "$g:$a: fetching source for version $v" - dir="$(retrieveSource "$gav")" + info "$g:$a: resolving source for version $v" + dir="$(resolveSource "$gav")" fi done From 5d417a7f44fd1943e3eaae59b713dd313d182412 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 16 Aug 2022 13:09:17 -0500 Subject: [PATCH 371/429] melting-pot: skip previously successful builds This saves a lot of time when running the melting pot repeatedly. --- melting-pot.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/melting-pot.sh b/melting-pot.sh index 76beb90..fc21a14 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -613,6 +613,11 @@ generateScript() { done echo >> melt.sh echo 'do' >> melt.sh + echo ' # If the build passed previously, don'\''t repeat it.' >> melt.sh + echo ' test -f "$f/build.log" &&' >> melt.sh + echo ' tail -n6 "$f/build.log" | grep -qF '\''[INFO] BUILD SUCCESS'\'' &&' >> melt.sh + echo ' echo "[SUCCESS] $f (prior)" && continue' >> melt.sh + echo >> melt.sh echo ' cd "$f"' >> melt.sh echo ' sh "$dir/build.sh" >build.log 2>&1 &&' >> melt.sh echo ' echo "[SUCCESS] $f" || {' >> melt.sh From 8a5c4d460710f10a2eba9ee8c6d4a5e8c0504ccd Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 16 Aug 2022 13:30:42 -0500 Subject: [PATCH 372/429] melting-pot: fix aggregator POM schema location This POM isn't used by default anymore, but it's still good to fix this. --- melting-pot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index fc21a14..daf8c59 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -562,7 +562,7 @@ generatePOM() { echo '> pom.xml echo ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' >> pom.xml echo ' xsi:schemaLocation="http://maven.apache.org/POM/4.0.0' >> pom.xml - echo ' http://maven.apache.org/xsd/maven-4.0.0.xsd">' >> pom.xml + echo ' https://maven.apache.org/xsd/maven-4.0.0.xsd">' >> pom.xml echo ' 4.0.0' >> pom.xml echo >> pom.xml echo ' melting-pot' >> pom.xml From 33eefca60332af4f8a8e1e0b5f23d728b7f9238e Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 16 Aug 2022 15:10:23 -0500 Subject: [PATCH 373/429] release-version: fix up outdated forum links --- release-version.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/release-version.sh b/release-version.sh index 4c4e6bf..1f97d82 100755 --- a/release-version.sh +++ b/release-version.sh @@ -220,6 +220,32 @@ then -m 'And using HTTP now generates errors in Eclipse (and probably other IDEs).' fi +# Change forum references from forum.image.net to forum.image.sc. +if grep -q 'https*://forum.imagej.net' pom.xml >/dev/null 2>/dev/null +then + echo "================================================================" + echo "NOTE: Your POM still references forum.imagej.net. Fixing it now." + echo "================================================================" + sed 's;https*://forum.imagej.net;https://forum.image.sc;g' pom.xml > pom.new && + mv -f pom.new pom.xml && + git commit pom.xml \ + -m 'POM: fix forum.image.sc tag link' \ + -m 'The Discourse software updated the tags path from /tags/ to /tag/.' +fi + +# Ensure that references to forum.image.sc use /tag/, not /tags/. +if grep -q forum.image.sc/tags/ pom.xml >/dev/null 2>/dev/null +then + echo "==================================================================" + echo "NOTE: Your POM has an old-style forum.image.sc tag. Fixing it now." + echo "==================================================================" + sed 's;forum.image.sc/tags/;forum.image.sc/tag/;g' pom.xml > pom.new && + mv -f pom.new pom.xml && + git commit pom.xml \ + -m 'POM: fix forum.image.sc tag link' \ + -m 'The Discourse software updated the tags path from /tags/ to /tag/.' +fi + # Ensure license headers are up-to-date. test "$SKIP_LICENSE_UPDATE" -o -z "$licenseName" -o "$licenseName" = "N/A" || { mvn license:update-project-license license:update-file-header && From b7a05c5a5015762d6e462a6863369a1d513a0bce Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 16 Aug 2022 15:10:49 -0500 Subject: [PATCH 374/429] melting-pot: make prior success message clearer --- melting-pot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index daf8c59..c85c28f 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -616,7 +616,7 @@ generateScript() { echo ' # If the build passed previously, don'\''t repeat it.' >> melt.sh echo ' test -f "$f/build.log" &&' >> melt.sh echo ' tail -n6 "$f/build.log" | grep -qF '\''[INFO] BUILD SUCCESS'\'' &&' >> melt.sh - echo ' echo "[SUCCESS] $f (prior)" && continue' >> melt.sh + echo ' echo "[SKIPPED] $f (already succeeded)" && continue' >> melt.sh echo >> melt.sh echo ' cd "$f"' >> melt.sh echo ' sh "$dir/build.sh" >build.log 2>&1 &&' >> melt.sh From 0a90ffdce47676a5c7d8d2bb6c722f6bc9c637e1 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 16 Aug 2022 17:02:42 -0500 Subject: [PATCH 375/429] melting-pot: list dependencies before building This is useful to confirm that the build happened with the intended dependency versions. In cases where a version was hardcoded in a tag, that component version would not actually get set uniformly by the version property override mechanism. --- melting-pot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index c85c28f..652d435 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -698,7 +698,7 @@ meltDown() { # Generate build scripts. info "Generating build scripts" generatePOM - echo "mvn $args \\\\\n test \$@" > build.sh + echo "mvn $args \\\\\n dependency:list test \$@" > build.sh generateScript # Build everything. From 6ca99de5b0bb77024c9b9a6eae80473940967819 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 17 Aug 2022 06:24:13 -0500 Subject: [PATCH 376/429] melting-pot: fix local tag check on Linux Apparently, GNU grep doesn't like \t as tab? But \b works. --- melting-pot.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 652d435..827e0dc 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -441,8 +441,8 @@ resolveSource() { # Check whether the needed branch/tag exists. local scmBranch test "$2" && scmBranch="$2" || scmBranch="$(scmTag "$1")" - debug "git ls-remote \"file://$cachedRepoDir\" | grep -q \"\trefs/tags/$scmBranch$\"" - git ls-remote "file://$cachedRepoDir" | grep -q "\trefs/tags/$scmBranch$" || { + debug "git ls-remote \"file://$cachedRepoDir\" | grep -q \"\brefs/tags/$scmBranch$\"" + git ls-remote "file://$cachedRepoDir" | grep -q "\brefs/tags/$scmBranch$" || { # Couldn't find the scmBranch as a tag in the cached repo. Either the # tag is new, or it's not a tag ref at all (e.g. it's a branch). # So let's update from the original remote repository. From 6448eed775f03d2e20270403ef86359fd10829d2 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 18 Aug 2022 13:40:59 -0500 Subject: [PATCH 377/429] melting-pot: fix cached repo tag updating And avoid use of ambiguous dir variable. And fix fast-fail behavior when source resolution fails. --- melting-pot.sh | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 827e0dc..c4bc3bb 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -441,13 +441,23 @@ resolveSource() { # Check whether the needed branch/tag exists. local scmBranch test "$2" && scmBranch="$2" || scmBranch="$(scmTag "$1")" + test "$scmBranch" || die "$1: cannot glean SCM tag" 14 debug "git ls-remote \"file://$cachedRepoDir\" | grep -q \"\brefs/tags/$scmBranch$\"" git ls-remote "file://$cachedRepoDir" | grep -q "\brefs/tags/$scmBranch$" || { # Couldn't find the scmBranch as a tag in the cached repo. Either the # tag is new, or it's not a tag ref at all (e.g. it's a branch). # So let's update from the original remote repository. - info "$1: local tag not found for ref '$scmBranch'; updating cached repository: $cachedRepoDir" - (cd "$cachedRepoDir" && debug "git fetch --tags" && git fetch --tags) + info "$1: local tag not found for ref '$scmBranch'" + info "$1: updating cached repository: $cachedRepoDir" + cd "$cachedRepoDir" + debug "git fetch --tags" + if [ "$debug" ] + then + git fetch --tags + else + git fetch --tags > /dev/null + fi + cd - > /dev/null } # Shallow clone the source at the given version into melting-pot structure. @@ -467,7 +477,7 @@ resolveSource() { # Gets the list of dependencies for the project in the CWD. deps() { - cd "$1" + cd "$1" || die "No such directory: $1" 16 debug "mvn -DincludeScope=runtime -B dependency:list" local depList="$(mvn -DincludeScope=runtime -B dependency:list)" || die "Problem fetching dependencies!" 5 @@ -643,17 +653,18 @@ meltDown() { test -f "$1/pom.xml" || die "Not a Maven project: $1" 12 info "Local Maven project: $1" mkdir -p "LOCAL" - local dir="LOCAL/PROJECT" - ln -s "$1" "$dir" + local projectDir="LOCAL/PROJECT" + ln -s "$1" "$projectDir" else # Treat specified project as a GAV. info "Fetching project source" - resolveSource "$1" "$branch" + local projectDir=$(resolveSource "$1" "$branch") + test $? -eq 0 || exit $? fi # Get the project dependencies. info "Determining project dependencies" - local deps="$(deps "$dir")" + local deps="$(deps "$projectDir")" test "$deps" || die "Cannot glean project dependencies" 7 local args="-Denforcer.skip" @@ -676,7 +687,7 @@ meltDown() { if [ "$(isIncluded "$gav")" ] then info "$g:$a: resolving source for version $v" - dir="$(resolveSource "$gav")" + resolveSource "$gav" >/dev/null fi done From 98057ccebd82d821e3369c0610b036c997b562eb Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 18 Aug 2022 13:43:15 -0500 Subject: [PATCH 378/429] release-version: fix two-pass property extraction It should only use -U (which is slower) after the first attempt fails. The point of the initial attempt is to avoid using -U. --- release-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-version.sh b/release-version.sh index 1f97d82..a6207b1 100755 --- a/release-version.sh +++ b/release-version.sh @@ -113,7 +113,7 @@ Options include: # -- Extract project details -- echoArg='${project.version}:${license.licenseName}:${project.parent.groupId}:${project.parent.artifactId}:${project.parent.version}' -projectDetails=$(mvn -N -U -Dexec.executable=echo -Dexec.args="$echoArg" exec:exec -q) +projectDetails=$(mvn -N -Dexec.executable=echo -Dexec.args="$echoArg" exec:exec -q) test $? -eq 0 || projectDetails=$(mvn -U -N -Dexec.executable=echo -Dexec.args="$echoArg" exec:exec -q) test $? -eq 0 || die "Could not extract version from pom.xml. Error follows:\n$projectDetails" currentVersion=${projectDetails%%:*} From 29fd15fa723fee750c1fee1a46d5fa6089d24e8f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 18 Aug 2022 13:44:29 -0500 Subject: [PATCH 379/429] release-version: add an extra sanity check In theory, the property extraction will exit non-zero whenever there is an error, so this new check should never be triggered. But I'm paranoid, so just in case, let's check the projectDetails result string, too. --- release-version.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release-version.sh b/release-version.sh index a6207b1..71d43d8 100755 --- a/release-version.sh +++ b/release-version.sh @@ -116,6 +116,8 @@ echoArg='${project.version}:${license.licenseName}:${project.parent.groupId}:${p projectDetails=$(mvn -N -Dexec.executable=echo -Dexec.args="$echoArg" exec:exec -q) test $? -eq 0 || projectDetails=$(mvn -U -N -Dexec.executable=echo -Dexec.args="$echoArg" exec:exec -q) test $? -eq 0 || die "Could not extract version from pom.xml. Error follows:\n$projectDetails" +echo "$projectDetails" | grep -Fqv '[ERROR]' || + die "Error extracting version from pom.xml. Error follows:\n$projectDetails" currentVersion=${projectDetails%%:*} projectDetails=${projectDetails#*:} licenseName=${projectDetails%%:*} From 22675f34ab95dae404d848abcc3351d0f2946af1 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 26 Aug 2022 15:31:57 -0500 Subject: [PATCH 380/429] ci-build.sh: dump logs for failing unit tests --- ci-build.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ci-build.sh b/ci-build.sh index ea27f46..3475d07 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -219,6 +219,20 @@ EOL # --== POST-BUILD ACTIONS ==-- + # Dump logs for any failing unit tests. + if [ -d target/surefire-reports ] + then + find target/surefire-reports -name '*.txt' | while read report + do + if grep -qF 'FAILURE!' "$report" + then + echo + echo "[$report]" + cat "$report" + fi + done + fi + if [ "$deployOK" -a "$success" -eq 0 ]; then echo echo "== Invalidating SciJava Maven repository cache ==" From fd386c7f849a3255e8e2d9c56bf6768fd71797a0 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 6 Sep 2022 17:31:49 -0500 Subject: [PATCH 381/429] github-actionify: consolidate to one build.yml Still needed: change ci-build.sh not to deploy when building PRs. However, as things stand, this will only happen when the PR's branch is from the same repository, rather than from a fork. --- github-actionify.sh | 48 ++++++++++++--------------------------------- 1 file changed, 12 insertions(+), 36 deletions(-) diff --git a/github-actionify.sh b/github-actionify.sh index 5bb0a00..e16c0c4 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -9,10 +9,8 @@ dir="$(dirname "$0")" ciDir=.github -ciSlugBuildMain=workflows/build-main.yml -ciSlugBuildPR=workflows/build-pr.yml -ciConfigBuildMain=$ciDir/$ciSlugBuildMain -ciConfigBuildPR=$ciDir/$ciSlugBuildPR +ciSlugBuild=workflows/build.yml +ciConfigBuild=$ciDir/$ciSlugBuild ciSetupScript=$ciDir/setup.sh ciBuildScript=$ciDir/build.sh pomMinVersion='17.1.1' @@ -104,9 +102,8 @@ process() { # -- GitHub Action sanity checks -- test -e "$ciDir" -a ! -d "$ciDir" && die "$ciDir is not a directory" - test -e "$ciConfigBuildMain" -a ! -f "$ciConfigBuildMain" && die "$ciConfigBuildMain is not a regular file" - test -e "$ciConfigBuildPR" -a ! -f "$ciConfigBuildPR" && die "$ciConfigBuildPR is not a regular file" - test -e "$ciConfigBuildMain" && warn "$ciConfigBuildMain already exists" + test -e "$ciConfigBuild" -a ! -f "$ciConfigBuild" && die "$ciConfigBuild is not a regular file" + test -e "$ciConfigBuild" && warn "$ciConfigBuild already exists" test -e "$ciBuildScript" && warn "$ciBuildScript already exists" test -e "$ciSetupScript" && warn "$ciSetupScript already exists" @@ -137,7 +134,7 @@ process() { # -- Do things -- - # Add/update the main GitHub Actions configuration file. + # Add/update the GitHub Actions build configuration file. cat >"$tmpFile" <>"$tmpFile" - cat >>"$tmpFile" <"$tmpFile" <>"$tmpFile" <"$tmpFile" <"$tmpFile" </dev/null 2>&1 then info "Updating README.md GitHub Action badge" - sed "s;travis-ci.*;$domain/$repoSlug/actions/$ciSlugBuildMain/badge.svg)](https://$domain/$repoSlug/actions/$ciSlugBuildMain);g" README.md >"$tmpFile" + sed "s;travis-ci.*;$domain/$repoSlug/actions/$ciSlugBuild/badge.svg)](https://$domain/$repoSlug/actions/$ciSlugBuild);g" README.md >"$tmpFile" update README.md 'update README.md badge link' - elif grep -qF "$domain/$repoSlug/actions/$ciSlugBuildMain/badge.svg" README.md >/dev/null 2>&1 + elif grep -qF "$domain/$repoSlug/actions/$ciSlugBuild/badge.svg" README.md >/dev/null 2>&1 then info "GitHub Action badge already present in README.md" else info "Adding GitHub Action badge to README.md" - echo "[![](https://$domain/$repoSlug/actions/$ciSlugBuildMain/badge.svg)](https://$domain/$repoSlug/actions/$ciSlugBuildMain)" >"$tmpFile" + echo "[![Build Status](https://$domain/$repoSlug/actions/$ciSlugBuild/badge.svg)](https://$domain/$repoSlug/actions/$ciSlugBuild)" >"$tmpFile" echo >>"$tmpFile" test -f README.md && cat README.md >>"$tmpFile" update README.md 'add README.md badge link' From 5843231594c5e464bed4f27855fda3d4bdba33f1 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 12 Sep 2022 15:45:07 -0500 Subject: [PATCH 382/429] class-version.sh: work around new hexdump bug It seems that newer versions of hexdump do not allow seeking from stdin anymore? At least, that's what others online have concluded: https://www.medo64.com/2021/06/hexdumps-illegal-seek/ Fortunately, in this case, we can just decode all 8 of the first bytes, and then take the last 4. --- ci-setup-github-actions.sh | 3 ++- class-version.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ci-setup-github-actions.sh b/ci-setup-github-actions.sh index 000f6d3..649c944 100755 --- a/ci-setup-github-actions.sh +++ b/ci-setup-github-actions.sh @@ -5,7 +5,8 @@ # echo "BUILD_REPOSITORY=https://github.com/${GITHUB_REPOSITORY}" -echo "BUILD_OS=${RUNNER_OS}" +echo "BUILD_OS=${RUNNER_OS}" +echo "BUILD_DEPLOY_FORBIDDEN=..." # CTR START HERE - use GITHUB-specific env vars set upon PR to decide this echo "BUILD_REPOSITORY=https://github.com/${GITHUB_REPOSITORY}" >> $GITHUB_ENV echo "BUILD_OS=${RUNNER_OS}" >> $GITHUB_ENV diff --git a/class-version.sh b/class-version.sh index 5d128c2..8da1057 100755 --- a/class-version.sh +++ b/class-version.sh @@ -4,7 +4,7 @@ class_version() { # extract bytes 4-7 - info=$(head -c 8 | hexdump -s 4 -e '4/1 "%d\n" "\n"') + info=$(head -c 8 | hexdump -e '4/1 "%d\n" "\n"' | tail -n4) minor1="$(echo "$info" | sed -n 1p)" minor2="$(echo "$info" | sed -n 2p)" major1="$(echo "$info" | sed -n 3p)" From 71fd73aad741903cbf308bf2e8b817a61657ad67 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 14 Sep 2022 10:00:54 -0500 Subject: [PATCH 383/429] ci-build: do not deploy when building PRs If my reading of the GitHub Actions documentation is correct, then organization secrets will be available to pull request builds from a branch of the same repository (as opposed to from a project fork). In that case, we still want the build script to skip deployment, because we don't want to be deploying snapshots from from PRs, only those from the mainline integration branch. --- ci-build.sh | 9 +++++++-- ci-setup-github-actions.sh | 15 ++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index 3475d07..78d957f 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -93,7 +93,7 @@ EOL # --== DEPLOYMENT CHECKS ==-- - # Determine whether deploying will be possible. + # Determine whether deploying is both possible and warranted. echo "Performing deployment checks" deployOK= @@ -139,7 +139,12 @@ EOL fi ;; esac - deployOK=1 + if [ "$BUILD_BASE_REF" -o "$BUILD_HEAD_REF" ] + then + echo "No deploy -- proposed change: $BUILD_BASE_REF -> $BUILD_HEAD_REF" + else + deployOK=1 + fi fi fi fi diff --git a/ci-setup-github-actions.sh b/ci-setup-github-actions.sh index 649c944..a0e7a66 100755 --- a/ci-setup-github-actions.sh +++ b/ci-setup-github-actions.sh @@ -4,9 +4,14 @@ # ci-setup-github-actions.sh - Set CI-related environment variables from GitHub Actions. # -echo "BUILD_REPOSITORY=https://github.com/${GITHUB_REPOSITORY}" -echo "BUILD_OS=${RUNNER_OS}" -echo "BUILD_DEPLOY_FORBIDDEN=..." # CTR START HERE - use GITHUB-specific env vars set upon PR to decide this +echo "BUILD_REPOSITORY=https://github.com/$GITHUB_REPOSITORY" +echo "BUILD_REPOSITORY=https://github.com/$GITHUB_REPOSITORY" >> $GITHUB_ENV -echo "BUILD_REPOSITORY=https://github.com/${GITHUB_REPOSITORY}" >> $GITHUB_ENV -echo "BUILD_OS=${RUNNER_OS}" >> $GITHUB_ENV +echo "BUILD_OS=$RUNNER_OS" +echo "BUILD_OS=$RUNNER_OS" >> $GITHUB_ENV + +echo "BUILD_BASE_REF=$GITHUB_BASE_REF" +echo "BUILD_BASE_REF=$GITHUB_BASE_REF" >> $GITHUB_ENV + +echo "BUILD_HEAD_REF=$GITHUB_HEAD_REF" +echo "BUILD_HEAD_REF=$GITHUB_HEAD_REF" >> $GITHUB_ENV From 6451dc35e5384a8d58ffbed73011d58035864933 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 19 Sep 2022 13:21:14 -0500 Subject: [PATCH 384/429] melting-pot: convert from tabs to spaces I'm normally pro-tabs. But this script has a lot of script generation logic, and I want columns to line up nicely... which they won't at different tab widths. --- melting-pot.sh | 964 ++++++++++++++++++++++++------------------------- 1 file changed, 482 insertions(+), 482 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index c4bc3bb..10465c0 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -78,87 +78,87 @@ die() { error $1; exit $2; } unknownArg() { error "Unknown option: $@"; usage=1; } checkPrereqs() { - while [ $# -gt 0 ] - do - which $1 > /dev/null 2> /dev/null - test $? -ne 0 && die "Missing prerequisite: $1" 255 - shift - done + while [ $# -gt 0 ] + do + which $1 > /dev/null 2> /dev/null + test $? -ne 0 && die "Missing prerequisite: $1" 255 + shift + done } verifyPrereqs() { - checkPrereqs git mvn xmllint - git --version | grep -q 'git version 2' || - die "Please use git v2.x; older versions (<=1.7.9.5 at least) mishandle 'git clone --depth 1'" 254 + checkPrereqs git mvn xmllint + git --version | grep -q 'git version 2' || + die "Please use git v2.x; older versions (<=1.7.9.5 at least) mishandle 'git clone --depth 1'" 254 } parseArguments() { - while [ $# -gt 0 ] - do - case "$1" in - -b|--branch) - branch="$2" - shift - ;; - -c|--changes) - test "$changes" && changes="$changes,$2" || changes="$2" - shift - ;; - -i|--includes) - test "$includes" && includes="$includes,$2" || includes="$2" - shift - ;; - -e|--excludes) - test "$excludes" && excludes="$excludes,$2" || excludes="$2" - shift - ;; - -r|--remoteRepos) - test "$remoteRepos" && remoteRepos="$remoteRepos,$2" || remoteRepos="$2" - shift - ;; - -l|--localRepo) - repoBase="$2" - shift - ;; - -o|--outputDir) - outputDir="$2" - shift - ;; - -p|--prune) - prune=1 - ;; - -v|--verbose) - verbose=1 - ;; - -d|--debug) - debug=1 - ;; - -f|--force) - force=1 - ;; - -s|--skipBuild) - skipBuild=1 - ;; - -h|--help) - usage=1 - ;; - -*) - unknownArg "$1" - ;; - *) - test -z "$project" && project="$1" || - unknownArg "$1" - ;; - esac - shift - done - - test -z "$project" -a -z "$usage" && - error "No project specified!" && usage=1 - - if [ "$usage" ] - then - echo "Usage: $(basename "$0") [-b ] [-c ] \\ + while [ $# -gt 0 ] + do + case "$1" in + -b|--branch) + branch="$2" + shift + ;; + -c|--changes) + test "$changes" && changes="$changes,$2" || changes="$2" + shift + ;; + -i|--includes) + test "$includes" && includes="$includes,$2" || includes="$2" + shift + ;; + -e|--excludes) + test "$excludes" && excludes="$excludes,$2" || excludes="$2" + shift + ;; + -r|--remoteRepos) + test "$remoteRepos" && remoteRepos="$remoteRepos,$2" || remoteRepos="$2" + shift + ;; + -l|--localRepo) + repoBase="$2" + shift + ;; + -o|--outputDir) + outputDir="$2" + shift + ;; + -p|--prune) + prune=1 + ;; + -v|--verbose) + verbose=1 + ;; + -d|--debug) + debug=1 + ;; + -f|--force) + force=1 + ;; + -s|--skipBuild) + skipBuild=1 + ;; + -h|--help) + usage=1 + ;; + -*) + unknownArg "$1" + ;; + *) + test -z "$project" && project="$1" || + unknownArg "$1" + ;; + esac + shift + done + + test -z "$project" -a -z "$usage" && + error "No project specified!" && usage=1 + + if [ "$usage" ] + then + echo "Usage: $(basename "$0") [-b ] [-c ] \\ [-i ] [-e ] [-r ] [-l ] [-o ] [-pvfsh] @@ -219,425 +219,425 @@ groupIds org.scijava, net.imagej, net.imglib2 and io.scif in the pot. The -e flag is used to exclude net.imglib2:imglib2-roi from the pot. " - exit 1 - fi + exit 1 + fi - # If project is a local directory path, get its absolute path. - test -d "$project" && project=$(cd "$project" && pwd) + # If project is a local directory path, get its absolute path. + test -d "$project" && project=$(cd "$project" && pwd) - # Assign default parameter values. - test "$outputDir" || outputDir="melting-pot" - test "$repoBase" || repoBase="$HOME/.m2/repository" + # Assign default parameter values. + test "$outputDir" || outputDir="melting-pot" + test "$repoBase" || repoBase="$HOME/.m2/repository" } createDir() { - test -z "$force" -a -e "$1" && - die "Directory already exists: $1" 2 + test -z "$force" -a -e "$1" && + die "Directory already exists: $1" 2 - rm -rf "$1" - mkdir -p "$1" - cd "$1" + rm -rf "$1" + mkdir -p "$1" + cd "$1" } groupId() { - echo "${1%%:*}" + echo "${1%%:*}" } artifactId() { - local result="${1#*:}" # strip groupId - echo "${result%%:*}" + local result="${1#*:}" # strip groupId + echo "${result%%:*}" } version() { - local result="${1#*:}" # strip groupId - case "$result" in - *:*) - result="${result#*:}" # strip artifactId - case "$result" in - *:*:*:*) - # G:A:P:C:V:S - result="${result#*:}" # strip packaging - result="${result#*:}" # strip classifier - ;; - *:*:*) - # G:A:P:V:S - result="${result#*:}" # strip packaging - ;; - *) - # G:A:V or G:A:V:? - ;; - esac - echo "${result%%:*}" - ;; - esac + local result="${1#*:}" # strip groupId + case "$result" in + *:*) + result="${result#*:}" # strip artifactId + case "$result" in + *:*:*:*) + # G:A:P:C:V:S + result="${result#*:}" # strip packaging + result="${result#*:}" # strip classifier + ;; + *:*:*) + # G:A:P:V:S + result="${result#*:}" # strip packaging + ;; + *) + # G:A:V or G:A:V:? + ;; + esac + echo "${result%%:*}" + ;; + esac } classifier() { - local result="${1#*:}" # strip groupId - case "$result" in - *:*) - result="${result#*:}" # strip artifactId - case "$result" in - *:*:*:*) - # G:A:P:C:V:S - result="${result#*:}" # strip packaging - ;; - *:*:*) - # G:A:P:V:S - result="" - ;; - *:*) - # G:A:V:C - result="${result#*:}" # strip version - ;; - *) - # G:A:V - result="" - ;; - esac - echo "${result%%:*}" - ;; - esac + local result="${1#*:}" # strip groupId + case "$result" in + *:*) + result="${result#*:}" # strip artifactId + case "$result" in + *:*:*:*) + # G:A:P:C:V:S + result="${result#*:}" # strip packaging + ;; + *:*:*) + # G:A:P:V:S + result="" + ;; + *:*) + # G:A:V:C + result="${result#*:}" # strip version + ;; + *) + # G:A:V + result="" + ;; + esac + echo "${result%%:*}" + ;; + esac } # Converts the given GAV into a path in the local repository cache. repoPath() { - local gPath="$(echo "$(groupId "$1")" | tr :. /)" - local aPath="$(artifactId "$1")" - local vPath="$(version "$1")" - echo "$repoBase/$gPath/$aPath/$vPath" + local gPath="$(echo "$(groupId "$1")" | tr :. /)" + local aPath="$(artifactId "$1")" + local vPath="$(version "$1")" + echo "$repoBase/$gPath/$aPath/$vPath" } # Gets the path to the given GAV's POM file in the local repository cache. pomPath() { - local pomFile="$(artifactId "$1")-$(version "$1").pom" - echo "$(repoPath "$1")/$pomFile" + local pomFile="$(artifactId "$1")-$(version "$1").pom" + echo "$(repoPath "$1")/$pomFile" } # Fetches the POM for the given GAV into the local repository cache. downloadPOM() { - local g="$(groupId "$1")" - local a="$(artifactId "$1")" - local v="$(version "$1")" - debug "mvn dependency:get \\ - -DrepoUrl=\"$remoteRepos\" \\ - -DgroupId=\"$g\" \\ - -DartifactId=\"$a\" \\ - -Dversion=\"$v\" \\ - -Dpackaging=pom" - mvn dependency:get \ - -DrepoUrl="$remoteRepos" \ - -DgroupId="$g" \ - -DartifactId="$a" \ - -Dversion="$v" \ - -Dpackaging=pom > /dev/null || - die "Problem fetching $g:$a:$v from $remoteRepos" 4 + local g="$(groupId "$1")" + local a="$(artifactId "$1")" + local v="$(version "$1")" + debug "mvn dependency:get \\ + -DrepoUrl=\"$remoteRepos\" \\ + -DgroupId=\"$g\" \\ + -DartifactId=\"$a\" \\ + -Dversion=\"$v\" \\ + -Dpackaging=pom" + mvn dependency:get \ + -DrepoUrl="$remoteRepos" \ + -DgroupId="$g" \ + -DartifactId="$a" \ + -Dversion="$v" \ + -Dpackaging=pom > /dev/null || + die "Problem fetching $g:$a:$v from $remoteRepos" 4 } # Gets the POM path for the given GAV, ensuring it exists locally. pom() { - local pomPath="$(pomPath "$1")" - test -f "$pomPath" || downloadPOM "$1" - test -f "$pomPath" || die "Cannot access POM: $pomPath" 9 - echo "$pomPath" + local pomPath="$(pomPath "$1")" + test -f "$pomPath" || downloadPOM "$1" + test -f "$pomPath" || die "Cannot access POM: $pomPath" 9 + echo "$pomPath" } # For the given XML file on disk ($1), gets the value of the # specified XPath expression of the form "//$2/$3/$4/...". xpath() { - local xmlFile="$1" - shift - local expression="$@" - local xpath="/" - while [ $# -gt 0 ] - do - # NB: Ignore namespace issues; see: http://stackoverflow.com/a/8266075 - xpath="$xpath/*[local-name()='$1']" - shift - done - local value=$(xmllint --xpath "$xpath" "$xmlFile" 2> /dev/null | - sed -E 's/^[^>]*>(.*)<[^<]*$/\1/') - debug "xpath $xmlFile $expression -> $value" - echo "$value" + local xmlFile="$1" + shift + local expression="$@" + local xpath="/" + while [ $# -gt 0 ] + do + # NB: Ignore namespace issues; see: http://stackoverflow.com/a/8266075 + xpath="$xpath/*[local-name()='$1']" + shift + done + local value=$(xmllint --xpath "$xpath" "$xmlFile" 2> /dev/null | + sed -E 's/^[^>]*>(.*)<[^<]*$/\1/') + debug "xpath $xmlFile $expression -> $value" + echo "$value" } # For the given GAV ($1), recursively gets the value of the # specified XPath expression of the form "//$2/$3/$4/...". pomValue() { - local pomPath="$(pom "$1")" - test "$pomPath" || die "Cannot discern POM path for $1" 6 - shift - local value="$(xpath "$pomPath" $@)" - if [ "$value" ] - then - echo "$value" - else - # Path not found in POM; look in the parent POM. - local pg="$(xpath "$pomPath" project parent groupId)" - if [ "$pg" ] - then - # There is a parent POM declaration in this POM. - local pa="$(xpath "$pomPath" project parent artifactId)" - local pv="$(xpath "$pomPath" project parent version)" - pomValue "$pg:$pa:$pv" $@ - fi - fi + local pomPath="$(pom "$1")" + test "$pomPath" || die "Cannot discern POM path for $1" 6 + shift + local value="$(xpath "$pomPath" $@)" + if [ "$value" ] + then + echo "$value" + else + # Path not found in POM; look in the parent POM. + local pg="$(xpath "$pomPath" project parent groupId)" + if [ "$pg" ] + then + # There is a parent POM declaration in this POM. + local pa="$(xpath "$pomPath" project parent artifactId)" + local pv="$(xpath "$pomPath" project parent version)" + pomValue "$pg:$pa:$pv" $@ + fi + fi } # Gets the SCM URL for the given GAV. scmURL() { - pomValue "$1" project scm connection | sed 's/^scm:git://' | - sed 's_git:\(//github.com/\)_https:\1_' + pomValue "$1" project scm connection | sed 's/^scm:git://' | + sed 's_git:\(//github.com/\)_https:\1_' } # Gets the SCM tag for the given GAV. scmTag() { - local tag=$(pomValue "$1" project scm tag) - if [ -z "$tag" -o "$tag" = "HEAD" ] - then - # The value was not set properly, - # so we try to guess the tag naming scheme. :-/ - warn "$1: improper scm tag value; scanning remote tags..." - local a=$(artifactId "$1") - local v=$(version "$1") - local scmURL="$(scmURL "$1")" - # TODO: Avoid network use. We can scan the locally cached repo. - # But this gets complicated when the locally cached repo is - # out of date, and the needed tag is not there yet... - debug "git ls-remote --tags \"$scmURL\" | sed 's/.*refs\/tags\///'" - local allTags="$(git ls-remote --tags "$scmURL" | sed 's/.*refs\/tags\///' || - error "$1: Invalid scm url: $scmURL")" - for tag in "$a-$v" "$v" "v$v" - do - echo "$allTags" | grep -q "^$tag$" && { - info "$1: inferred tag: $tag" - echo "$tag" - return - } - done - error "$1: inscrutable tag scheme" - else - echo "$tag" - fi + local tag=$(pomValue "$1" project scm tag) + if [ -z "$tag" -o "$tag" = "HEAD" ] + then + # The value was not set properly, + # so we try to guess the tag naming scheme. :-/ + warn "$1: improper scm tag value; scanning remote tags..." + local a=$(artifactId "$1") + local v=$(version "$1") + local scmURL="$(scmURL "$1")" + # TODO: Avoid network use. We can scan the locally cached repo. + # But this gets complicated when the locally cached repo is + # out of date, and the needed tag is not there yet... + debug "git ls-remote --tags \"$scmURL\" | sed 's/.*refs\/tags\///'" + local allTags="$(git ls-remote --tags "$scmURL" | sed 's/.*refs\/tags\///' || + error "$1: Invalid scm url: $scmURL")" + for tag in "$a-$v" "$v" "v$v" + do + echo "$allTags" | grep -q "^$tag$" && { + info "$1: inferred tag: $tag" + echo "$tag" + return + } + done + error "$1: inscrutable tag scheme" + else + echo "$tag" + fi } # Ensures the source code for the given GAV exists in the melting-pot # structure, and is up-to-date with the remote. Returns the directory. resolveSource() { - local g=$(groupId "$1") - local a=$(artifactId "$1") - local cachedRepoDir="$meltingPotCache/$g/$a" - if [ ! -d "$cachedRepoDir" ] - then - # Source does not exist locally. Clone it into the melting pot cache. - local scmURL="$(scmURL "$1")" - test "$scmURL" || die "$1: cannot glean SCM URL" 10 - info "$1: cached repository not found; cloning from remote: $scmURL" - debug "git clone --bare \"$scmURL\" \"$cachedRepoDir\"" - git clone --bare "$scmURL" "$cachedRepoDir" 2> /dev/null || - die "$1: could not clone project source from $scmURL" 3 - fi - - # Check whether the needed branch/tag exists. - local scmBranch - test "$2" && scmBranch="$2" || scmBranch="$(scmTag "$1")" - test "$scmBranch" || die "$1: cannot glean SCM tag" 14 - debug "git ls-remote \"file://$cachedRepoDir\" | grep -q \"\brefs/tags/$scmBranch$\"" - git ls-remote "file://$cachedRepoDir" | grep -q "\brefs/tags/$scmBranch$" || { - # Couldn't find the scmBranch as a tag in the cached repo. Either the - # tag is new, or it's not a tag ref at all (e.g. it's a branch). - # So let's update from the original remote repository. - info "$1: local tag not found for ref '$scmBranch'" - info "$1: updating cached repository: $cachedRepoDir" - cd "$cachedRepoDir" - debug "git fetch --tags" - if [ "$debug" ] - then - git fetch --tags - else - git fetch --tags > /dev/null - fi - cd - > /dev/null - } - - # Shallow clone the source at the given version into melting-pot structure. - local destDir="$g/$a" - debug "git clone \"file://$cachedRepoDir\" --branch \"$scmBranch\" --depth 1 \"$destDir\"" - git clone "file://$cachedRepoDir" --branch "$scmBranch" --depth 1 "$destDir" 2> /dev/null || - die "$1: could not clone branch '$scmBranch' from local cache" 15 - - # Now verify that the cloned pom.xml contains the expected version! - local expectedVersion=$(version "$1") - local actualVersion=$(xpath "$destDir/pom.xml" project version) - test "$expectedVersion" = "$actualVersion" || - warn "$1: POM contains wrong version: $actualVersion" - - echo "$destDir" + local g=$(groupId "$1") + local a=$(artifactId "$1") + local cachedRepoDir="$meltingPotCache/$g/$a" + if [ ! -d "$cachedRepoDir" ] + then + # Source does not exist locally. Clone it into the melting pot cache. + local scmURL="$(scmURL "$1")" + test "$scmURL" || die "$1: cannot glean SCM URL" 10 + info "$1: cached repository not found; cloning from remote: $scmURL" + debug "git clone --bare \"$scmURL\" \"$cachedRepoDir\"" + git clone --bare "$scmURL" "$cachedRepoDir" 2> /dev/null || + die "$1: could not clone project source from $scmURL" 3 + fi + + # Check whether the needed branch/tag exists. + local scmBranch + test "$2" && scmBranch="$2" || scmBranch="$(scmTag "$1")" + test "$scmBranch" || die "$1: cannot glean SCM tag" 14 + debug "git ls-remote \"file://$cachedRepoDir\" | grep -q \"\brefs/tags/$scmBranch$\"" + git ls-remote "file://$cachedRepoDir" | grep -q "\brefs/tags/$scmBranch$" || { + # Couldn't find the scmBranch as a tag in the cached repo. Either the + # tag is new, or it's not a tag ref at all (e.g. it's a branch). + # So let's update from the original remote repository. + info "$1: local tag not found for ref '$scmBranch'" + info "$1: updating cached repository: $cachedRepoDir" + cd "$cachedRepoDir" + debug "git fetch --tags" + if [ "$debug" ] + then + git fetch --tags + else + git fetch --tags > /dev/null + fi + cd - > /dev/null + } + + # Shallow clone the source at the given version into melting-pot structure. + local destDir="$g/$a" + debug "git clone \"file://$cachedRepoDir\" --branch \"$scmBranch\" --depth 1 \"$destDir\"" + git clone "file://$cachedRepoDir" --branch "$scmBranch" --depth 1 "$destDir" 2> /dev/null || + die "$1: could not clone branch '$scmBranch' from local cache" 15 + + # Now verify that the cloned pom.xml contains the expected version! + local expectedVersion=$(version "$1") + local actualVersion=$(xpath "$destDir/pom.xml" project version) + test "$expectedVersion" = "$actualVersion" || + warn "$1: POM contains wrong version: $actualVersion" + + echo "$destDir" } # Gets the list of dependencies for the project in the CWD. deps() { - cd "$1" || die "No such directory: $1" 16 - debug "mvn -DincludeScope=runtime -B dependency:list" - local depList="$(mvn -DincludeScope=runtime -B dependency:list)" || - die "Problem fetching dependencies!" 5 - echo "$depList" | grep '^\[INFO\] [^ ]' | - sed 's/\[INFO\] //' | sed 's/ .*//' | sort - cd - > /dev/null + cd "$1" || die "No such directory: $1" 16 + debug "mvn -DincludeScope=runtime -B dependency:list" + local depList="$(mvn -DincludeScope=runtime -B dependency:list)" || + die "Problem fetching dependencies!" 5 + echo "$depList" | grep '^\[INFO\] [^ ]' | + sed 's/\[INFO\] //' | sed 's/ .*//' | sort + cd - > /dev/null } # Checks whether the given GA(V) matches the specified filter pattern. gaMatch() { - local ga="$1" - local filter="$2" - local g="$(groupId "$ga")" - local a="$(artifactId "$ga")" - local fg="$(groupId "$filter")" - local fa="$(artifactId "$filter")" - test "$fg" = "$g" -o "$fg" = "*" || return - test "$fa" = "$a" -o "$fa" = "*" || return - echo 1 + local ga="$1" + local filter="$2" + local g="$(groupId "$ga")" + local a="$(artifactId "$ga")" + local fg="$(groupId "$filter")" + local fa="$(artifactId "$filter")" + test "$fg" = "$g" -o "$fg" = "*" || return + test "$fa" = "$a" -o "$fa" = "*" || return + echo 1 } # Determines whether the given GA(V) version is being overridden. isChanged() { - local IFS="," + local IFS="," - local change - for change in $changes - do - test "$(gaMatch "$1" "$change")" && echo 1 && return - done + local change + for change in $changes + do + test "$(gaMatch "$1" "$change")" && echo 1 && return + done } # Determines whether the given GA(V) meets the inclusion criteria. isIncluded() { - # do not include the changed artifacts we are testing against - test "$(isChanged "$1")" && return - - local IFS="," - - # ensure GA is not excluded - local exclude - for exclude in $excludes - do - test "$(gaMatch "$1" "$exclude")" && return - done - - # ensure GA is included - test -z "$includes" && echo 1 && return - local include - for include in $includes - do - test "$(gaMatch "$1" "$include")" && echo 1 && return - done + # do not include the changed artifacts we are testing against + test "$(isChanged "$1")" && return + + local IFS="," + + # ensure GA is not excluded + local exclude + for exclude in $excludes + do + test "$(gaMatch "$1" "$exclude")" && return + done + + # ensure GA is included + test -z "$includes" && echo 1 && return + local include + for include in $includes + do + test "$(gaMatch "$1" "$include")" && echo 1 && return + done } # Deletes components which do not depend on a changed GAV. pruneReactor() { - local dir - for dir in */* - do - info "Checking relevance of component $dir" - local deps="$(deps "$dir")" - test "$deps" || die "Cannot glean dependencies for '$dir'" 8 - - # Determine whether the component depends on a changed GAV. - local keep - unset keep - local dep - for dep in $deps - do - test "$(isChanged "$dep")" && keep=1 && break - done - - # If the component is irrelevant, prune it. - if [ -z "$keep" ] - then - info "Pruning irrelevant component: $dir" - rm -rf "$dir" - fi - done + local dir + for dir in */* + do + info "Checking relevance of component $dir" + local deps="$(deps "$dir")" + test "$deps" || die "Cannot glean dependencies for '$dir'" 8 + + # Determine whether the component depends on a changed GAV. + local keep + unset keep + local dep + for dep in $deps + do + test "$(isChanged "$dep")" && keep=1 && break + done + + # If the component is irrelevant, prune it. + if [ -z "$keep" ] + then + info "Pruning irrelevant component: $dir" + rm -rf "$dir" + fi + done } # Tests if the given directory contains the appropriate source code. isProject() { - local a="$(xpath "$1/pom.xml" project artifactId)" - test "$1" = "LOCAL/PROJECT" -o "$a" = "$(basename "$1")" && echo 1 + local a="$(xpath "$1/pom.xml" project artifactId)" + test "$1" = "LOCAL/PROJECT" -o "$a" = "$(basename "$1")" && echo 1 } # Generates an aggregator POM for all modules in the current directory. generatePOM() { - echo '' > pom.xml - echo '> pom.xml - echo ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' >> pom.xml - echo ' xsi:schemaLocation="http://maven.apache.org/POM/4.0.0' >> pom.xml - echo ' https://maven.apache.org/xsd/maven-4.0.0.xsd">' >> pom.xml - echo ' 4.0.0' >> pom.xml - echo >> pom.xml - echo ' melting-pot' >> pom.xml - echo ' melting-pot' >> pom.xml - echo ' 0.0.0-SNAPSHOT' >> pom.xml - echo ' pom' >> pom.xml - echo >> pom.xml - echo ' Melting Pot' >> pom.xml - echo >> pom.xml - echo ' ' >> pom.xml - local dir - for dir in */* - do - if [ "$(isProject "$dir")" ] - then - echo " $dir" >> pom.xml - else - # Check for a child component of a multi-module project. - local childDir="$dir/$(basename "$dir")" - test "$(isProject "$childDir")" && - echo " $childDir" >> pom.xml - fi - done - echo ' ' >> pom.xml - echo '' >> pom.xml + echo '' > pom.xml + echo '> pom.xml + echo ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' >> pom.xml + echo ' xsi:schemaLocation="http://maven.apache.org/POM/4.0.0' >> pom.xml + echo ' https://maven.apache.org/xsd/maven-4.0.0.xsd">' >> pom.xml + echo ' 4.0.0' >> pom.xml + echo >> pom.xml + echo ' melting-pot' >> pom.xml + echo ' melting-pot' >> pom.xml + echo ' 0.0.0-SNAPSHOT' >> pom.xml + echo ' pom' >> pom.xml + echo >> pom.xml + echo ' Melting Pot' >> pom.xml + echo >> pom.xml + echo ' ' >> pom.xml + local dir + for dir in */* + do + if [ "$(isProject "$dir")" ] + then + echo " $dir" >> pom.xml + else + # Check for a child component of a multi-module project. + local childDir="$dir/$(basename "$dir")" + test "$(isProject "$childDir")" && + echo " $childDir" >> pom.xml + fi + done + echo ' ' >> pom.xml + echo '' >> pom.xml } # Generates melt.sh script for all modules in the current directory. generateScript() { - echo '#!/bin/sh' > melt.sh - echo 'trap "exit" INT' >> melt.sh - echo 'echo "Melting the pot..."' >> melt.sh - echo 'dir=$(pwd)' >> melt.sh - echo 'failCount=0' >> melt.sh - echo 'for f in \' >> melt.sh - local dir - for dir in */* - do - if [ "$(isProject "$dir")" ] - then - echo " $dir \\" >> melt.sh - else - # Check for a child component of a multi-module project. - local childDir="$dir/$(basename "$dir")" - test "$(isProject "$childDir")" && - echo " $childDir \\" >> melt.sh - fi - done - echo >> melt.sh - echo 'do' >> melt.sh - echo ' # If the build passed previously, don'\''t repeat it.' >> melt.sh - echo ' test -f "$f/build.log" &&' >> melt.sh - echo ' tail -n6 "$f/build.log" | grep -qF '\''[INFO] BUILD SUCCESS'\'' &&' >> melt.sh - echo ' echo "[SKIPPED] $f (already succeeded)" && continue' >> melt.sh - echo >> melt.sh - echo ' cd "$f"' >> melt.sh - echo ' sh "$dir/build.sh" >build.log 2>&1 &&' >> melt.sh - echo ' echo "[SUCCESS] $f" || {' >> melt.sh - echo ' echo "[FAILURE] $f"' >> melt.sh - echo ' failCount=$((failCount+1))' >> melt.sh - echo ' }' >> melt.sh - echo ' cd - >/dev/null' >> melt.sh - echo 'done' >> melt.sh - echo 'test "$failCount" -gt 255 && failCount=255' >> melt.sh - echo 'exit "$failCount"' >> melt.sh + echo '#!/bin/sh' > melt.sh + echo 'trap "exit" INT' >> melt.sh + echo 'echo "Melting the pot..."' >> melt.sh + echo 'dir=$(pwd)' >> melt.sh + echo 'failCount=0' >> melt.sh + echo 'for f in \' >> melt.sh + local dir + for dir in */* + do + if [ "$(isProject "$dir")" ] + then + echo " $dir \\" >> melt.sh + else + # Check for a child component of a multi-module project. + local childDir="$dir/$(basename "$dir")" + test "$(isProject "$childDir")" && + echo " $childDir \\" >> melt.sh + fi + done + echo >> melt.sh + echo 'do' >> melt.sh + echo ' # If the build passed previously, don'\''t repeat it.' >> melt.sh + echo ' test -f "$f/build.log" &&' >> melt.sh + echo ' tail -n6 "$f/build.log" | grep -qF '\''[INFO] BUILD SUCCESS'\'' &&' >> melt.sh + echo ' echo "[SKIPPED] $f (already succeeded)" && continue' >> melt.sh + echo >> melt.sh + echo ' cd "$f"' >> melt.sh + echo ' sh "$dir/build.sh" >build.log 2>&1 &&' >> melt.sh + echo ' echo "[SUCCESS] $f" || {' >> melt.sh + echo ' echo "[FAILURE] $f"' >> melt.sh + echo ' failCount=$((failCount+1))' >> melt.sh + echo ' }' >> melt.sh + echo ' cd - >/dev/null' >> melt.sh + echo 'done' >> melt.sh + echo 'test "$failCount" -gt 255 && failCount=255' >> melt.sh + echo 'exit "$failCount"' >> melt.sh } # Creates and tests an appropriate multi-module reactor for the given project. @@ -645,84 +645,84 @@ generateScript() { # the multi-module build, with each changed GAV overridding the originally # specified version for the corresponding GA. meltDown() { - # Fetch the project source code. - if [ -d "$1" ] - then - # Use local directory for the specified project. - test -d "$1" || die "No such directory: $1" 11 - test -f "$1/pom.xml" || die "Not a Maven project: $1" 12 - info "Local Maven project: $1" - mkdir -p "LOCAL" - local projectDir="LOCAL/PROJECT" - ln -s "$1" "$projectDir" - else - # Treat specified project as a GAV. - info "Fetching project source" - local projectDir=$(resolveSource "$1" "$branch") - test $? -eq 0 || exit $? - fi - - # Get the project dependencies. - info "Determining project dependencies" - local deps="$(deps "$projectDir")" - test "$deps" || die "Cannot glean project dependencies" 7 - - local args="-Denforcer.skip" - - # Process the dependencies. - info "Processing project dependencies" - local dep - for dep in $deps - do - local g="$(groupId "$dep")" - local a="$(artifactId "$dep")" - local v="$(version "$dep")" - local c="$(classifier "$dep")" - test -z "$c" || continue # skip secondary artifacts - local gav="$g:$a:$v" - - test -z "$(isChanged "$gav")" && - args="$args \\\\\n -D$g.$a.version=$v -D$a.version=$v" - - if [ "$(isIncluded "$gav")" ] - then - info "$g:$a: resolving source for version $v" - resolveSource "$gav" >/dev/null - fi - done - - # Override versions of changed GAVs. - info "Processing changed components" - local TLS=, - local gav - for gav in $changes - do - local a="$(artifactId "$gav")" - local v="$(version "$gav")" - args="$args \\\\\n -D$a.version=$v" - done - unset TLS - - # Prune the build, if applicable. - test "$prune" && pruneReactor - - # Generate build scripts. - info "Generating build scripts" - generatePOM - echo "mvn $args \\\\\n dependency:list test \$@" > build.sh - generateScript - - # Build everything. - if [ "$skipBuild" ] - then - info "Skipping the build; run melt.sh to do it." - else - info "Building the project!" - # NB: All code is fresh; no need to clean. - sh melt.sh || die "Melt failed" 13 - fi - - info "Melt complete: $1" + # Fetch the project source code. + if [ -d "$1" ] + then + # Use local directory for the specified project. + test -d "$1" || die "No such directory: $1" 11 + test -f "$1/pom.xml" || die "Not a Maven project: $1" 12 + info "Local Maven project: $1" + mkdir -p "LOCAL" + local projectDir="LOCAL/PROJECT" + ln -s "$1" "$projectDir" + else + # Treat specified project as a GAV. + info "Fetching project source" + local projectDir=$(resolveSource "$1" "$branch") + test $? -eq 0 || exit $? + fi + + # Get the project dependencies. + info "Determining project dependencies" + local deps="$(deps "$projectDir")" + test "$deps" || die "Cannot glean project dependencies" 7 + + local args="-Denforcer.skip" + + # Process the dependencies. + info "Processing project dependencies" + local dep + for dep in $deps + do + local g="$(groupId "$dep")" + local a="$(artifactId "$dep")" + local v="$(version "$dep")" + local c="$(classifier "$dep")" + test -z "$c" || continue # skip secondary artifacts + local gav="$g:$a:$v" + + test -z "$(isChanged "$gav")" && + args="$args \\\\\n -D$g.$a.version=$v -D$a.version=$v" + + if [ "$(isIncluded "$gav")" ] + then + info "$g:$a: resolving source for version $v" + resolveSource "$gav" >/dev/null + fi + done + + # Override versions of changed GAVs. + info "Processing changed components" + local TLS=, + local gav + for gav in $changes + do + local a="$(artifactId "$gav")" + local v="$(version "$gav")" + args="$args \\\\\n -D$a.version=$v" + done + unset TLS + + # Prune the build, if applicable. + test "$prune" && pruneReactor + + # Generate build scripts. + info "Generating build scripts" + generatePOM + echo "mvn $args \\\\\n dependency:list test \$@" > build.sh + generateScript + + # Build everything. + if [ "$skipBuild" ] + then + info "Skipping the build; run melt.sh to do it." + else + info "Building the project!" + # NB: All code is fresh; no need to clean. + sh melt.sh || die "Melt failed" 13 + fi + + info "Melt complete: $1" } # -- Main -- From 6952ca701375c84960ebb5a036e42f1606629165 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 19 Sep 2022 19:54:12 -0500 Subject: [PATCH 385/429] melting-pot: align append redirects to same file We could use a heredoc... but I resist! It's fine the way it is! <_< --- melting-pot.sh | 89 +++++++++++++++++++++++++------------------------- 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 10465c0..ef0ed30 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -568,76 +568,77 @@ isProject() { # Generates an aggregator POM for all modules in the current directory. generatePOM() { - echo '' > pom.xml - echo '> pom.xml - echo ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' >> pom.xml - echo ' xsi:schemaLocation="http://maven.apache.org/POM/4.0.0' >> pom.xml - echo ' https://maven.apache.org/xsd/maven-4.0.0.xsd">' >> pom.xml - echo ' 4.0.0' >> pom.xml - echo >> pom.xml - echo ' melting-pot' >> pom.xml - echo ' melting-pot' >> pom.xml - echo ' 0.0.0-SNAPSHOT' >> pom.xml - echo ' pom' >> pom.xml - echo >> pom.xml - echo ' Melting Pot' >> pom.xml - echo >> pom.xml - echo ' ' >> pom.xml + echo '' > pom.xml + echo '> pom.xml + echo ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' >> pom.xml + echo ' xsi:schemaLocation="http://maven.apache.org/POM/4.0.0' >> pom.xml + echo ' https://maven.apache.org/xsd/maven-4.0.0.xsd">' >> pom.xml + echo ' 4.0.0' >> pom.xml + echo >> pom.xml + echo ' melting-pot' >> pom.xml + echo ' melting-pot' >> pom.xml + echo ' 0.0.0-SNAPSHOT' >> pom.xml + echo ' pom' >> pom.xml + echo >> pom.xml + echo ' Melting Pot' >> pom.xml + echo >> pom.xml + echo ' ' >> pom.xml local dir for dir in */* do if [ "$(isProject "$dir")" ] then - echo " $dir" >> pom.xml + echo " $dir" >> pom.xml else # Check for a child component of a multi-module project. local childDir="$dir/$(basename "$dir")" test "$(isProject "$childDir")" && - echo " $childDir" >> pom.xml + echo " $childDir" >> pom.xml fi done - echo ' ' >> pom.xml - echo '' >> pom.xml + echo ' ' >> pom.xml + echo '' >> pom.xml } # Generates melt.sh script for all modules in the current directory. generateScript() { - echo '#!/bin/sh' > melt.sh - echo 'trap "exit" INT' >> melt.sh - echo 'echo "Melting the pot..."' >> melt.sh - echo 'dir=$(pwd)' >> melt.sh - echo 'failCount=0' >> melt.sh - echo 'for f in \' >> melt.sh + echo '#!/bin/sh' > melt.sh + echo 'trap "exit" INT' >> melt.sh + echo 'echo "Melting the pot..."' >> melt.sh + echo 'dir=$(pwd)' >> melt.sh + echo 'failCount=0' >> melt.sh + echo 'for f in \' >> melt.sh local dir for dir in */* do if [ "$(isProject "$dir")" ] then - echo " $dir \\" >> melt.sh + echo " $dir \\" >> melt.sh else # Check for a child component of a multi-module project. local childDir="$dir/$(basename "$dir")" test "$(isProject "$childDir")" && - echo " $childDir \\" >> melt.sh + echo " $childDir \\" >> melt.sh fi done - echo >> melt.sh - echo 'do' >> melt.sh - echo ' # If the build passed previously, don'\''t repeat it.' >> melt.sh - echo ' test -f "$f/build.log" &&' >> melt.sh - echo ' tail -n6 "$f/build.log" | grep -qF '\''[INFO] BUILD SUCCESS'\'' &&' >> melt.sh - echo ' echo "[SKIPPED] $f (already succeeded)" && continue' >> melt.sh - echo >> melt.sh - echo ' cd "$f"' >> melt.sh - echo ' sh "$dir/build.sh" >build.log 2>&1 &&' >> melt.sh - echo ' echo "[SUCCESS] $f" || {' >> melt.sh - echo ' echo "[FAILURE] $f"' >> melt.sh - echo ' failCount=$((failCount+1))' >> melt.sh - echo ' }' >> melt.sh - echo ' cd - >/dev/null' >> melt.sh - echo 'done' >> melt.sh - echo 'test "$failCount" -gt 255 && failCount=255' >> melt.sh - echo 'exit "$failCount"' >> melt.sh + echo >> melt.sh + echo 'do' >> melt.sh + echo ' # If the build passed previously, don'\''t repeat it.' >> melt.sh + echo ' test -f "$f/build.log" &&' >> melt.sh + echo ' tail -n6 "$f/build.log" |' >> melt.sh + echo ' grep -qF '\''[INFO] BUILD SUCCESS'\'' &&' >> melt.sh + echo ' echo "[SKIPPED] $f (already succeeded)" && continue' >> melt.sh + echo >> melt.sh + echo ' cd "$f"' >> melt.sh + echo ' sh "$dir/build.sh" >build.log 2>&1 &&' >> melt.sh + echo ' echo "[SUCCESS] $f" || {' >> melt.sh + echo ' echo "[FAILURE] $f"' >> melt.sh + echo ' failCount=$((failCount+1))' >> melt.sh + echo ' }' >> melt.sh + echo ' cd - >/dev/null' >> melt.sh + echo 'done' >> melt.sh + echo 'test "$failCount" -gt 255 && failCount=255' >> melt.sh + echo 'exit "$failCount"' >> melt.sh } # Creates and tests an appropriate multi-module reactor for the given project. From c59f4fc88147e872566379b1719b816fedc09f02 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 19 Sep 2022 15:52:51 -0500 Subject: [PATCH 386/429] melting-pot: cache previously successful dep sets --- melting-pot.sh | 121 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 105 insertions(+), 16 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index ef0ed30..9dbc914 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -600,12 +600,12 @@ generatePOM() { echo '' >> pom.xml } -# Generates melt.sh script for all modules in the current directory. -generateScript() { +# Generates melt.sh and helper scripts for all modules in the current directory. +generateScripts() { echo '#!/bin/sh' > melt.sh echo 'trap "exit" INT' >> melt.sh echo 'echo "Melting the pot..."' >> melt.sh - echo 'dir=$(pwd)' >> melt.sh + echo 'dir=$(cd "$(dirname "$0")" && pwd)' >> melt.sh echo 'failCount=0' >> melt.sh echo 'for f in \' >> melt.sh local dir @@ -623,22 +623,109 @@ generateScript() { done echo >> melt.sh echo 'do' >> melt.sh - echo ' # If the build passed previously, don'\''t repeat it.' >> melt.sh - echo ' test -f "$f/build.log" &&' >> melt.sh - echo ' tail -n6 "$f/build.log" |' >> melt.sh - echo ' grep -qF '\''[INFO] BUILD SUCCESS'\'' &&' >> melt.sh - echo ' echo "[SKIPPED] $f (already succeeded)" && continue' >> melt.sh - echo >> melt.sh + echo ' if [ "$("$dir/prior-success.sh" "$f")" ]' >> melt.sh + echo ' then' >> melt.sh + echo ' echo "[SKIPPED] $f (prior success)"' >> melt.sh + echo ' continue' >> melt.sh + echo ' fi' >> melt.sh echo ' cd "$f"' >> melt.sh - echo ' sh "$dir/build.sh" >build.log 2>&1 &&' >> melt.sh - echo ' echo "[SUCCESS] $f" || {' >> melt.sh - echo ' echo "[FAILURE] $f"' >> melt.sh - echo ' failCount=$((failCount+1))' >> melt.sh - echo ' }' >> melt.sh + echo ' "$dir/build.sh" >build.log 2>&1 && {' >> melt.sh + echo ' echo "[SUCCESS] $f"' >> melt.sh + echo ' "$dir/record-success.sh" "$f"' >> melt.sh + echo ' } || {' >> melt.sh + echo ' echo "[FAILURE] $f"' >> melt.sh + echo ' failCount=$((failCount+1))' >> melt.sh + echo ' }' >> melt.sh echo ' cd - >/dev/null' >> melt.sh echo 'done' >> melt.sh echo 'test "$failCount" -gt 255 && failCount=255' >> melt.sh echo 'exit "$failCount"' >> melt.sh + chmod +x melt.sh + +cat <<\PRIOR > prior-success.sh +#!/bin/sh +test "$1" || { echo "[ERROR] Please specify project to check."; exit 1; } + +stderr() { >&2 echo "$@"; } +debug() { test "$DEBUG" && stderr "[DEBUG] $@"; } +warn() { stderr "[WARNING] $@"; } + +dir=$(cd "$(dirname "$0")" && pwd) + +# Check build.log for BUILD SUCCESS. +buildLog="$dir/$1/build.log" +test -f "$buildLog" && tail -n6 "$buildLog" | grep -qF '[INFO] BUILD SUCCESS' && { + echo "build.log" + exit 0 +} + +# Check success.log for matching dependency configuration. +successLog="$HOME/.cache/scijava/melting-pot/$1.success.log" +test -f "$successLog" || exit 0 +success= +for deps in $(cat "$successLog") +do + debug "Checking dep config: $deps" + mismatch= + for dep in $(echo "$deps" | tr ',' '\n') + do + # g:a:p:v:s -> -Dg.a.version=v + s=${dep##*:} + case "$s" in + test) continue ;; # skip test dependencies + esac + gapv=${dep%:*} + g=${gapv%%:*} + apv=${gapv#*:} + a=${apv%%:*} + v=${apv##*:} + arg=" -D$g.$a.version=$v " + if ! grep -Fq "$arg" "$dir/build.sh" + then + # G:A property is not set to this V. + # Now check if the property is even declared. + if grep -Fq " -D$g.$a.version=" "$dir/build.sh" + then + # G:A version is mismatched. + debug "$dep [MISMATCH]" + mismatch=1 + break + else + # G:A version is not managed. + warn "Unmanaged dependency: $dep" + fi + fi + done + test "$mismatch" || { + success=$deps + break + } +done +echo "$success" +PRIOR + chmod +x prior-success.sh + +cat <<\RECORD > record-success.sh +#!/bin/sh +test "$1" || { echo "[ERROR] Please specify project to update."; exit 1; } + +dir=$(cd "$(dirname "$0")" && pwd) +buildLog="$dir/$1/build.log" +test -f "$buildLog" || exit 1 +successLog="$HOME/.cache/scijava/melting-pot/$1.success.log" +mkdir -p "$(dirname "$successLog")" + +# Record dependency configuration of successful build. +deps=$(grep "^\[INFO\] " "$buildLog" | + sed -e "s/^.\{10\}//" -e "s/ -- .*//" | + sort | tr '\n' ',') +test -f "$successLog" && grep -Fxq "$deps" "$successLog" || { + echo "$deps" > "$successLog".new + test -f "$successLog" && cat "$successLog" >> "$successLog".new + mv -f "$successLog".new "$successLog" +} +RECORD + chmod +x record-success.sh } # Creates and tests an appropriate multi-module reactor for the given project. @@ -710,8 +797,10 @@ meltDown() { # Generate build scripts. info "Generating build scripts" generatePOM - echo "mvn $args \\\\\n dependency:list test \$@" > build.sh - generateScript + echo "#!/bin/sh" > build.sh + echo "mvn $args \\\\\n dependency:list test \$@" >> build.sh + chmod +x build.sh + generateScripts # Build everything. if [ "$skipBuild" ] From 15b885760743000be4059b12dbc3936344160540 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 20 Sep 2022 10:20:02 -0500 Subject: [PATCH 387/429] melting-pot: do not generate an aggregator POM Unfortunately, due to a bug in Maven, this aggregator POM does not work in an intuitive way. In particular, executing build.sh in the top-level melting-pot directory does not pin the versions as desired within each submodule. Shockingly, they often are different versions than when you cd into a particular groupId/artifactId directory and run ../../build.sh from there. The problem is probably related to: * https://stackoverflow.com/q/45041888/1207769 * https://issues.apache.org/jira/browse/MNG-6141 * https://issues.apache.org/jira/browse/MNG-5761 Anyway, the melting pot does not use this aggregator POM anymore anyway, so let's refrain from generating it, to avoid any potential reliance on something that's broken. --- melting-pot.sh | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 9dbc914..557450f 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -566,40 +566,6 @@ isProject() { test "$1" = "LOCAL/PROJECT" -o "$a" = "$(basename "$1")" && echo 1 } -# Generates an aggregator POM for all modules in the current directory. -generatePOM() { - echo '' > pom.xml - echo '> pom.xml - echo ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' >> pom.xml - echo ' xsi:schemaLocation="http://maven.apache.org/POM/4.0.0' >> pom.xml - echo ' https://maven.apache.org/xsd/maven-4.0.0.xsd">' >> pom.xml - echo ' 4.0.0' >> pom.xml - echo >> pom.xml - echo ' melting-pot' >> pom.xml - echo ' melting-pot' >> pom.xml - echo ' 0.0.0-SNAPSHOT' >> pom.xml - echo ' pom' >> pom.xml - echo >> pom.xml - echo ' Melting Pot' >> pom.xml - echo >> pom.xml - echo ' ' >> pom.xml - local dir - for dir in */* - do - if [ "$(isProject "$dir")" ] - then - echo " $dir" >> pom.xml - else - # Check for a child component of a multi-module project. - local childDir="$dir/$(basename "$dir")" - test "$(isProject "$childDir")" && - echo " $childDir" >> pom.xml - fi - done - echo ' ' >> pom.xml - echo '' >> pom.xml -} - # Generates melt.sh and helper scripts for all modules in the current directory. generateScripts() { echo '#!/bin/sh' > melt.sh @@ -796,7 +762,6 @@ meltDown() { # Generate build scripts. info "Generating build scripts" - generatePOM echo "#!/bin/sh" > build.sh echo "mvn $args \\\\\n dependency:list test \$@" >> build.sh chmod +x build.sh From 47cac66ff99d845340c64e2437ceec2c3ceaad95 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 20 Sep 2022 15:24:24 -0500 Subject: [PATCH 388/429] melting-pot: fix macOS bug in record-success func BSD grep's bug, really. Let's work around it! --- melting-pot.sh | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 557450f..582e1e2 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -675,6 +675,22 @@ cat <<\RECORD > record-success.sh #!/bin/sh test "$1" || { echo "[ERROR] Please specify project to update."; exit 1; } +containsLine() { + pattern=$1 + file=$2 + test -f "$file" || return + # HACK: The obvious way to do this is: + # + # grep -qxF "$pattern" "$file" + # + # Unfortunately, BSD grep dies with "out of memory" when the pattern is 5111 + # characters or longer. So let's do something needlessly complex instead! + cat "$file" | while read line + do + test "$pattern" = "$line" && echo 1 && break + done +} + dir=$(cd "$(dirname "$0")" && pwd) buildLog="$dir/$1/build.log" test -f "$buildLog" || exit 1 @@ -685,11 +701,12 @@ mkdir -p "$(dirname "$successLog")" deps=$(grep "^\[INFO\] " "$buildLog" | sed -e "s/^.\{10\}//" -e "s/ -- .*//" | sort | tr '\n' ',') -test -f "$successLog" && grep -Fxq "$deps" "$successLog" || { +if [ -z "$(containsLine "$deps" "$successLog")" ] +then echo "$deps" > "$successLog".new test -f "$successLog" && cat "$successLog" >> "$successLog".new mv -f "$successLog".new "$successLog" -} +fi RECORD chmod +x record-success.sh } From 3aaa02a57c36501547711496657ba901ba1217c9 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 20 Sep 2022 15:25:55 -0500 Subject: [PATCH 389/429] melting-pot: do not clone repos with prior success This speeds up the melting pot generation even more! \^_^/ --- melting-pot.sh | 58 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 582e1e2..872ed1d 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -566,8 +566,8 @@ isProject() { test "$1" = "LOCAL/PROJECT" -o "$a" = "$(basename "$1")" && echo 1 } -# Generates melt.sh and helper scripts for all modules in the current directory. -generateScripts() { +# Generates melt.sh, covering all projects in the current directory. +generateMeltScript() { echo '#!/bin/sh' > melt.sh echo 'trap "exit" INT' >> melt.sh echo 'echo "Melting the pot..."' >> melt.sh @@ -607,8 +607,11 @@ generateScripts() { echo 'test "$failCount" -gt 255 && failCount=255' >> melt.sh echo 'exit "$failCount"' >> melt.sh chmod +x melt.sh +} -cat <<\PRIOR > prior-success.sh +# Generates helper scripts, including prior-success.sh and record-success.sh. +generateHelperScripts() { + cat <<\PRIOR > prior-success.sh #!/bin/sh test "$1" || { echo "[ERROR] Please specify project to check."; exit 1; } @@ -671,7 +674,7 @@ echo "$success" PRIOR chmod +x prior-success.sh -cat <<\RECORD > record-success.sh + cat <<\RECORD > record-success.sh #!/bin/sh test "$1" || { echo "[ERROR] Please specify project to update."; exit 1; } @@ -738,6 +741,10 @@ meltDown() { local deps="$(deps "$projectDir")" test "$deps" || die "Cannot glean project dependencies" 7 + # Generate helper scripts. We need prior-success.sh + # to decide whether to include each component. + generateHelperScripts + local args="-Denforcer.skip" # Process the dependencies. @@ -754,12 +761,6 @@ meltDown() { test -z "$(isChanged "$gav")" && args="$args \\\\\n -D$g.$a.version=$v -D$a.version=$v" - - if [ "$(isIncluded "$gav")" ] - then - info "$g:$a: resolving source for version $v" - resolveSource "$gav" >/dev/null - fi done # Override versions of changed GAVs. @@ -774,15 +775,40 @@ meltDown() { done unset TLS - # Prune the build, if applicable. - test "$prune" && pruneReactor - - # Generate build scripts. - info "Generating build scripts" + # Generate build script. + info "Generating build.sh script" echo "#!/bin/sh" > build.sh echo "mvn $args \\\\\n dependency:list test \$@" >> build.sh chmod +x build.sh - generateScripts + + # Clone source code. + info "Cloning source code" + for dep in $deps + do + local g="$(groupId "$dep")" + local a="$(artifactId "$dep")" + local v="$(version "$dep")" + local c="$(classifier "$dep")" + test -z "$c" || continue # skip secondary artifacts + local gav="$g:$a:$v" + if [ "$(isIncluded "$gav")" ] + then + if [ "$(./prior-success.sh "$g/$a")" ] + then + info "$g:$a: skipping version $v due to prior successful build" + continue + fi + info "$g:$a: resolving source for version $v" + resolveSource "$gav" >/dev/null + fi + done + + # Prune the build, if applicable. + test "$prune" && pruneReactor + + # Generate melt script. + info "Generating melt.sh script" + generateMeltScript # Build everything. if [ "$skipBuild" ] From 2f1c2e3159d47a9ae066f28292569efe1f7e3ccc Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 20 Sep 2022 16:19:45 -0500 Subject: [PATCH 390/429] melting-pot: explain why we prepend configs --- melting-pot.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/melting-pot.sh b/melting-pot.sh index 872ed1d..d17b5c0 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -706,6 +706,10 @@ deps=$(grep "^\[INFO\] " "$buildLog" | sort | tr '\n' ',') if [ -z "$(containsLine "$deps" "$successLog")" ] then + # NB: *Prepend*, rather than append, the new successful configuration. + # We do this because it is more likely this new configuration will be + # encountered again in the future, as dependency versions are highly + # likely to repeatedly increment, rather than moving backwards. echo "$deps" > "$successLog".new test -f "$successLog" && cat "$successLog" >> "$successLog".new mv -f "$successLog".new "$successLog" From 3b686deb8d4bda0eed7bc1f0fa963e344f3efcd7 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 22 Sep 2022 12:58:43 -0500 Subject: [PATCH 391/429] melting-pot: handle optional deps + empty dep sets Optional dependencies now have their scope transformed from "runtime (optional)" to "runtime-optional", to eschew spaces. Empty dep sets no longer emit an "Unmanaged dependency: none" warning. --- melting-pot.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index d17b5c0..031febc 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -642,6 +642,7 @@ do s=${dep##*:} case "$s" in test) continue ;; # skip test dependencies + none) continue ;; # empty dependency config esac gapv=${dep%:*} g=${gapv%%:*} @@ -701,8 +702,8 @@ successLog="$HOME/.cache/scijava/melting-pot/$1.success.log" mkdir -p "$(dirname "$successLog")" # Record dependency configuration of successful build. -deps=$(grep "^\[INFO\] " "$buildLog" | - sed -e "s/^.\{10\}//" -e "s/ -- .*//" | +deps=$(grep '^\[INFO\] ' "$buildLog" | + sed -e 's/^.\{10\}//' -e 's/ -- .*//' -e 's/ (\([^)]*\))/-\1/' | sort | tr '\n' ',') if [ -z "$(containsLine "$deps" "$successLog")" ] then From 398e4e04b7e967e60eec971c48656570ef5d6eb3 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 23 Sep 2022 08:24:15 -0500 Subject: [PATCH 392/429] Add project (Un)license --- UNLICENSE | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 UNLICENSE diff --git a/UNLICENSE b/UNLICENSE new file mode 100644 index 0000000..68a49da --- /dev/null +++ b/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to From 26cf1f37b2469de711754b8c1bf769f4d3147457 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 23 Sep 2022 08:24:36 -0500 Subject: [PATCH 393/429] Add mailmap, for better "git shortlog -nse" --- .mailmap | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .mailmap diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000..b48538b --- /dev/null +++ b/.mailmap @@ -0,0 +1,2 @@ +Jack Yuan <55564584+yongleyuan@users.noreply.github.com> +Tobias Pietzsch From 9246a849557407304e58f8a116ad183330145595 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 23 Sep 2022 09:35:53 -0500 Subject: [PATCH 394/429] melting-pot: give better info about mismatches --- melting-pot.sh | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 031febc..e7db32e 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -617,6 +617,7 @@ test "$1" || { echo "[ERROR] Please specify project to check."; exit 1; } stderr() { >&2 echo "$@"; } debug() { test "$DEBUG" && stderr "[DEBUG] $@"; } +info() { stderr "[INFO] $@"; } warn() { stderr "[WARNING] $@"; } dir=$(cd "$(dirname "$0")" && pwd) @@ -631,10 +632,12 @@ test -f "$buildLog" && tail -n6 "$buildLog" | grep -qF '[INFO] BUILD SUCCESS' && # Check success.log for matching dependency configuration. successLog="$HOME/.cache/scijava/melting-pot/$1.success.log" test -f "$successLog" || exit 0 +row=1 +mismatch1= success= for deps in $(cat "$successLog") do - debug "Checking dep config: $deps" + debug "$1: Checking dep config: $deps" mismatch= for dep in $(echo "$deps" | tr ',' '\n') do @@ -649,29 +652,36 @@ do apv=${gapv#*:} a=${apv%%:*} v=${apv##*:} - arg=" -D$g.$a.version=$v " - if ! grep -Fq "$arg" "$dir/build.sh" + bomV=$(grep -o " -D$g\.$a\.version=[^ ]*" "$dir/build.sh" | sed 's;.*=;;') + if [ "$bomV" != "$v" ] then # G:A property is not set to this V. # Now check if the property is even declared. - if grep -Fq " -D$g.$a.version=" "$dir/build.sh" + if [ "$bomV" ] then # G:A version is mismatched. - debug "$dep [MISMATCH]" - mismatch=1 - break + mismatch="$mismatch\n* $g:$a:$v != $bomV" else - # G:A version is not managed. - warn "Unmanaged dependency: $dep" + # G:A version is not pinned. + warn "$1: Unpinned dependency: $dep" fi fi done - test "$mismatch" || { + if [ "$mismatch" ] + then + test "$row" -eq 1 && mismatch1=$mismatch || + debug "$1: Mismatched dependencies (vs. success #$row):$mismatch" + else success=$deps break - } + fi + row=$((row+1)) done -echo "$success" +test "$success" && echo "$success" || { + test "$mismatch1" && + info "$1: Mismatched dependencies:$mismatch1" || + info "$1: No prior successes" +} PRIOR chmod +x prior-success.sh From c2b56cf623ef1ba4b00d9c06ef25713246cd0893 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 26 Sep 2022 08:36:19 -0500 Subject: [PATCH 395/429] github-actionify: delete .travis more robustly Even with -f, git complains when the file does not exist. --- abd | 0 github-actionify.sh | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 abd diff --git a/abd b/abd new file mode 100644 index 0000000..e69de29 diff --git a/github-actionify.sh b/github-actionify.sh index e16c0c4..a52079d 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -240,7 +240,8 @@ EOL fi # remove old Travis CI configuration - $EXEC git rm -rf .travis.yml .travis + test ! -e .travis.yml || $EXEC git rm -rf .travis.yml + test ! -e .travis || $EXEC git rm -rf .travis $EXEC git diff-index --quiet HEAD -- && info "No old CI configuration to remove." || $EXEC git commit -m "${msgPrefix}remove Travis CI configuration" From 00b78cbe0dd4aecf612bdb78bfa5c0598e83c8e6 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 26 Sep 2022 09:05:43 -0500 Subject: [PATCH 396/429] release-version: clean up release-only files To avoid these becoming committed to the mainline repository branch, we now do two things: 1. If they are present at the beginning, remove them. 2. After the release process completes, delete the newly generated ones. --- release-version.sh | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/release-version.sh b/release-version.sh index 71d43d8..d8f661a 100755 --- a/release-version.sh +++ b/release-version.sh @@ -208,6 +208,21 @@ test "$FETCH_HEAD" = HEAD || test "$FETCH_HEAD" = "$(git merge-base $FETCH_HEAD $HEAD)" || die "'$defaultBranch' is not up-to-date" +# Check for release-only files committed to the main branch. +for release_file in release.properties pom.xml.releaseBackup +do + if [ -e "$release_file" ] + then + echo "==========================================================================" + echo "NOTE: $release_file was committed to source control. Removing now." + echo "==========================================================================" + git rm -rf "$release_file" && + git commit "$release_file" \ + -m 'Remove $release_file' \ + -m 'It should only exist on release tags.' + fi +done + # Ensure that schema location URL uses HTTPS, not HTTP. if grep -q http://maven.apache.org/xsd/maven-4.0.0.xsd pom.xml >/dev/null 2>/dev/null then @@ -312,5 +327,8 @@ $DRY_RUN git checkout @{-1} && if test -z "$SKIP_PUSH" then $DRY_RUN git push "$REMOTE" HEAD $tag -fi || -exit +fi + +# Remove files generated by the release process. They can end up +# committed to the mainline branch and hosing up later releases. +$DRY_RUN rm -f release.properties pom.xml.releaseBackup From 3e44d6041d061a4018d223859f1819b4993eb059 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 26 Sep 2022 15:00:35 -0500 Subject: [PATCH 397/429] melting-pot: handle colored Maven output When caching successful dependency configurations, we scrape the dependency:list output from build.log. But if the version of Maven is relatively new, then colored output might be enabled, potentially hosing up our rather fragile parsing logic. This commit broadens the regexes used to work with both colored and colorless dependency:list output. --- melting-pot.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index e7db32e..251c73b 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -478,8 +478,8 @@ resolveSource() { # Gets the list of dependencies for the project in the CWD. deps() { cd "$1" || die "No such directory: $1" 16 - debug "mvn -DincludeScope=runtime -B dependency:list" - local depList="$(mvn -DincludeScope=runtime -B dependency:list)" || + debug "mvn -B -DincludeScope=runtime dependency:list" + local depList="$(mvn -B -DincludeScope=runtime dependency:list)" || die "Problem fetching dependencies!" 5 echo "$depList" | grep '^\[INFO\] [^ ]' | sed 's/\[INFO\] //' | sed 's/ .*//' | sort @@ -712,8 +712,8 @@ successLog="$HOME/.cache/scijava/melting-pot/$1.success.log" mkdir -p "$(dirname "$successLog")" # Record dependency configuration of successful build. -deps=$(grep '^\[INFO\] ' "$buildLog" | - sed -e 's/^.\{10\}//' -e 's/ -- .*//' -e 's/ (\([^)]*\))/-\1/' | +deps=$(grep '^\[[^ ]*INFO[^ ]*\] ' "$buildLog" | + sed -e 's/^[^ ]* *//' -e 's/ -- .*//' -e 's/ (\([^)]*\))/-\1/' | sort | tr '\n' ',') if [ -z "$(containsLine "$deps" "$successLog")" ] then @@ -760,6 +760,9 @@ meltDown() { # to decide whether to include each component. generateHelperScripts + # NB: We do *not* include -B here, because we want build.sh to preserve + # colored output if the version of Maven is new enough. We will take care + # elsewhere when parsing it to be flexible about whether colors are present. local args="-Denforcer.skip" # Process the dependencies. From 766d5995be7e2fdb13975f9cf54001edc44ba570 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 26 Sep 2022 15:06:23 -0500 Subject: [PATCH 398/429] melting-pot: handle colored Maven output even more The prior-success.sh script also needs to be aware of it when parsing the build.log for "BUILD SUCCESS". --- melting-pot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index 251c73b..9ffe79c 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -624,7 +624,7 @@ dir=$(cd "$(dirname "$0")" && pwd) # Check build.log for BUILD SUCCESS. buildLog="$dir/$1/build.log" -test -f "$buildLog" && tail -n6 "$buildLog" | grep -qF '[INFO] BUILD SUCCESS' && { +test -f "$buildLog" && tail -n6 "$buildLog" | grep -qF 'BUILD SUCCESS' && { echo "build.log" exit 0 } From da025c00a26e9371e465b0475361de1739fc6df2 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 17 Oct 2022 14:05:51 -0500 Subject: [PATCH 399/429] class-version.sh: fix bug in first class detection So that files such as: META-INF/services/ch.qos.logback.classic.spi.Configurator hypothetically speaking, of course ;-) are not misconstrued as classes. --- class-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/class-version.sh b/class-version.sh index 8da1057..a1ecfe7 100755 --- a/class-version.sh +++ b/class-version.sh @@ -49,7 +49,7 @@ class_version() { } first_class() { - jar tf "$1" | grep '\.class' | head -n 1 + jar tf "$1" | grep '\.class$' | head -n 1 } for file in "$@" From 09a06a29fbe213d6046497eceb7e0315eba025a3 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 21 Oct 2022 12:08:54 -0500 Subject: [PATCH 400/429] melting-pot: highlight messages with ANSI color To make the deluge of output easier for humans to parse. --- melting-pot.sh | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 9ffe79c..45ea94d 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -69,11 +69,11 @@ meltingPotCache="$HOME/.cache/scijava/melting-pot" # -- Functions -- -stderr() { >&2 echo "$@"; } +stderr() { >&2 printf "$@\n"; } debug() { test "$debug" && stderr "+ $@"; } -info() { test "$verbose" && stderr "[INFO] $@"; } -warn() { stderr "[WARNING] $@"; } -error() { stderr "[ERROR] $@"; } +info() { test "$verbose" && stderr "\e[0;37m[INFO] $@\e[0m"; } +warn() { stderr "\e[0;33m[WARNING] $@\e[0m"; } +error() { stderr "\e[0;31m[ERROR] $@\e[0m"; } die() { error $1; exit $2; } unknownArg() { error "Unknown option: $@"; usage=1; } @@ -591,15 +591,15 @@ generateMeltScript() { echo 'do' >> melt.sh echo ' if [ "$("$dir/prior-success.sh" "$f")" ]' >> melt.sh echo ' then' >> melt.sh - echo ' echo "[SKIPPED] $f (prior success)"' >> melt.sh + echo ' printf "\e[0;36m[SKIPPED] $f (prior success)\e[0m\n"' >> melt.sh echo ' continue' >> melt.sh echo ' fi' >> melt.sh echo ' cd "$f"' >> melt.sh echo ' "$dir/build.sh" >build.log 2>&1 && {' >> melt.sh - echo ' echo "[SUCCESS] $f"' >> melt.sh + echo ' printf "\e[0;32m[SUCCESS] $f\e[0m\n"' >> melt.sh echo ' "$dir/record-success.sh" "$f"' >> melt.sh echo ' } || {' >> melt.sh - echo ' echo "[FAILURE] $f"' >> melt.sh + echo ' printf "\e[0;31m[FAILURE] $f\e[0m\n"' >> melt.sh echo ' failCount=$((failCount+1))' >> melt.sh echo ' }' >> melt.sh echo ' cd - >/dev/null' >> melt.sh @@ -613,12 +613,15 @@ generateMeltScript() { generateHelperScripts() { cat <<\PRIOR > prior-success.sh #!/bin/sh -test "$1" || { echo "[ERROR] Please specify project to check."; exit 1; } +test "$1" || { + printf "\e[0;31m[ERROR] Please specify project to check.\e[0m\n" + exit 1 +} -stderr() { >&2 echo "$@"; } -debug() { test "$DEBUG" && stderr "[DEBUG] $@"; } -info() { stderr "[INFO] $@"; } -warn() { stderr "[WARNING] $@"; } +stderr() { >&2 printf "$@\n"; } +debug() { test "$DEBUG" && stderr "\e[0;37m[DEBUG] $@\e[0m"; } +info() { stderr "\e[0;37m[INFO] $@\e[0m"; } +warn() { stderr "\e[0;33m[WARNING] $@\e[0m"; } dir=$(cd "$(dirname "$0")" && pwd) @@ -687,7 +690,10 @@ PRIOR cat <<\RECORD > record-success.sh #!/bin/sh -test "$1" || { echo "[ERROR] Please specify project to update."; exit 1; } +test "$1" || { + printf "\e[0;31m[ERROR] Please specify project to update.\e[0m\n" + exit 1 +} containsLine() { pattern=$1 From dd4fd4847615af14b83d6003fdc549f647610a5f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 26 Oct 2022 12:04:44 -0500 Subject: [PATCH 401/429] melting-pot: give friendlier dep change output The fact that the dependencies do not match since the last successful build, and therefore the component build is going to be executed, is not necessarily a bad thing. The word "mismatch" conveys something wrong; let's instead just point out what's changed, more neutrally. --- melting-pot.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 45ea94d..fc96662 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -663,7 +663,7 @@ do if [ "$bomV" ] then # G:A version is mismatched. - mismatch="$mismatch\n* $g:$a:$v != $bomV" + mismatch="$mismatch\n* $g:$a:$v -> $bomV" else # G:A version is not pinned. warn "$1: Unpinned dependency: $dep" @@ -673,7 +673,7 @@ do if [ "$mismatch" ] then test "$row" -eq 1 && mismatch1=$mismatch || - debug "$1: Mismatched dependencies (vs. success #$row):$mismatch" + debug "$1: Dependency changes since success #$row:$mismatch" else success=$deps break @@ -682,7 +682,7 @@ do done test "$success" && echo "$success" || { test "$mismatch1" && - info "$1: Mismatched dependencies:$mismatch1" || + info "$1: Dependency changes since last success:$mismatch1" || info "$1: No prior successes" } PRIOR From b0c3d0f2fc65abbb96903ec0da944ba7209250a3 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 27 Oct 2022 10:18:47 -0500 Subject: [PATCH 402/429] Remove spurious empty file --- abd | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 abd diff --git a/abd b/abd deleted file mode 100644 index e69de29..0000000 From 11f4d55c5f536da7621c146b6f1a327778e63e3f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 27 Oct 2022 09:56:46 -0500 Subject: [PATCH 403/429] melting-pot: use correct remote repo parameter I cannot find any reference to a parameter called repoUrl for the maven-dependency-plugin. Not on the get-mojo doc page, and not in the maven-dependency-plugin source code. I think this feature was probably broken previously. Let's use remoteRepositories instead. --- melting-pot.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index fc96662..c80bb7c 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -320,13 +320,13 @@ downloadPOM() { local a="$(artifactId "$1")" local v="$(version "$1")" debug "mvn dependency:get \\ - -DrepoUrl=\"$remoteRepos\" \\ + -DremoteRepositories=\"$remoteRepos\" \\ -DgroupId=\"$g\" \\ -DartifactId=\"$a\" \\ -Dversion=\"$v\" \\ -Dpackaging=pom" mvn dependency:get \ - -DrepoUrl="$remoteRepos" \ + -DremoteRepositories="$remoteRepos" \ -DgroupId="$g" \ -DartifactId="$a" \ -Dversion="$v" \ From f0b105ebbaefeb2e3a2692791fc307a4b3b44bc4 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 27 Oct 2022 09:52:21 -0500 Subject: [PATCH 404/429] melting-pot: test deployed binaries, too In Java, binary API compatibility and source API compatibility are two distinct things. Recompiling the same source code against different dependencies than before might result in different bytecode, which works correctly at runtime with those new dependencies -- but when using the previously deployed bytecode against those new dependencies, errors happen because binary type signatures changed. For example, generic types might erase differently with the new dependencies, or different synthetic methods might be silently inserted. So, we need to check the new dependencies against both scenarios: 1. Do tests pass when run on the existing deployed component binary? 2. Does source compile with passing tests against the new dependencies? This commit is dedicated to Stephan Saalfeld. --- melting-pot.sh | 62 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index c80bb7c..17a368c 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -466,6 +466,9 @@ resolveSource() { git clone "file://$cachedRepoDir" --branch "$scmBranch" --depth 1 "$destDir" 2> /dev/null || die "$1: could not clone branch '$scmBranch' from local cache" 15 + # Save the GAV string to a file, for convenience. + echo "$1" > "$destDir/gav" + # Now verify that the cloned pom.xml contains the expected version! local expectedVersion=$(version "$1") local actualVersion=$(xpath "$destDir/pom.xml" project version) @@ -784,7 +787,7 @@ meltDown() { local gav="$g:$a:$v" test -z "$(isChanged "$gav")" && - args="$args \\\\\n -D$g.$a.version=$v -D$a.version=$v" + args="$args \\\\\n -D$g.$a.version=$v -D$a.version=$v" done # Override versions of changed GAVs. @@ -795,14 +798,65 @@ meltDown() { do local a="$(artifactId "$gav")" local v="$(version "$gav")" - args="$args \\\\\n -D$a.version=$v" + args="$args \\\\\n -D$a.version=$v" done unset TLS # Generate build script. info "Generating build.sh script" - echo "#!/bin/sh" > build.sh - echo "mvn $args \\\\\n dependency:list test \$@" >> build.sh + echo '#!/bin/sh' > build.sh + echo >> build.sh + echo 'mvnPin() {' >> build.sh + echo " mvn $args \\\\\n \$@" >> build.sh + echo '}' >> build.sh + echo >> build.sh + echo 'unpackArtifact() {' >> build.sh + echo ' # Download and unpack the given artifact' >> build.sh + echo ' # (G:A:V) to the specified location.' >> build.sh + echo ' gav=$1' >> build.sh + echo ' out=$2' >> build.sh + echo >> build.sh + echo ' repoPrefix=$HOME/.m2/repository # TODO: generalize this' >> build.sh + echo ' g=${gav%%:*}; r=${gav#*:}; a=${r%%:*}; v=${r##*:}' >> build.sh + echo ' gavPath="$(echo "$g" | tr "." "/")/$a/$v/$a-$v"' >> build.sh + echo ' jarPath="$repoPrefix/$gavPath.jar"' >> build.sh + echo >> build.sh + echo ' # HACK: The best goal to use would be dependency:unpack,' >> build.sh + echo ' # or failing that, dependency:copy followed by jar xf.' >> build.sh + echo ' # But those goals do not support remoteRepositories;' >> build.sh + echo ' # see https://issues.apache.org/jira/browse/MDEP-390.' >> build.sh + echo ' # So we use dependency:get and then extract it by hand.' >> build.sh + echo ' mvnPin dependency:get \' >> build.sh + echo " -DremoteRepositories=\"$remoteRepos\" \\" >> build.sh + echo ' -Dartifact="$gav" &&' >> build.sh + echo >> build.sh + echo ' test -f "$jarPath" &&' >> build.sh + echo ' mkdir -p "$out" &&' >> build.sh + echo ' cd "$out" &&' >> build.sh + echo ' jar xf "$jarPath" &&' >> build.sh + echo ' cd - >/dev/null' >> build.sh + echo '}' >> build.sh + echo >> build.sh + echo 'mvnPin dependency:list &&' >> build.sh + echo >> build.sh + echo 'if [ -f gav ]' >> build.sh + echo 'then' >> build.sh + echo ' echo' >> build.sh + echo ' echo "================================================"' >> build.sh + echo ' echo "========= Testing with deployed binary ========="' >> build.sh + echo ' echo "================================================"' >> build.sh + echo ' unpackArtifact "$(cat gav)" target/classes &&' >> build.sh + echo ' mvnPin \' >> build.sh + echo ' -Dmaven.main.skip=true \' >> build.sh + echo ' -Dmaven.resources.skip=true \' >> build.sh + echo ' test $@' >> build.sh + echo 'fi &&' >> build.sh + echo >> build.sh + echo 'echo &&' >> build.sh + echo 'echo "================================================" &&' >> build.sh + echo 'echo "============ Rebuilding from source ============" &&' >> build.sh + echo 'echo "================================================" &&' >> build.sh + echo 'mvnPin clean test $@' >> build.sh chmod +x build.sh # Clone source code. From 3f0bbdba668d2f6e821e46327b8e34150740547d Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Dec 2022 20:18:11 -0600 Subject: [PATCH 405/429] ci-build.sh: give hints when the build goes wrong In particular, people get javadoc errors a lot, which are confusing because when even a single javadoc error happens, Maven's javadoc plugin logs every javadoc *error and warning* at error level, resulting in the dreaded [ERROR] prefix all over all the javadoc output, burying the actual javadoc errors amongst the typically more plentiful warnings. You can actually find these errors though, because they are prefixed with 'error:' rather than 'warning:'. So we now grep them out and regurgitate them after declaring the build failed, as a helpful hint to why the failure occurred. And if no javadoc-specific errors are found, then we grep for common words indicating error or failure, and regurgitate those instead. And if none of those keywords are found, we declare failure at helping. This commit is dedicated to Nicolas Chiaruttini. --- ci-build.sh | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index 78d957f..78d308f 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -19,6 +19,35 @@ checkSuccess() { # Log non-zero exit code. test $1 -eq 0 || echo "==> FAILED: EXIT CODE $1" 1>&2 + if [ $1 -ne 0 -a -f "$2" ] + then + # The operation failed and a log file was provided. + # Do some heuristics, because we like being helpful! + javadocErrors=$(grep error: "$2") + generalErrors=$(grep -i '\b\(errors\?\|fail\|failures\?\)\b' "$2") + if [ "$javadocErrors" ] + then + echo + echo '/----------------------------------------------------------\' + echo '| ci-build.sh analysis: I noticed probable javadoc errors: |' + echo '\----------------------------------------------------------/' + echo "$javadocErrors" + elif [ "$generalErrors" ] + then + echo + echo '/-------------------------------------------------------\' + echo '| ci-build.sh analysis: I noticed the following errors: |' + echo '\-------------------------------------------------------/' + echo "$generalErrors" + else + echo + echo '/----------------------------------------------------------------------\' + echo '| ci-build.sh analysis: I see no problems in the operation log. Sorry! |' + echo '\----------------------------------------------------------------------/' + echo + fi + fi + # Record the first non-zero exit code. test $success -eq 0 && success=$1 } @@ -208,19 +237,19 @@ EOL if [ "$deployOK" -a -f release.properties ]; then echo echo "== Cutting and deploying release version ==" - mvn $BUILD_ARGS release:perform - checkSuccess $? + BUILD_ARGS="$BUILD_ARGS release:perform" elif [ "$deployOK" ]; then echo echo "== Building and deploying main branch SNAPSHOT ==" - mvn -Pdeploy-to-scijava $BUILD_ARGS deploy - checkSuccess $? + BUILD_ARGS="-Pdeploy-to-scijava $BUILD_ARGS deploy" else echo echo "== Building the artifact locally only ==" - mvn $BUILD_ARGS install javadoc:javadoc - checkSuccess $? + BUILD_ARGS="$BUILD_ARGS install javadoc:javadoc" fi + # Check the build result. + { mvn $BUILD_ARGS; echo $? > exit-code; } | tee mvn-log + checkSuccess "$(cat exit-code)" mvn-log # --== POST-BUILD ACTIONS ==-- From a2c8fae403556b12996f2dc92b327b2e47170cc0 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 18 Jan 2023 11:15:29 -0600 Subject: [PATCH 406/429] release-version: fix Maven project attributes It is tempting to change all http: strings to https: in this day and age. But the strings that go at the top of a Maven POM are not actually URLs, they are namespace identifiers that must be those exact strings -- including the http: prefix -- for the document to be a valid Maven POM. See also: https://imagesc.zulipchat.com/#narrow/stream/327237-SciJava/topic/release-version.20error.20with.20xsi.20namespace --- release-version.sh | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/release-version.sh b/release-version.sh index d8f661a..9e736a2 100755 --- a/release-version.sh +++ b/release-version.sh @@ -231,12 +231,41 @@ then echo "=====================================================================" sed 's;http://maven.apache.org/xsd/maven-4.0.0.xsd;https://maven.apache.org/xsd/maven-4.0.0.xsd;' pom.xml > pom.new && mv -f pom.new pom.xml && - git commit pom.xml \ - -m 'POM: use HTTPS for schema location URL' \ + git commit pom.xml -m 'POM: use HTTPS for schema location URL' \ -m 'Maven no longer supports plain HTTP for the schema location.' \ -m 'And using HTTP now generates errors in Eclipse (and probably other IDEs).' fi +# Check project xmlns, xmlns:xsi, and xsi:schemaLocation attributes. +grep -q 'xmlns="http://maven.apache.org/POM/4.0.0"' pom.xml >/dev/null 2>/dev/null && + grep -q 'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' pom.xml >/dev/null 2>/dev/null && + grep -q 'xsi:schemaLocation="http://maven.apache.org/POM/4.0.0\b"' pom.xml >/dev/null 2>/dev/null || +{ + echo "=====================================================================" + echo "NOTE: Your POM's project attributes are incorrect. Fixing it now." + echo "=====================================================================" + sed 's;xmlns="[^"]*";xmlns="http://maven.apache.org/POM/4.0.0";' pom.xml > pom.new && + mv -f pom.new pom.xml && + sed 's;xmlns:xsi="[^"]*";xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";' pom.xml > pom.new && + mv -f pom.new pom.xml && + sed 's;xsi:schemaLocation="[^"]*";xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd";' pom.xml > pom.new && + mv -f pom.new pom.xml && + git commit pom.xml -m 'POM: fix project attributes' \ + -m 'The XML schema for Maven POMs is located at:' \ + -m ' https://maven.apache.org/xsd/maven-4.0.0.xsd' \ + -m 'Its XML namespace is the string:' \ + -m ' http://maven.apache.org/POM/4.0.0' \ + -m 'So that exact string must be the value of xmlns. It must also +match the first half of xsi:schemaLocation, which maps that +namespace to an actual URL online where the schema resides. +Otherwise, the document is not a Maven POM.' \ + -m 'Similarly, the xmlns:xsi attribute of an XML document declaring a +particular schema should always use the string identifier:' \ + -m ' http://www.w3.org/2001/XMLSchema-instance' \ + -m "because that's the namespace identifier for instances of an XML schema." \ + -m "For details, see the specification at: https://www.w3.org/TR/xmlschema-1/" +} + # Change forum references from forum.image.net to forum.image.sc. if grep -q 'https*://forum.imagej.net' pom.xml >/dev/null 2>/dev/null then From dc424ec2d8719b0dd139d5fc6b5a638648434032 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 19 Jan 2023 11:51:31 -0600 Subject: [PATCH 407/429] release-version: improve project attr checks --- release-version.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release-version.sh b/release-version.sh index 9e736a2..2d4afb5 100755 --- a/release-version.sh +++ b/release-version.sh @@ -237,9 +237,9 @@ then fi # Check project xmlns, xmlns:xsi, and xsi:schemaLocation attributes. -grep -q 'xmlns="http://maven.apache.org/POM/4.0.0"' pom.xml >/dev/null 2>/dev/null && - grep -q 'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' pom.xml >/dev/null 2>/dev/null && - grep -q 'xsi:schemaLocation="http://maven.apache.org/POM/4.0.0\b"' pom.xml >/dev/null 2>/dev/null || +grep -qF 'xmlns="http://maven.apache.org/POM/4.0.0"' pom.xml >/dev/null 2>/dev/null && + grep -qF 'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' pom.xml >/dev/null 2>/dev/null && + grep -qF 'xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 ' pom.xml >/dev/null 2>/dev/null || { echo "=====================================================================" echo "NOTE: Your POM's project attributes are incorrect. Fixing it now." From 48b0aa7da358a3e674df0ae146a84656230ce5ad Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 19 Jan 2023 12:02:46 -0600 Subject: [PATCH 408/429] release-version: fix outdated comment --- release-version.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-version.sh b/release-version.sh index 2d4afb5..08f6422 100755 --- a/release-version.sh +++ b/release-version.sh @@ -340,8 +340,8 @@ test -n "$tag" && # HACK: SciJava projects use SSH (git@github.com:...) for developerConnection. # The release:perform command wants to use the developerConnection URL when # checking out the release tag. But reading from this URL requires credentials -# which we would rather Travis not need. So we replace the scm.url in the -# release.properties file to use the public (https://github.com/...) URL. +# which the CI system typically does not have. So we replace the scm.url in +# the release.properties file to use the public (https://github.com/...) URL. # This is OK, since release:perform does not need write access to the repo. $DRY_RUN sed -i.bak -e 's|^scm.url=scm\\:git\\:git@github.com\\:|scm.url=scm\\:git\\:https\\://github.com/|' release.properties && $DRY_RUN rm release.properties.bak && From 224ea6212ea05a204e161b285de1f0b55117111f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 19 Jan 2023 12:03:05 -0600 Subject: [PATCH 409/429] release-version: add debugging statements Enable via: DEBUG=1 release-version.sh --- release-version.sh | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/release-version.sh b/release-version.sh index 08f6422..197ebe3 100755 --- a/release-version.sh +++ b/release-version.sh @@ -14,7 +14,12 @@ export LC_ALL=C # -- Functions -- -die () { +debug() { + test "$DEBUG" || return + echo "[DEBUG] $@" +} + +die() { echo "$*" >&2 exit 1 } @@ -111,6 +116,7 @@ Options include: " # -- Extract project details -- +debug "Extracting project details" echoArg='${project.version}:${license.licenseName}:${project.parent.groupId}:${project.parent.artifactId}:${project.parent.version}' projectDetails=$(mvn -N -Dexec.executable=echo -Dexec.args="$echoArg" exec:exec -q) @@ -124,10 +130,12 @@ licenseName=${projectDetails%%:*} parentGAV=${projectDetails#*:} # -- Sanity checks -- +debug "Performing sanity checks" # Check that we have push rights to the repository. if [ ! "$SKIP_PUSH" ] then + debug "Checking repository push rights" push=$(git remote -v | grep origin | grep '(push)') test "$push" || die 'No push URL found for remote origin. Please use "git remote -v" to double check your remote settings.' @@ -136,6 +144,7 @@ Please use "git remote set-url origin ..." to change it.' fi # Discern the version to release. +debug "Gleaning release version" pomVersion=${currentVersion%-SNAPSHOT} test "$VERSION" -o ! -t 0 || { printf 'Version? [%s]: ' "$pomVersion" @@ -161,6 +170,7 @@ test -f "$VALID_SEMVER_BUMP" || die "Missing helper script at '$VALID_SEMVER_BUMP' Do you have a full clone of https://github.com/scijava/scijava-scripts?" test "$SKIP_VERSION_CHECK" || { + debug "Checking conformance to SemVer" sh -$- "$VALID_SEMVER_BUMP" "$pomVersion" "$VERSION" || die "If you are sure, try again with --skip-version-check flag." } @@ -171,6 +181,7 @@ test -f "$MAVEN_HELPER" || die "Missing helper script at '$MAVEN_HELPER' Do you have a full clone of https://github.com/scijava/scijava-scripts?" test "$SKIP_VERSION_CHECK" -o "$parentGAV" != "${parentGAV#$}" || { + debug "Checking pom-scijava parent version" latestParentVersion=$(sh -$- "$MAVEN_HELPER" latest-version "$parentGAV") currentParentVersion=${parentGAV##*:} test "$currentParentVersion" = "$latestParentVersion" || @@ -180,11 +191,13 @@ Or if you know better, try again with --skip-version-check flag." } # Check that the working copy is clean. +debug "Checking if working copy is clean" no_changes_pending || die 'There are uncommitted changes!' test -z "$(git ls-files -o --exclude-standard)" || die 'There are untracked files! Please stash them before releasing.' # Discern default branch. +debug "Discerning default branch" currentBranch=$(git rev-parse --abbrev-ref --symbolic-full-name HEAD) upstreamBranch=$(git rev-parse --abbrev-ref --symbolic-full-name @{u}) remote=${upstreamBranch%/*} @@ -192,6 +205,7 @@ defaultBranch=$(git remote show "$remote" | grep "HEAD branch" | sed 's/.*: //') # Check that we are on the main branch. test "$SKIP_BRANCH_CHECK" || { + debug "Checking current branch" test "$currentBranch" = "$defaultBranch" || die "Non-default branch: $currentBranch. If you are certain you want to release from this branch, try again with --skip-branch-check flag." @@ -201,6 +215,7 @@ try again with --skip-branch-check flag." REMOTE="${REMOTE:-$remote}" # Check that the main branch isn't behind the upstream branch. +debug "Ensuring local branch is up-to-date" HEAD="$(git rev-parse HEAD)" && git fetch "$REMOTE" "$defaultBranch" && FETCH_HEAD="$(git rev-parse FETCH_HEAD)" && @@ -209,6 +224,7 @@ test "$FETCH_HEAD" = "$(git merge-base $FETCH_HEAD $HEAD)" || die "'$defaultBranch' is not up-to-date" # Check for release-only files committed to the main branch. +debug "Checking for spurious release-only files" for release_file in release.properties pom.xml.releaseBackup do if [ -e "$release_file" ] @@ -224,6 +240,7 @@ do done # Ensure that schema location URL uses HTTPS, not HTTP. +debug "Checking that schema location URL uses HTTPS" if grep -q http://maven.apache.org/xsd/maven-4.0.0.xsd pom.xml >/dev/null 2>/dev/null then echo "=====================================================================" @@ -237,6 +254,7 @@ then fi # Check project xmlns, xmlns:xsi, and xsi:schemaLocation attributes. +debug "Checking correctness of POM project XML attributes" grep -qF 'xmlns="http://maven.apache.org/POM/4.0.0"' pom.xml >/dev/null 2>/dev/null && grep -qF 'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' pom.xml >/dev/null 2>/dev/null && grep -qF 'xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 ' pom.xml >/dev/null 2>/dev/null || @@ -267,6 +285,7 @@ particular schema should always use the string identifier:' \ } # Change forum references from forum.image.net to forum.image.sc. +debug "Checking correctness of forum URL references" if grep -q 'https*://forum.imagej.net' pom.xml >/dev/null 2>/dev/null then echo "================================================================" @@ -280,6 +299,7 @@ then fi # Ensure that references to forum.image.sc use /tag/, not /tags/. +debug "Checking correctness of forum tag references" if grep -q forum.image.sc/tags/ pom.xml >/dev/null 2>/dev/null then echo "==================================================================" @@ -294,6 +314,7 @@ fi # Ensure license headers are up-to-date. test "$SKIP_LICENSE_UPDATE" -o -z "$licenseName" -o "$licenseName" = "N/A" || { + debug "Ensuring that license headers are up-to-date" mvn license:update-project-license license:update-file-header && git add LICENSE.txt || die 'Failed to update copyright blurbs. You can skip the license update using the --skip-license-update flag.' @@ -308,6 +329,7 @@ Alternately, try again with the --skip-license-update flag.' } # Prepare new release without pushing (requires the release plugin >= 2.1). +debug "Preparing new release" $DRY_RUN mvn $BATCH_MODE release:prepare -DpushChanges=false -Dresume=false $TAG \ $PROFILE $DEV_VERSION -DreleaseVersion="$VERSION" \ "-Darguments=-Dgpg.skip=true ${EXTRA_ARGS# }" || @@ -317,6 +339,7 @@ Use "mvn javadoc:javadoc | grep error" to check for javadoc syntax errors.' # Squash the maven-release-plugin's two commits into one. if test -z "$DRY_RUN" then + debug "Squashing release commits" test "[maven-release-plugin] prepare for next development iteration" = \ "$(git show -s --format=%s HEAD)" || die "maven-release-plugin's commits are unexpectedly missing!" @@ -328,6 +351,7 @@ then fi && # Extract the name of the new tag. +debug "Extracting new tag name" if test -z "$DRY_RUN" then tag=$(sed -n 's/^scm.tag=//p' < release.properties) @@ -336,6 +360,7 @@ else fi && # Rewrite the tag to include release.properties. +debug "Rewriting tag to include release.properties" test -n "$tag" && # HACK: SciJava projects use SSH (git@github.com:...) for developerConnection. # The release:perform command wants to use the developerConnection URL when @@ -355,9 +380,13 @@ $DRY_RUN git checkout @{-1} && # Push the current branch and the tag. if test -z "$SKIP_PUSH" then + debug "Pushing changes" $DRY_RUN git push "$REMOTE" HEAD $tag fi # Remove files generated by the release process. They can end up # committed to the mainline branch and hosing up later releases. +debug "Cleaning up" $DRY_RUN rm -f release.properties pom.xml.releaseBackup + +debug "Release complete!" From 89c9b82b98a94c6a03102f8c45cf54f8d1873677 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 21 Feb 2023 14:17:11 -0600 Subject: [PATCH 410/429] melting-pot: warn less about unpinned deps In particular: only warn if it's an unpinned dependency from the most recent success (i.e. first row of the cached successes), rather than warning about every single unpinned dependency present in every single cached past success, no matter how old. --- melting-pot.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index 17a368c..e50f93a 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -669,7 +669,8 @@ do mismatch="$mismatch\n* $g:$a:$v -> $bomV" else # G:A version is not pinned. - warn "$1: Unpinned dependency: $dep" + test "$row" -ne 1 || + warn "$1: Unpinned dependency: $dep" fi fi done From 7abb010e575979fc331f80c85f0f13e1d879e7f2 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 27 Feb 2023 12:40:13 -0600 Subject: [PATCH 411/429] melting-pot: also log dependency:tree It's nice to see where dependencies are coming from, even if the view is not complete due to pruning. --- melting-pot.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index e50f93a..8f13b2e 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -484,7 +484,7 @@ deps() { debug "mvn -B -DincludeScope=runtime dependency:list" local depList="$(mvn -B -DincludeScope=runtime dependency:list)" || die "Problem fetching dependencies!" 5 - echo "$depList" | grep '^\[INFO\] [^ ]' | + echo "$depList" | grep '^\[INFO\] \w' | sed 's/\[INFO\] //' | sed 's/ .*//' | sort cd - > /dev/null } @@ -722,7 +722,7 @@ successLog="$HOME/.cache/scijava/melting-pot/$1.success.log" mkdir -p "$(dirname "$successLog")" # Record dependency configuration of successful build. -deps=$(grep '^\[[^ ]*INFO[^ ]*\] ' "$buildLog" | +deps=$(grep '^\[[^ ]*INFO[^ ]*\] \w' "$buildLog" | sed -e 's/^[^ ]* *//' -e 's/ -- .*//' -e 's/ (\([^)]*\))/-\1/' | sort | tr '\n' ',') if [ -z "$(containsLine "$deps" "$successLog")" ] @@ -838,7 +838,7 @@ meltDown() { echo ' cd - >/dev/null' >> build.sh echo '}' >> build.sh echo >> build.sh - echo 'mvnPin dependency:list &&' >> build.sh + echo 'mvnPin dependency:list dependency:tree &&' >> build.sh echo >> build.sh echo 'if [ -f gav ]' >> build.sh echo 'then' >> build.sh From cc4cc7df27ff4535335445723dcbe6e0e353101c Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 27 Feb 2023 12:52:54 -0600 Subject: [PATCH 412/429] melting-pot: fall back to default branch if no tag --- melting-pot.sh | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 8f13b2e..1add86b 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -415,7 +415,7 @@ scmTag() { return } done - error "$1: inscrutable tag scheme" + error "$1: inscrutable tag scheme -- using default branch" else echo "$tag" fi @@ -441,24 +441,43 @@ resolveSource() { # Check whether the needed branch/tag exists. local scmBranch test "$2" && scmBranch="$2" || scmBranch="$(scmTag "$1")" - test "$scmBranch" || die "$1: cannot glean SCM tag" 14 - debug "git ls-remote \"file://$cachedRepoDir\" | grep -q \"\brefs/tags/$scmBranch$\"" - git ls-remote "file://$cachedRepoDir" | grep -q "\brefs/tags/$scmBranch$" || { - # Couldn't find the scmBranch as a tag in the cached repo. Either the - # tag is new, or it's not a tag ref at all (e.g. it's a branch). - # So let's update from the original remote repository. - info "$1: local tag not found for ref '$scmBranch'" + + if [ "$scmBranch" ] + then + # Successfully gleaned SCM branch/tag. + debug "git ls-remote \"file://$cachedRepoDir\" | grep -q \"\brefs/tags/$scmBranch$\"" + git ls-remote "file://$cachedRepoDir" | grep -q "\brefs/tags/$scmBranch$" || { + # Couldn't find the scmBranch as a tag in the cached repo. Either the + # tag is new, or it's not a tag ref at all (e.g. it's a branch). + # So let's update from the original remote repository. + info "$1: local tag not found for ref '$scmBranch'" + info "$1: updating cached repository: $cachedRepoDir" + cd "$cachedRepoDir" + debug "git fetch --tags" + if [ "$debug" ] + then + git fetch --tags + else + git fetch --tags > /dev/null + fi + cd - > /dev/null + } + else + # No SCM branch/tag; fall back to the default branch. info "$1: updating cached repository: $cachedRepoDir" cd "$cachedRepoDir" - debug "git fetch --tags" + debug "git fetch" if [ "$debug" ] then - git fetch --tags + git fetch else - git fetch --tags > /dev/null + git fetch > /dev/null fi + head=$(cat HEAD) + scmBranch=${head##*/} + info "$1: detected default branch as $scmBranch" cd - > /dev/null - } + fi # Shallow clone the source at the given version into melting-pot structure. local destDir="$g/$a" From 7db8ec05c8211dae01ddaa063f125997d1091c41 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 27 Feb 2023 13:52:09 -0600 Subject: [PATCH 413/429] melting-pot: always build SNAPSHOT components If their version is a SNAPSHOT, or they depend on SNAPSHOTs, or both. --- melting-pot.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 1add86b..9f8cd6f 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -678,7 +678,11 @@ do a=${apv%%:*} v=${apv##*:} bomV=$(grep -o " -D$g\.$a\.version=[^ ]*" "$dir/build.sh" | sed 's;.*=;;') - if [ "$bomV" != "$v" ] + if [ "$bomV" != "${bomV#*-SNAPSHOT*}" ] + then + warn "$1: Snapshot dependency pin detected: $g:$a:$bomV -- forcing a rebuild" + exit 0 + elif [ "$bomV" != "$v" ] then # G:A property is not set to this V. # Now check if the property is even declared. @@ -744,7 +748,7 @@ mkdir -p "$(dirname "$successLog")" deps=$(grep '^\[[^ ]*INFO[^ ]*\] \w' "$buildLog" | sed -e 's/^[^ ]* *//' -e 's/ -- .*//' -e 's/ (\([^)]*\))/-\1/' | sort | tr '\n' ',') -if [ -z "$(containsLine "$deps" "$successLog")" ] +if [ "$deps" = "${deps%*-SNAPSHOT*}" -a -z "$(containsLine "$deps" "$successLog")" ] then # NB: *Prepend*, rather than append, the new successful configuration. # We do this because it is more likely this new configuration will be @@ -891,7 +895,10 @@ meltDown() { local gav="$g:$a:$v" if [ "$(isIncluded "$gav")" ] then - if [ "$(./prior-success.sh "$g/$a")" ] + if [ "$v" != "${v%-SNAPSHOT}" ] + then + info "$g:$a: forcing inclusion due to SNAPSHOT version" + elif [ "$(./prior-success.sh "$g/$a")" ] then info "$g:$a: skipping version $v due to prior successful build" continue From 22a568f2c1a068d2adcb3ec8a2afde4cff4e0697 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 2 Mar 2023 15:21:28 -0600 Subject: [PATCH 414/429] class-version.sh: make detection more robust If a class lives in META-INF/versions/ we can ignore it because it's specially targeting a different version of Java than other classes. If a module-info.class is present, we can ignore it because it must target Java 9 or newer even if the other classes target an earlier version. --- class-version.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/class-version.sh b/class-version.sh index a1ecfe7..f2dbece 100755 --- a/class-version.sh +++ b/class-version.sh @@ -49,7 +49,11 @@ class_version() { } first_class() { - jar tf "$1" | grep '\.class$' | head -n 1 + jar tf "$1" | + grep '\.class$' | + grep -v '^META-INF/' | + grep -v 'module-info\.class' | + head -n 1 } for file in "$@" From e3cd694f78102be981a02c6222f17f0928b004fc Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 11 May 2023 08:17:16 -0500 Subject: [PATCH 415/429] class-version.sh: support GAV arguments --- class-version.sh | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/class-version.sh b/class-version.sh index f2dbece..d90d34f 100755 --- a/class-version.sh +++ b/class-version.sh @@ -56,26 +56,37 @@ first_class() { head -n 1 } -for file in "$@" +for arg in "$@" do - case "$file" in + case "$arg" in + *:*:*) + ga=${arg%:*} + g=${ga%%:*} + a=${ga#*:} + v=${arg##*:} + f="$HOME/.m2/repository/$(echo "$g" | tr '.' '/')/$a/$v/$a-$v.jar" + test -f "$f" || mvn dependency:get -D"$arg" + arg="$f" + ;; + esac + case "$arg" in *.class) - version=$(cat "$file" | class_version) + version=$(cat "$arg" | class_version) ;; *.jar) - class=$(first_class "$file") + class=$(first_class "$arg") if [ -z "$class" ] then - echo "$file: No classes" + echo "$arg: No classes" continue fi - version=$(unzip -p "$file" "$class" | class_version) + version=$(unzip -p "$arg" "$class" | class_version) ;; *) - >&2 echo "Unsupported file: $file" + >&2 echo "Unsupported argument: $arg" continue esac # report the results - echo "$file: $version" + echo "$arg: $version" done From f5a378ed730276ff3222a4fd868b7ec827c8d319 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 25 Jul 2023 17:23:52 -0500 Subject: [PATCH 416/429] release-version.sh: avoid mvn ANSI colors Firstly, the mvn echo logic needs to pass -B. But even then, it seems that at least some versions of mvn (3.8.7 is an offender on my system) still prepend an [0m even in batch mode. So we also work around this bug by forcibly stripping ANSI color codes. Thanks to ChatGPT for the regex -- saved me a few minutes of time! --- release-version.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/release-version.sh b/release-version.sh index 197ebe3..b0eb696 100755 --- a/release-version.sh +++ b/release-version.sh @@ -119,11 +119,16 @@ Options include: debug "Extracting project details" echoArg='${project.version}:${license.licenseName}:${project.parent.groupId}:${project.parent.artifactId}:${project.parent.version}' -projectDetails=$(mvn -N -Dexec.executable=echo -Dexec.args="$echoArg" exec:exec -q) -test $? -eq 0 || projectDetails=$(mvn -U -N -Dexec.executable=echo -Dexec.args="$echoArg" exec:exec -q) +projectDetails=$(mvn -B -N -Dexec.executable=echo -Dexec.args="$echoArg" exec:exec -q) +test $? -eq 0 || projectDetails=$(mvn -B -U -N -Dexec.executable=echo -Dexec.args="$echoArg" exec:exec -q) test $? -eq 0 || die "Could not extract version from pom.xml. Error follows:\n$projectDetails" -echo "$projectDetails" | grep -Fqv '[ERROR]' || +printf '%s' "$projectDetails\n" | grep -Fqv '[ERROR]' || die "Error extracting version from pom.xml. Error follows:\n$projectDetails" +# HACK: Even with -B, some versions of mvn taint the output with the [0m +# color reset sequence. So we forcibly remove such sequences, just to be safe. +projectDetails=$(printf '%s' "$projectDetails" | sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[mGK]//g") +# And also remove extraneous newlines, particularly any trailing ones. +projectDetails=$(printf '%s' "$projectDetails" | tr -d '\n') currentVersion=${projectDetails%%:*} projectDetails=${projectDetails#*:} licenseName=${projectDetails%%:*} @@ -185,7 +190,7 @@ test "$SKIP_VERSION_CHECK" -o "$parentGAV" != "${parentGAV#$}" || { latestParentVersion=$(sh -$- "$MAVEN_HELPER" latest-version "$parentGAV") currentParentVersion=${parentGAV##*:} test "$currentParentVersion" = "$latestParentVersion" || - die "Newer version of parent '${parentGAV%:*}' is available: $latestParentVersion. + die "Newer version of parent '$parentGAV' is available: $latestParentVersion. I recommend you update it before releasing. Or if you know better, try again with --skip-version-check flag." } From 7b53900a9c252b76e301364d66a1d1e60994f17b Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Thu, 14 Sep 2023 11:28:52 -0500 Subject: [PATCH 417/429] ci-build.sh: allow external build args This allows us to pass e.g. -X for debugging on individual jobs --- ci-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index 78d308f..13b4f0f 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -183,7 +183,7 @@ EOL # --== Maven build arguments ==-- - BUILD_ARGS="-B -Djdk.tls.client.protocols=TLSv1,TLSv1.1,TLSv1.2" + BUILD_ARGS="$BUILD_ARGS -B -Djdk.tls.client.protocols=TLSv1,TLSv1.1,TLSv1.2" # --== GPG SETUP ==-- From 5571e4ebdda60364b5ee6578eaffd69e423a4fd5 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 17 Oct 2023 13:27:02 +0200 Subject: [PATCH 418/429] check-branch.sh: support any project with Makefile This assumes the Makefile has a 'test' target. --- check-branch.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/check-branch.sh b/check-branch.sh index c34cd48..fdbfc4c 100755 --- a/check-branch.sh +++ b/check-branch.sh @@ -24,7 +24,15 @@ do prefix="$(printf %04d $count)" filename="tmp/$prefix-$commit" start=$(date +%s) - mvn clean verify > "$filename" 2>&1 && result=SUCCESS || result=FAILURE + if [ -f Makefile ] + then + make test > "$filename" 2>&1 && result=SUCCESS || result=FAILURE + elif [ -f pom.xml ] + then + mvn clean verify > "$filename" 2>&1 && result=SUCCESS || result=FAILURE + else + result=SKIPPED + fi end=$(date +%s) time=$(expr "$end" - "$start") echo "$prefix $commit $result $time" From 27b43f6ff08b33b65e4f93153a54a31ce087dda0 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 18 Oct 2023 19:20:00 +0200 Subject: [PATCH 419/429] sj-version.sh: include pom-scijava-base versions --- sj-version.sh | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/sj-version.sh b/sj-version.sh index 834e52d..a0486a3 100755 --- a/sj-version.sh +++ b/sj-version.sh @@ -15,16 +15,29 @@ props() { if [ -e "$1" ] then # extract version properties from the given file path - versions=$(cat "$1") + pomContent=$(cat "$1") else - url="$repo/org/scijava/pom-scijava/$1/pom-scijava-$1.pom" - versions=$(curl -s "$url") # assume argument is a version number of pom-scijava + pomURL="$repo/org/scijava/pom-scijava/$1/pom-scijava-$1.pom" + pomContent=$(curl -s "$pomURL") fi - echo "$versions" | \ - grep '\.version>' | \ - sed -E -e 's/^ (.*)/\1 [DEV]/' | \ - sed -E -e 's/^ *<(.*)\.version>(.*)<\/.*\.version>/\1 = \2/' | \ + + # grep the pom-scijava-base parent version of out of the POM, + # then rip out the version properties from that one as well! + psbVersion=$(echo "$pomContent" | + grep -A1 'pom-scijava-base' | + grep '' | sed 's;.*>\([^<]*\)<.*;\1;') + psbContent= + if [ "$psbVersion" ] + then + psbURL="$repo/org/scijava/pom-scijava-base/$psbVersion/pom-scijava-base-$psbVersion.pom" + psbContent=$(curl -s "$psbURL") + fi + + { echo "$pomContent"; echo "$psbContent"; } | + grep '\.version>' | + sed -E -e 's/^ (.*)/\1 [DEV]/' | + sed -E -e 's/^ *<(.*)\.version>(.*)<\/.*\.version>/\1 = \2/' | sort } From 3cc654e8d036c1718865e7ac1c09fa5d5e1ca2b3 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 7 Nov 2023 11:15:33 -0600 Subject: [PATCH 420/429] ci-build.sh: save the resolved platform from uname Just in case we want to do anything else with it, reason about whether the platform is e.g. Linux, print it out, etc. --- ci-build.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index 13b4f0f..196fbee 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -9,10 +9,7 @@ dir="$(dirname "$0")" -MACOS= -case "$(uname -s)" in - Darwin) MACOS=1;; -esac +platform=$(uname -s) success=0 checkSuccess() { @@ -188,7 +185,7 @@ EOL # --== GPG SETUP ==-- # Install GPG on macOS - if [ "$MACOS" ]; then + if [ "$platform" = Darwin ]; then HOMEBREW_NO_AUTO_UPDATE=1 brew install gnupg2 fi From 7bd22bf4749d6f92eeefafab0ead47ea4f617785 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 7 Nov 2023 11:16:17 -0600 Subject: [PATCH 421/429] ci-build.sh: skip deploy if NO_DEPLOY flag is set So that downstream projects (e.g. scijava-common) can set this flag judiciously to control the circumstances of deployment more flexibly. --- ci-build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index 196fbee..67e9287 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -133,7 +133,9 @@ EOL else scmURL=${scmURL%.git} scmURL=${scmURL%/} - if [ ! "$SIGNING_ASC" -o ! "$GPG_KEY_NAME" -o ! "$GPG_PASSPHRASE" -o ! "$MAVEN_PASS" -o ! "$OSSRH_PASS" ]; then + if [ "$NO_DEPLOY" ]; then + echo "No deploy -- the NO_DEPLOY flag is set" + elif [ ! "$SIGNING_ASC" -o ! "$GPG_KEY_NAME" -o ! "$GPG_PASSPHRASE" -o ! "$MAVEN_PASS" -o ! "$OSSRH_PASS" ]; then echo "No deploy -- secure environment variables not available" elif [ "$BUILD_REPOSITORY" -a "$BUILD_REPOSITORY" != "$scmURL" ]; then echo "No deploy -- repository fork: $BUILD_REPOSITORY != $scmURL" From b7eb4eef9cd10911a83e460d956fb01b1e92aec6 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 4 Mar 2024 11:36:12 -0600 Subject: [PATCH 422/429] class-version.sh: update JDK names The platform was called "J2SE" from 1.2 thru 6.0, and referred to as "Java SE" starting with Java 7. But all of that is stupid and confusing marketing. Let's just call it "Java" everywhere. The "1.x" vs "x" issue still remains, but it's less terrible than before. --- class-version.sh | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/class-version.sh b/class-version.sh index d90d34f..a5b4954 100755 --- a/class-version.sh +++ b/class-version.sh @@ -17,27 +17,24 @@ class_version() { # derive Java version case $major in 45) - version="JDK 1.1" + version="Java 1.0/1.1" ;; 46) - version="JDK 1.2" + version="Java 1.2" ;; 47) - version="JDK 1.3" + version="Java 1.3" ;; 48) - version="JDK 1.4" + version="Java 1.4" ;; 49) - version="J2SE 5.0" - ;; - 50) - version="J2SE 6.0" + version="Java 5" ;; *) - if [ "$major" -gt 50 ] + if [ "$major" -gt 49 ] then - version="J2SE $(expr $major - 44)" + version="Java $(expr $major - 44)" else version="Unknown" fi From 49845be5394918525f87b19c0e3c1b266b2c53d8 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Mon, 8 Jul 2024 14:20:30 -0500 Subject: [PATCH 423/429] Add support for custom OSSRH_USER value This is now important because apparently it is no longer allowed to deploy to OSS Sonatype (at least not s01.oss.sonatype.org) using the plain username and password configuration -- only with the user token. And the token has different alphabet gobbledegook for the username. --- ci-build.sh | 5 ++++- github-actionify.sh | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index 67e9287..8dc4768 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -80,6 +80,9 @@ if [ -f pom.xml ]; then elif [ -f "$customSettings" ]; then cp "$customSettings" "$settingsFile" else + if [ -z "$OSSRH_USER" ]; then + OSSRH_USER=scijava-ci + fi cat >"$settingsFile" < @@ -95,7 +98,7 @@ if [ -f pom.xml ]; then sonatype-nexus-releases - scijava-ci + $OSSRH_USER $(escapeXML "$OSSRH_PASS") diff --git a/github-actionify.sh b/github-actionify.sh index a52079d..6707c5f 100755 --- a/github-actionify.sh +++ b/github-actionify.sh @@ -129,6 +129,7 @@ process() { GPG_PASSPHRASE: \${{ secrets.GPG_PASSPHRASE }} MAVEN_USER: \${{ secrets.MAVEN_USER }} MAVEN_PASS: \${{ secrets.MAVEN_PASS }} + OSSRH_USER: \${{ secrets.OSSRH_USER }} OSSRH_PASS: \${{ secrets.OSSRH_PASS }} SIGNING_ASC: \${{ secrets.SIGNING_ASC }}" From 756eb8e79cf8f5f5aa1d429db39419e209f3ac8b Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Aug 2024 05:57:13 -0500 Subject: [PATCH 424/429] ci-build.sh: fix 'proposed change' flow direction --- ci-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index 8dc4768..e1093eb 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -172,7 +172,7 @@ EOL esac if [ "$BUILD_BASE_REF" -o "$BUILD_HEAD_REF" ] then - echo "No deploy -- proposed change: $BUILD_BASE_REF -> $BUILD_HEAD_REF" + echo "No deploy -- proposed change: $BUILD_HEAD_REF -> $BUILD_BASE_REF" else deployOK=1 fi From ef9a438028f48f2acc37a0b9a77900d10435d9a8 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 4 Oct 2024 17:42:03 -0500 Subject: [PATCH 425/429] melting-pot.sh: do not use symlinking on Windows The `ln -s` feature does not work in MINGW; instead it copies. And that cannot work as written because it's copying a folder into a subfolder of itself, resulting in an infinite loop. May be a bug in MINGW's ln command, but let's avoid the issue. --- melting-pot.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 9f8cd6f..055eb7e 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -773,10 +773,18 @@ meltDown() { # Use local directory for the specified project. test -d "$1" || die "No such directory: $1" 11 test -f "$1/pom.xml" || die "Not a Maven project: $1" 12 - info "Local Maven project: $1" - mkdir -p "LOCAL" - local projectDir="LOCAL/PROJECT" - ln -s "$1" "$projectDir" + case "$(uname)" in + MINGW*) + warn "Skipping inclusion of local project due to lack of symlink support." + local projectDir="$1" + ;; + *) + info "Local Maven project: $1" + mkdir -p "LOCAL" + local projectDir="LOCAL/PROJECT" + ln -s "$1" "$projectDir" + ;; + esac else # Treat specified project as a GAV. info "Fetching project source" From 66a91736e26d7851fd226da01cb61f2c369bdcab Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Fri, 4 Oct 2024 17:43:25 -0500 Subject: [PATCH 426/429] melting-pot.sh: specify version pins in a file This change avoids the dreaded "Argument list too long" that happens on Windows with too many -Dx.version=y arguments. One concern I had was whether specifying version overrides as settings.xml properties inside an active-by-default profile would still override the version properties of individual project POMs. But it works. I do not know whether it still works for project POMs with their own profiles that set version properties contingently; in that case, there would be multiple values competing, so it would likely come down to profile evaluation order. Fortunately, that is not common for SciJava-based projects. We also avoid usage of the `\\\\\n` sequence, since it appears to be handled incorrectly by MinGW's /bin/sh. --- melting-pot.sh | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 055eb7e..e78e4ad 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -666,7 +666,7 @@ do mismatch= for dep in $(echo "$deps" | tr ',' '\n') do - # g:a:p:v:s -> -Dg.a.version=v + # g:a:p:v:s -> v s=${dep##*:} case "$s" in test) continue ;; # skip test dependencies @@ -677,7 +677,7 @@ do apv=${gapv#*:} a=${apv%%:*} v=${apv##*:} - bomV=$(grep -o " -D$g\.$a\.version=[^ ]*" "$dir/build.sh" | sed 's;.*=;;') + bomV=$(grep -o " <$g\.$a\.version>[^>]*" "$dir/version-pins.xml" | sed 's;[^>]*>\([^>]*\)<.*;\1;') if [ "$bomV" != "${bomV#*-SNAPSHOT*}" ] then warn "$1: Snapshot dependency pin detected: $g:$a:$bomV -- forcing a rebuild" @@ -801,13 +801,9 @@ meltDown() { # to decide whether to include each component. generateHelperScripts - # NB: We do *not* include -B here, because we want build.sh to preserve - # colored output if the version of Maven is new enough. We will take care - # elsewhere when parsing it to be flexible about whether colors are present. - local args="-Denforcer.skip" - # Process the dependencies. info "Processing project dependencies" + local versionProps="" local dep for dep in $deps do @@ -817,9 +813,9 @@ meltDown() { local c="$(classifier "$dep")" test -z "$c" || continue # skip secondary artifacts local gav="$g:$a:$v" - test -z "$(isChanged "$gav")" && - args="$args \\\\\n -D$g.$a.version=$v -D$a.version=$v" + versionProps="$versionProps + <$g.$a.version>$v <$a.version>$v" done # Override versions of changed GAVs. @@ -830,16 +826,39 @@ meltDown() { do local a="$(artifactId "$gav")" local v="$(version "$gav")" - args="$args \\\\\n -D$a.version=$v" + versionProps="$versionProps + <$a.version>$v" done unset TLS + # Generate version-pins.xml. + info "Generating version-pins.xml configuration" + echo ' version-pins.xml + echo ' xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.1.0 https://maven.apache.org/xsd/settings-1.1.0.xsd">' >> version-pins.xml + echo ' ' >> version-pins.xml + echo ' ' >> version-pins.xml + echo ' version-pins' >> version-pins.xml + echo ' ' >> version-pins.xml + echo ' true' >> version-pins.xml + echo ' ' >> version-pins.xml + echo ' ' >> version-pins.xml + echo "$versionProps" >> version-pins.xml + echo ' ' >> version-pins.xml + echo ' ' >> version-pins.xml + echo ' ' >> version-pins.xml + echo '' >> version-pins.xml + # Generate build script. info "Generating build.sh script" echo '#!/bin/sh' > build.sh echo >> build.sh + echo 'dir=$(cd "$(dirname "$0")" && pwd)' >> build.sh + echo >> build.sh echo 'mvnPin() {' >> build.sh - echo " mvn $args \\\\\n \$@" >> build.sh + # NB: We do *not* include -B here, because we want build.sh to preserve + # colored output if the version of Maven is new enough. We will take care + # elsewhere when parsing it to be flexible about whether colors are present. + echo ' mvn -s "$dir/version-pins.xml" -Denforcer.skip $@' >> build.sh echo '}' >> build.sh echo >> build.sh echo 'unpackArtifact() {' >> build.sh From 75fc655f5bc12ea7e6afc36b799da53990c85a0f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 6 Oct 2024 09:29:36 -0500 Subject: [PATCH 427/429] melting-pot.sh: use valid short property names There are some artifactIds that begin with a digit rather than a letter, which results in invalid version property names. Prepending the property name with underscore avoids the issue. --- melting-pot.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index e78e4ad..29de050 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -809,13 +809,15 @@ meltDown() { do local g="$(groupId "$dep")" local a="$(artifactId "$dep")" + local aa + echo "$a" | grep -q '^[0-9]' && aa="_$a" || aa="$a" local v="$(version "$dep")" local c="$(classifier "$dep")" test -z "$c" || continue # skip secondary artifacts local gav="$g:$a:$v" test -z "$(isChanged "$gav")" && versionProps="$versionProps - <$g.$a.version>$v <$a.version>$v" + <$g.$a.version>$v <$aa.version>$v" done # Override versions of changed GAVs. From 0e97dd09375b32892b61a81ca3b9bff56fb801f7 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 14 Jan 2025 18:49:14 -0600 Subject: [PATCH 428/429] class-version.sh: fix on-demand artifact download --- class-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/class-version.sh b/class-version.sh index a5b4954..96ddb53 100755 --- a/class-version.sh +++ b/class-version.sh @@ -62,7 +62,7 @@ do a=${ga#*:} v=${arg##*:} f="$HOME/.m2/repository/$(echo "$g" | tr '.' '/')/$a/$v/$a-$v.jar" - test -f "$f" || mvn dependency:get -D"$arg" + test -f "$f" || mvn dependency:get -Dartifact="$arg" arg="$f" ;; esac From 950438f8bf082247e6c8798e35418c5b9eee53c6 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 15 May 2025 17:56:20 -0500 Subject: [PATCH 429/429] class-version.sh: recognize G:A:V:C coords --- class-version.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/class-version.sh b/class-version.sh index 96ddb53..ed77a66 100755 --- a/class-version.sh +++ b/class-version.sh @@ -55,8 +55,20 @@ first_class() { for arg in "$@" do + # Resolve Maven dependency coordinates into local files. case "$arg" in - *:*:*) + *:*:*:*) # g:a:v:c + gav=${arg%:*} + g=${gav%%:*} + av=${gav#*:} + a=${av%:*} + v=${av#*:} + c=${arg##*:} + f="$HOME/.m2/repository/$(echo "$g" | tr '.' '/')/$a/$v/$a-$v-$c.jar" + test -f "$f" || mvn dependency:get -Dartifact="$g:$a:$v:jar:$c" + arg="$f" + ;; + *:*:*) # g:a:v ga=${arg%:*} g=${ga%%:*} a=${ga#*:} @@ -66,6 +78,7 @@ do arg="$f" ;; esac + # Handle the various local file cases. case "$arg" in *.class) version=$(cat "$arg" | class_version) @@ -84,6 +97,6 @@ do continue esac - # report the results + # Report the results. echo "$arg: $version" done