8000 Debug CI failure · Arthurm1/scip-java@ab73d3b · GitHub
[go: up one dir, main page]
Skip to content

Commit ab73d3b

Browse files
olafurpgolafurpg
committed
Debug CI failure
1 parent 7fcb53c commit ab73d3b

File tree

7 files changed

+132
-90
lines changed

7 files changed

+132
-90
lines changed

‎.github/workflows/ci.yml

Lines changed: 71 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -5,76 +5,76 @@ on:
55
- main
66
pull_request:
77
jobs:
8-
test:
9-
runs-on: ${{ matrix.os }}
10-
name: Tests
11-
strategy:
12-
fail-fast: false
13-
matrix:
14-
# NOTE(olafurpg) Windows is not enabled because it times out due to reasons I don't understand.
15-
# os: [windows-latest, ubuntu-latest]
16-
os: [ubuntu-latest]
17-
java: [8, 11, 17]
18-
steps:
19-
- uses: actions/checkout@v2
20-
- uses: actions/setup-java@v3
21-
with:
22-
distribution: "temurin"
23-
cache: "sbt"
24-
java-version: ${{ matrix.java }}
25-
- name: Main project tests
26-
run: sbt test
8+
# test:
9+
# runs-on: ${{ matrix.os }}
10+
# name: Tests
11+
# strategy:
12+
# fail-fast: false
13+
# matrix:
14+
# # NOTE(olafurpg) Windows is not enabled because it times out due to reasons I don't understand.
15+
# # os: [windows-latest, ubuntu-latest]
16+
# os: [ubuntu-latest]
17+
# java: [8, 11, 17]
18+
# steps:
19+
# - uses: actions/checkout@v2
20+
# - uses: actions/setup-java@v3
21+
# with:
22+
# distribution: "temurin"
23+
# cache: "sbt"
24+
# java-version: ${{ matrix.java }}
25+
# - name: Main project tests
26+
# run: sbt test
2727

28-
docker_test:
29-
runs-on: ${{ matrix.os }}
30-
name: Docker CLI tests
31-
strategy:
32-
fail-fast: true
33-
matrix:
34-
os: [ubuntu-latest]
35-
steps:
36-
- uses: actions/checkout@v2
28+
# docker_test:
29+
# runs-on: ${{ matrix.os }}
30+
# name: Docker CLI tests
31+
# strategy:
32+
# fail-fast: true
33+
# matrix:
34+
# os: [ubuntu-latest]
35+
# steps:
36+
# - uses: actions/checkout@v2
3737

38-
- uses: actions/setup-java@v3
39-
with:
40-
distribution: "temurin"
41-
cache: "sbt"
42-
java-version: 17
38+
# - uses: actions/setup-java@v3
39+
# with:
40+
# distribution: "temurin"
41+
# cache: "sbt"
42+
# java-version: 17
4343

44-
- name: Build Dockerised CLI
45-
run: sbt cli/docker
44+
# - name: Build Dockerised CLI
45+
# run: sbt cli/docker
4646

47-
- run: |
47+
# - run: |
4848

49-
set -eu
50-
check_repo() {
51-
REPO=$1
52-
mkdir -p .repos/$REPO
53-
git clone https://github.com/$REPO.git .repos/$REPO
49+
# set -eu
50+
# check_repo() {
51+
# REPO=$1
52+
# mkdir -p .repos/$REPO
53+
# git clone https://github.com/$REPO.git .repos/$REPO
5454

55-
docker run -v $PWD/.repos/$REPO:/sources -w /sources sourcegraph/scip-java:latest scip-java index
56-
file .repos/$REPO/index.scip || (echo "$REPO SCIP index doesn't exist!"; exit 1)
57-
}
55+
# docker run -v $PWD/.repos/$REPO:/sources -w /sources sourcegraph/scip-java:latest scip-java index
56+
# file .repos/$REPO/index.scip || (echo "$REPO SCIP index doesn't exist!"; exit 1)
57+
# }
5858

59-
sudo apt install parallel
60-
export -f check_repo
59+
# sudo apt install parallel
60+
# export -f check_repo
6161

62-
parallel -j4 check_repo ::: circe/circe indeedeng/iwf-java-sdk
62+
# parallel -j4 check_repo ::: circe/circe indeedeng/iwf-java-sdk
6363

64-
bazel:
65-
runs-on: ubuntu-latest
66-
steps:
67-
- uses: actions/checkout@v2
68-
- run: yarn global add @bazel/bazelisk
69-
- run: bazel build //... --//semanticdb-javac:enabled=true
70-
- run: bazel run scip-semanticdb:bazel -- --sourceroot "$PWD"
71-
- run: du -h index.scip
72-
- run: bazel build //... --@scip_java//semanticdb-javac:enabled=true
73-
working-directory: examples/bazel-example
74-
- run: bazel run @scip_java//scip-semanticdb:bazel -- --sourceroot "$PWD"
75-
working-directory: examples/bazel-example
76-
- run: du -h index.scip
77-
working-directory: examples/bazel-example
64+
# bazel:
65+
# runs-on: ubuntu-latest
66+
# steps:
67+
# - uses: actions/checkout@v2
68+
# - run: yarn global add @bazel/bazelisk
69+
# - run: bazel build //... --//semanticdb-javac:enabled=true
70+
# - run: bazel run scip-semanticdb:bazel -- --sourceroot "$PWD"
71+
# - run: du -h index.scip
72+
# - run: bazel build //... --@scip_java//semanticdb-javac:enabled=true
73+
# working-directory: examples/bazel-example
74+
# - run: bazel run @scip_java//scip-semanticdb:bazel -- --sourceroot "$PWD"
75+
# working-directory: examples/bazel-example
76+
# - run: du -h index.scip
77+
# working-directory: examples/bazel-example
7878

7979
bazel_aspect:
8080
runs-on: ubuntu-latest
@@ -83,22 +83,19 @@ jobs:
8383
- run: yarn global add @bazel/bazelisk
8484
- run: sbt cli/pack
8585
- run: echo "$PWD/scip-java/target/pack/bin" >> $GITHUB_PATH
86-
- run: |
87-
scip-java index --build-tool=bazel --bazel-scip-java-binary=$(which scip-java)
88-
- run: du -h index.scip
8986
- run: |
9087
scip-java index --build-tool=bazel --bazel-scip-java-binary=$(which scip-java)
9188
working-directory: examples/bazel-example
9289
- run: du -h index.scip
9390
working-directory: examples/bazel-example
9491

95-
check:
96-
runs-on: ubuntu-latest
97-
steps:
98-
- uses: actions/checkout@v2
99-
- uses: actions/setup-java@v3
100-
with:
101-
distribution: "temurin"
102-
java-version: 17
103-
cache: "sbt"
104-
- run: sbt checkAll
92+
# check:
93+
# runs-on: ubuntu-latest
94+
# steps:
95+
# - uses: actions/checkout@v2
96+
# - uses: actions/setup-java@v3
97+
# with:
98+
# distribution: "temurin"
99+
# java-version: 17
100+
# cache: "sbt"
101+
# - run: sbt checkAll

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,3 +65,4 @@ bazel-lsif-java
6565
VERSION
6666

6767
semanticdb-gradle-plugin/gradle
68+
aspects/scip_java.bzl

‎build.sbt

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,10 @@ lazy val javacPlugin = project
167167
// referenced from META-INF/services/com.sun.source.util.Plugin
168168
"com.sourcegraph.semanticdb_javac.SemanticdbPlugin" ->
169169
"com.sourcegraph.semanticdb_javac.SemanticdbPlugin",
170+
"com.sourcegraph.semanticdb_javac.PrintJavaVersion" ->
171+
"com.sourcegraph.semanticdb_javac.PrintJavaVersion",
172+
"com.sourcegraph.semanticdb_javac.InjectSemanticdbOptions" ->
173+
"com.sourcegraph.semanticdb_javac.InjectSemanticdbOptions",
170174
"com.google.**" -> "com.sourcegraph.shaded.com.google.@1",
171175
// Need to shade the semanticdb-javac compiler plugin in order to be
172176
// able to index the plugin code itself.
@@ -257,7 +261,7 @@ lazy val cli = project
257261
}
258262

259263
addJar(
260-
(javacPlugin / Compile / assembly).value,
264+
(javacPlugin / Compile / Keys.`package`).value,
261265
"semanticdb-plugin.jar"
262266
)
263267
addJar(

‎examples/bazel-example/.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
aspects/scip_java.bzl
21
bazel-bazel-example
2+
aspects/scip_java.bzl

‎scip-java/src/main/resources/scip-java/scip_java.bzl

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ def _scip_java(target, ctx):
8282
output = build_config_path,
8383
content = build_config.to_json(),
8484
)
85+
86+
deps = [javac_action.inputs, annotations.processor_classpath]
8587
ctx.actions.run_shell(
8688
command = "\"{}\" index --no-cleanup --index-semanticdb.allow-empty-index --cwd \"{}\" --targetroot {} --scip-config \"{}\" --output \"{}\"".format(
8789
ctx.var["scip_java_binary"],
@@ -94,7 +96,7 @@ def _scip_java(target, ctx):
9496
"JAVA_HOME": ctx.var["java_home"],
9597
"NO_PROGRESS_BAR": "true",
9698
},
97-
inputs = [build_config_path],
99+
inputs = depset([build_config_path], transitive = deps),
98100
outputs = [scip_output, targetroot],
99101
)
100102

‎scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BazelBuildTool.scala

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,12 @@ import java.nio.file.Path
88
import java.nio.file.SimpleFileVisitor
99
import java.nio.file.StandardOpenOption
1010
import java.nio.file.attribute.BasicFileAttributes
11-
1211
import scala.collection.mutable.ListBuffer
13-
1412
import com.sourcegraph.io.AbsolutePath
1513
import com.sourcegraph.scip_java.Embedded
1614
import com.sourcegraph.scip_java.commands.IndexCommand
15+
import moped.cli.Application
16+
import os.ProcessOutput.Readlines
1717

1818
class BazelBuildTool(index: IndexCommand) extends BuildTool("Bazel", index) {
1919

@@ -31,6 +31,8 @@ class BazelBuildTool(index: IndexCommand) extends BuildTool("Bazel", index) {
3131
val buildCommand = ListBuffer.empty[String]
3232
buildCommand += "bazel"
3333
buildCommand += "build"
34+
buildCommand += "--noshow_progress"
35+
buildCommand += "--sandbox_debug"
3436

3537
val targetSpecs = ListBuffer.empty[String]
3638
if (index.buildCommand.isEmpty) {
@@ -72,12 +74,18 @@ class BazelBuildTool(index: IndexCommand) extends BuildTool("Bazel", index) {
7274
}
7375
buildCommand += s"--define=scip_java_binary=$scipJavaBinary"
7476

77+
val sandbox = new SandboxCommandExtractor(index.app)
7578
index.app.info(buildCommand.mkString(" "))
7679
val commandResult = index
7780
.app
7881
.process(buildCommand.toList)
79-
.call(check = false)
82+
.call(check = false, stderr = Readlines(sandbox))
8083
if (commandResult.exitCode != 0) {
84+
// Automatically run the sandbox command to help the user debug the problem.
85+
index
86+
.app
87+
.process("bash", "-c", sandbox.commandLines().mkString("\n"))
88+
.call(check = false, stdout = os.Inherit, stderr = os.Inherit)
8189
index
8290
.app
8391
.error(
@@ -127,6 +135,35 @@ class BazelBuildTool(index: IndexCommand) extends BuildTool("Bazel", index) {
127135
}
128136
}
129137

138+
/**
139+
* Callback to `os.ProcessOutput.Readlines` that processes the stderr output
140+
* of a running Bazel process and extract the sandbox command. When the
141+
* --sandbox_debug flag is enabled, Bazel prints out the sandbox command that
142+
* failed but it doesn't show the stdout/stderr of that command. This
143+
* extractor captures the command so that we can automatically re-run the
144+
* command to print out the stdout/stderr.
145+
*/
146+
private class SandboxCommandExtractor(app: Application)
147+
extends (String => Unit) {
148+
private var isSandboxCommandPrinting = false
149+
private val lines = ListBuffer.empty[String]
150+
def commandLines(): List[String] = lines.toList
151+
def command(): List[String] = List("bash", "-c", lines.mkString("\n"))
152+
override def apply(line: String): Unit = {
153+
if (
154+
!isSandboxCommandPrinting && line.startsWith("ERROR:") &&
155+
line.endsWith("error executing command ")
156+
) {
157+
isSandboxCommandPrinting = true
158+
} else if (isSandboxCommandPrinting && !line.startsWith(" ")) {
159+
isSandboxCommandPrinting = false
160+
} else if (isSandboxCommandPrinting) {
161+
lines += line
162+
}
163+
app.env.standardError.println(line)
164+
}
165+
}
166+
130167
/**
131168
* Reads the scip_java.bzl file from resources and writes it to the
132169
* aspect/scip_java.bzl file inside the Bazel workspace.

‎scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
186186

187187
val compileAttempts = ListBuffer.empty[Try[Unit]]
188188
compileAttempts += compileJavaFiles(tmp, deps, config, javaFiles)
189-
compileAttempts += compileScalaFiles(deps, scalaFiles)
189+
compileAttempts += compileScalaFiles(deps, scalaFiles, tmp)
190190
compileAttempts += compileKotlinFiles(deps, config, kotlinFiles)
191191
val errors = compileAttempts.collect { case Failure(exception) =>
192192
exception
@@ -347,11 +347,12 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
347347

348348
private def compileScalaFiles(
349349
deps: Dependencies,
350-
allScalaFiles: List[Path]
350+
allScalaFiles: List[Path],
351+
tmp: Path
351352
): Try[Unit] =
352353
Try {
353354
if (deps.dependencies.nonEmpty && allScalaFiles.nonEmpty)
354-
withScalaPresentationCompiler(deps) { compiler =>
355+
withScalaPresentationCompiler(deps, tmp) { compiler =>
355356
allScalaFiles.foreach { path =>
356357
try compileScalaFile(compiler, path)
357358
catch {
@@ -393,9 +394,9 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
393394
)
394395
}
395396

396-
private def withScalaPresentationCompiler[T](
397-
deps: Dependencies
398-
)(fn: PresentationCompiler => T): T = {
397+
private def withScalaPresentationCompiler[T](deps: Dependencies, tmp: Path)(
398+
fn: PresentationCompiler => T
399+
): T = {
399400
val scalaVersion = deps
400401
.classpath
401402
.iterator
@@ -423,7 +424,7 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
423424
.iterator()
424425
if (compilers.hasNext) {
425426
val classpath = deps.classpath ++ scalaLibrary
426-
val argsfile = targetroot.resolve("javacopts.txt")
427+
val argsfile = tmp.resolve("javacopts.txt")
427428
Files.createDirectories(argsfile.getParent)
428429
Files.write(
429430
argsfile,
@@ -491,7 +492,7 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
491492
AbsolutePath.of(Paths.get(path), index.workingDirectory).toString
492493
)
493494
actualClasspath ++= deps.classpath.map(_.toString)
494-
val argsfile = targetroot.resolve("javacopts.txt")
495+
val argsfile = tmp.resolve("javacopts.txt")
495496
val arguments = ListBuffer.empty[String]
496497
arguments += "-encoding"
497498
arguments += "utf8"
@@ -677,7 +678,7 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
677678
!pathString.startsWith("bazel-bin") && !pathString.startsWith("bazel-out")
678679
) {
679680
path = AbsolutePath
680-
.of(Paths.get("bazel-out", pathString), workingDirectory)
681+
.of(Paths.get("bazel-bin", pathString), workingDirectory)
681682

682683
if (Files.isRegularFile(path))
683684
return Some(path)
@@ -758,7 +759,7 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
758759
processorpath: List[String] = Nil,
759760
processors: List[String] = Nil,
760761
javacOptions: List[String] = Nil,
761-
jvm: String = "8",
762+
jvm: String = "17",
762763
kind: String = ""
763764
)
764765
private object Config {

0 commit comments

Comments
 (0)
0