From dda90aa64cd056facb1357b3a8dcbee5655a83b9 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Thu, 10 Oct 2024 11:03:23 +0100 Subject: [PATCH 1/6] Unpack .srcjars in Bazel aspect and add them to scip config --- .../main/resources/scip-java/scip_java.bzl | 46 ++++++++++++++++--- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/scip-java/src/main/resources/scip-java/scip_java.bzl b/scip-java/src/main/resources/scip-java/scip_java.bzl index 948ee683..d1694a3d 100644 --- a/scip-java/src/main/resources/scip-java/scip_java.bzl +++ b/scip-java/src/main/resources/scip-java/scip_java.bzl @@ -57,10 +57,36 @@ def _scip_java(target, ctx): annotations = info.annotation_processing source_files = [] + source_jars = [] for src in ctx.rule.files.srcs: - source_files.append(src.path) + if src.path.endswith(".java"): + source_files.append(src.path) + elif src.path.endswith(".srcjar"): + source_jars.append(src) + if len(source_files) == 0: return None + + output_dir = [] + + for source_jar in source_jars: + dir = ctx.actions.declare_directory("extracted_" + source_jar.basename) + output_dir.append(dir) + + ctx.actions.run_shell( + inputs = javac_action.inputs, + outputs = [dir], + mnemonic = "ExtractSourceJars", + command = """ + unzip {input_file} -d {output_dir} + """.format( + output_dir = dir.path, + input_file = source_jar.path, + ), + progress_message = "Extracting source jar {jar}".format(jar = source_jar.path), + ) + + source_files.append(dir.path) classpath = [j.path for j in compilation.compilation_classpath.to_list()] bootclasspath = [j.path for j in compilation.boot_classpath] @@ -73,11 +99,16 @@ def _scip_java(target, ctx): launcher_javac_flags = [] compiler_javac_flags = [] - for value in compilation.javac_options: - if value.startswith("-J"): - launcher_javac_flags.append(value) - else: - compiler_javac_flags.append(value) + + for value in compilation.javac_options_list: + # NOTE(Anton): for some bizarre reason I see empty string starting the list of + # javac options - which then gets propagated into the JSON config, and ends up + # crashing the actual javac invokation. + if value != "": + if value.startswith("-J"): + launcher_javac_flags.append(value) + else: + compiler_javac_flags.append(value) build_config = struct(**{ "javaHome": ctx.var["java_home"], @@ -100,6 +131,7 @@ def _scip_java(target, ctx): ) deps = [javac_action.inputs, annotations.processor_classpath] + ctx.actions.run_shell( command = "\"{}\" index --no-cleanup --index-semanticdb.allow-empty-index --cwd \"{}\" --targetroot {} --scip-config \"{}\" --output \"{}\"".format( ctx.var["scip_java_binary"], @@ -113,7 +145,7 @@ def _scip_java(target, ctx): "NO_PROGRESS_BAR": "true", }, mnemonic = "ScipJavaIndex", - inputs = depset([build_config_path], transitive = deps), + inputs = depset([build_config_path] + output_dir, transitive = deps), outputs = [scip_output, targetroot], ) From 8a247cc7de2a021319ec5e9fcc99d53e8d34b422 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Thu, 10 Oct 2024 11:03:46 +0100 Subject: [PATCH 2/6] Use --verbose_failures when invoking scip-java aspect --- .../com/sourcegraph/scip_java/buildtools/BazelBuildTool.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BazelBuildTool.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BazelBuildTool.scala index a9918b73..c88ed4cd 100644 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BazelBuildTool.scala +++ b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BazelBuildTool.scala @@ -71,7 +71,8 @@ class BazelBuildTool(index: IndexCommand) extends BuildTool("Bazel", index) { "--output_groups=scip", s"--define=sourceroot=${index.workingDirectory}", s"--define=java_home=$javaHome", - s"--define=scip_java_binary=$scipJavaBinary" + s"--define=scip_java_binary=$scipJavaBinary", + "--verbose_failures" ) ++ targetSpecs val buildExitCode = runBazelBuild(buildCommand) From 27e9fc66d860df925182254a54722dd33ef6d13a Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Thu, 10 Oct 2024 11:04:03 +0100 Subject: [PATCH 3/6] Scip buildtool can now handle folders of source files --- .../scip_java/buildtools/ScipBuildTool.scala | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala index d0477aa8..a5882ef1 100644 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala +++ b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala @@ -683,15 +683,8 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) { Files.walkFileTree(targetroot, new DeleteVisitor) } - /** Recursively collects all Java files in the working directory */ - private def collectAllSourceFiles(config: Config, dir: Path): List[Path] = { - if (config.sourceFiles.nonEmpty) { - return config - .sourceFiles - .map(path => AbsolutePath.of(Paths.get(path), dir)) - .filter(path => Files.isRegularFile(path)) - } - val buf = ListBuffer.empty[Path] + private def collectAllSourceFiles(dir: Path) = { + val buf = List.newBuilder[Path] Files.walkFileTree( dir, new SimpleFileVisitor[Path] { @@ -719,7 +712,27 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) { ): FileVisitResult = FileVisitResult.CONTINUE } ) - buf.toList + buf.result() + } + + /** Recursively collects all Java files in the working directory */ + private def collectAllSourceFiles(config: Config, dir: Path): List[Path] = { + if (config.sourceFiles.nonEmpty) { + println(config.sourceFiles) + config + .sourceFiles + .flatMap { relativePath => + val path = AbsolutePath.of(Paths.get(relativePath), dir) + + if (Files.isRegularFile(path) && allPatterns.matches(path)) + List(path) + else if (Files.isDirectory(path)) + collectAllSourceFiles(path) + else + Nil + } + } else + collectAllSourceFiles(dir) } // HACK(olafurpg): I haven't figured out a reliable way to get annotation processor jars on the processorpath. From 187e67b5e27644c059f510c6d2ecc3b5f9ab7104 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Thu, 10 Oct 2024 11:12:14 +0100 Subject: [PATCH 4/6] Improve compatibility with different Bazel versions --- .../src/main/resources/scip-java/scip_java.bzl | 13 ++++++++++--- .../scip_java/buildtools/ScipBuildTool.scala | 1 - 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/scip-java/src/main/resources/scip-java/scip_java.bzl b/scip-java/src/main/resources/scip-java/scip_java.bzl index d1694a3d..74c1a425 100644 --- a/scip-java/src/main/resources/scip-java/scip_java.bzl +++ b/scip-java/src/main/resources/scip-java/scip_java.bzl @@ -70,7 +70,7 @@ def _scip_java(target, ctx): output_dir = [] for source_jar in source_jars: - dir = ctx.actions.declare_directory("extracted_" + source_jar.basename) + dir = ctx.actions.declare_directory("extracted_srcjar/" + source_jar.short_path) output_dir.append(dir) ctx.actions.run_shell( @@ -99,8 +99,15 @@ def _scip_java(target, ctx): launcher_javac_flags = [] compiler_javac_flags = [] - - for value in compilation.javac_options_list: + + # In different versions of bazel javac options are either a nested set or a depset or a list... + javac_options = [] + if hasattr(compilation, "javac_options_list"): + javac_options = compilation.javac_options_list + else: + javac_options = compilation.javac_options.to_list() + + for value in javac_options: # NOTE(Anton): for some bizarre reason I see empty string starting the list of # javac options - which then gets propagated into the JSON config, and ends up # crashing the actual javac invokation. diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala index a5882ef1..92726743 100644 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala +++ b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala @@ -718,7 +718,6 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) { /** Recursively collects all Java files in the working directory */ private def collectAllSourceFiles(config: Config, dir: Path): List[Path] = { if (config.sourceFiles.nonEmpty) { - println(config.sourceFiles) config .sourceFiles .flatMap { relativePath => From 91c742b97d457109f86fb887482800f40f8d4014 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Thu, 10 Oct 2024 11:17:45 +0100 Subject: [PATCH 5/6] remove redundant conversion --- scip-java/src/main/resources/scip-java/scip_java.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scip-java/src/main/resources/scip-java/scip_java.bzl b/scip-java/src/main/resources/scip-java/scip_java.bzl index 74c1a425..8976d4e6 100644 --- a/scip-java/src/main/resources/scip-java/scip_java.bzl +++ b/scip-java/src/main/resources/scip-java/scip_java.bzl @@ -105,7 +105,7 @@ def _scip_java(target, ctx): if hasattr(compilation, "javac_options_list"): javac_options = compilation.javac_options_list else: - javac_options = compilation.javac_options.to_list() + javac_options = compilation.javac_options for value in javac_options: # NOTE(Anton): for some bizarre reason I see empty string starting the list of From 4b5e1fc12360f5cd796ca08f1214e8c99bfc7413 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Thu, 10 Oct 2024 11:30:10 +0100 Subject: [PATCH 6/6] Add a srcjar example/test --- .../src/main/java/srcjar_example/BUILD | 13 +++++++++++++ .../src/main/java/srcjar_example/Foo.java | 8 ++++++++ 2 files changed, 21 insertions(+) create mode 100644 examples/bazel-example/src/main/java/srcjar_example/BUILD create mode 100644 examples/bazel-example/src/main/java/srcjar_example/Foo.java diff --git a/examples/bazel-example/src/main/java/srcjar_example/BUILD b/examples/bazel-example/src/main/java/srcjar_example/BUILD new file mode 100644 index 00000000..ff65eb50 --- /dev/null +++ b/examples/bazel-example/src/main/java/srcjar_example/BUILD @@ -0,0 +1,13 @@ +genrule( + name = "generated-srcjar", + outs = ["sources.srcjar"], + cmd = "echo 'package com.testing; public class Bar {};' > Bar.java && jar cf $(@) Bar.java", +) + +java_library( + name = "testing", + srcs = [ + "Foo.java", + ":generated-srcjar", + ], +) diff --git a/examples/bazel-example/src/main/java/srcjar_example/Foo.java b/examples/bazel-example/src/main/java/srcjar_example/Foo.java new file mode 100644 index 00000000..f2122ff2 --- /dev/null +++ b/examples/bazel-example/src/main/java/srcjar_example/Foo.java @@ -0,0 +1,8 @@ +// testing/Foo.java +package com.testing; + +public class Foo { + public Bar foo(Bar value) { + return value; + } +}