New approach for handling RBI files.

coinbase · bobg · Nov 18, 2020 · Nov 18, 2020 · Nov 24, 2020 · Nov 24, 2020
commit 426306d5148708e0ef81c3081bbe387e110d3562
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,6 @@
+# Emacs backups
+*~
+
 # Bazel Ignores
 **/bazel-*
 .bazelrc.user

diff --git a/ruby/private/gem.bzl b/ruby/private/gem.bzl
@@ -12,17 +12,12 @@ def rb_gem(name, version, gem_name, **kwargs):
     deps = kwargs.get("deps", [])
     source_date_epoch = kwargs.pop("source_date_epoch", None)
     srcs = kwargs.pop("srcs", [])
-    strip_paths = kwargs.pop("strip_paths", [])
     verbose = kwargs.pop("verbose", False)
 
-    if strip_paths and "require_paths" not in kwargs:
-        fail("Must specify 'require_paths' when using the 'strip_paths' argument")
-
     _rb_gemspec(
         name = _gemspec_name,
         gem_name = gem_name,
         version = version,
-        strip_paths = strip_paths,
         **kwargs
     )
 
@@ -34,6 +29,5 @@ def rb_gem(name, version, gem_name, **kwargs):
         deps = srcs + deps,
         visibility = ["//visibility:public"],
         source_date_epoch = source_date_epoch,
-        strip_paths = strip_paths,
         verbose = verbose,
     )
diff --git a/ruby/private/gem/gem.bzl b/ruby/private/gem/gem.bzl
@@ -1,9 +1,11 @@
 load("//ruby/private:providers.bzl", "RubyGem")
-load("//ruby/private/tools:paths.bzl", "strip_short_path")
+load("//ruby/private/tools:paths.bzl", "shorten_for_package")
 
 # Runs gem with arbitrary arguments
 # eg: run_gem(runtime_ctx, ["install" "foo"])
 def _rb_build_gem_impl(ctx):
+    print("xxx _rb_build_gem_impl ctx.label.package {}".format(ctx.label.package))
+
     metadata_file = ctx.actions.declare_file("{}_build_metadata".format(ctx.attr.gem_name))
     gemspec = ctx.attr.gemspec[RubyGem].gemspec
 
@@ -13,12 +15,16 @@ def _rb_build_gem_impl(ctx):
         file_deps = dep.files.to_list()
         _inputs.extend(file_deps)
         for f in file_deps:
-            dest_path = strip_short_path(f.short_path, ctx.attr.strip_paths)
+            print("xxx _rb_build_gem_impl f.short_path {}".format(f.short_path))
+            print("xxx _rb_build_gem_impl f.is_directory {}".format(f.is_directory))
             _srcs.append({
                 "src_path": f.path,
-                "dest_path": dest_path,
+                "dest_path": shorten_for_package(f, ctx.label.package),
             })
 
+    print("xxx _rb_build_gem_impl _inputs {}".format(_inputs))
+    print("xxx _rb_build_gem_impl _srcs {}".format(_srcs))
+
     ctx.actions.write(
         output = metadata_file,
         content = struct(
@@ -27,6 +33,7 @@ def _rb_build_gem_impl(ctx):
             output_path = ctx.outputs.gem.path,
             source_date_epoch = ctx.attr.source_date_epoch,
             verbose = ctx.attr.verbose,
+            package = ctx.label.package,
         ).to_json(),
     )
 
@@ -73,7 +80,6 @@ _ATTRS = {
     "source_date_epoch": attr.string(
         doc = "Sets source_date_epoch env var which should make output gems hermetic",
     ),
-    "strip_paths": attr.string_list(),
     "verbose": attr.bool(default = False),
 }
 

diff --git a/ruby/private/gem/gem_runner.rb b/ruby/private/gem/gem_runner.rb
@@ -31,16 +31,32 @@ def parse_opts
   metadata_file
 end
 
-def copy_srcs(dir, srcs, verbose)
+def copy_srcs(dir, srcs, pkg, verbose)
   # Sources need to be moved from their bazel_out locations
   # to the correct folder in the ruby gem.
+  puts "xxx copy_srcs dir #{dir} pkg #{pkg}"
   srcs.each do |src|
     src_path = src['src_path']
     dest_path = src['dest_path']
-    tmpname = File.join(dir, File.dirname(dest_path))
-    FileUtils.mkdir_p(tmpname)
-    puts "copying #{src_path} to #{tmpname}" if verbose
-    FileUtils.cp_r(src_path, tmpname)
+    puts "xxx copy_srcs src_path #{src_path} dest_path #{dest_path}"
+    if dest_path == pkg
+      tmpname = dir
+    else
+      if dest_path.start_with?(pkg+"/")
+        tmpname = File.join(dir, dest_path[pkg.length+1, dest_path.length-pkg.length-1])
+      else
+        tmpname = File.join(dir, dest_path)
+      end
+    end
+    if File.directory?(src_path)
+      puts "cp -r #{src_path}/ #{tmpname}" if verbose
+      FileUtils.mkdir_p(tmpname)
+      FileUtils.cp_r(src_path+"/.", tmpname)
+    else
+      tmpname = File.dirname(tmpname)
+      puts "cp #{src_path} #{tmpname}" if verbose
+      FileUtils.cp(src_path, tmpname)
+    end
     # Copying a directory will not dereference symlinks
     # in the directory. They need to be removed too.
     dereference_symlinks(tmpname, verbose) if File.directory?(tmpname)
@@ -68,9 +84,14 @@ def do_build(dir, gemspec_path, output_path)
     'build',
     File.join(dir, File.basename(gemspec_path))
   ]
+  puts "xxx do_build dir is #{dir} args is #{args}"
   # Older versions of rubygems work better if the
   # cwd is the root of the gem dir.
   Dir.chdir(dir) do
+    Dir.glob("**/*") do |f|
+      puts "xxx do_build found #{f}"
+    end
+
     Gem::GemRunner.new.run args
   end
   FileUtils.cp(File.join(dir, File.basename(output_path)), output_path)
@@ -79,9 +100,10 @@ def do_build(dir, gemspec_path, output_path)
 def build_gem(metadata)
   # We copy all related files to a tmpdir, build the entire gem in that tmpdir
   # and then copy the output gem into the correct bazel output location.
+  puts "xxx build_gem metadata #{metadata}"
   verbose = metadata['verbose']
   Dir.mktmpdir do |dir|
-    copy_srcs(dir, metadata['srcs'], verbose)
+    copy_srcs(dir, metadata['srcs'], metadata['package'], verbose)
     copy_gemspec(dir, metadata['gemspec_path'])
     do_build(dir, metadata['gemspec_path'], metadata['output_path'])
   end

diff --git a/ruby/private/gem/gemspec.bzl b/ruby/private/gem/gemspec.bzl
@@ -2,20 +2,22 @@ load(
     "//ruby/private/tools:deps.bzl",
     _transitive_deps = "transitive_deps",
 )
-load("//ruby/private/tools:paths.bzl", "strip_short_path")
 load(
     "//ruby/private:providers.bzl",
     "RubyGem",
     "RubyLibrary",
 )
+load("//ruby/private/tools:paths.bzl", "shorten_for_package")
 
 def _get_transitive_srcs(srcs, deps):
     return depset(
         srcs,
         transitive = [dep[RubyLibrary].transitive_ruby_srcs for dep in deps],
     )
 
-def _rb_gem_impl(ctx):
+def _rb_gemspec_impl(ctx):
+    print("xxx _rb_gemspec_impl ctx.label.package {}".format(ctx.label.package))
+
     gemspec = ctx.actions.declare_file("{}.gemspec".format(ctx.attr.gem_name))
     metadata_file = ctx.actions.declare_file("{}_metadata".format(ctx.attr.gem_name))
 
@@ -25,12 +27,15 @@ def _rb_gem_impl(ctx):
         # For some files the src_path and dest_path will be the same, but
         # for othrs the src_path will be in bazel)out while the dest_path
         # will be from the workspace root.
-        dest_path = strip_short_path(f.short_path, ctx.attr.strip_paths)
         _ruby_files.append({
             "src_path": f.path,
-            "dest_path": dest_path,
+            "dest_path": shorten_for_package(f, ctx.label.package),
+            "short_path": f.short_path,
         })
 
+    print("xxx _rb_gemspec_impl _ruby_files {}".format(_ruby_files))
+    print("xxx _rb_gemspec_impl require_paths {}".format(ctx.attr.require_paths))
+
     ctx.actions.write(
         output = metadata_file,
         content = struct(
@@ -41,6 +46,7 @@ def _rb_gem_impl(ctx):
             licenses = ctx.attr.licenses,
             require_paths = ctx.attr.require_paths,
             gem_runtime_dependencies = ctx.attr.gem_runtime_dependencies,
+            package = ctx.label.package,
         ).to_json(),
     )
 
@@ -101,7 +107,6 @@ _ATTRS = {
         default = [],
     ),
     "require_paths": attr.string_list(),
-    "strip_paths": attr.string_list(),
     "_gemspec_template": attr.label(
         allow_single_file = True,
         default = "gemspec_template.tpl",
@@ -122,7 +127,7 @@ _ATTRS = {
 }
 
 rb_gemspec = rule(
-    implementation = _rb_gem_impl,
+    implementation = _rb_gemspec_impl,
     attrs = _ATTRS,
     provides = [DefaultInfo, RubyGem],
 )
diff --git a/ruby/private/gem/gemspec_builder.rb b/ruby/private/gem/gemspec_builder.rb
@@ -30,8 +30,11 @@ def parse_opts
 def parse_metadata(metadata)
   # Expand all of the sources first
   metadata = expand_src_dirs(metadata)
+  puts "xxx expanded src dirs, metadata now #{metadata}"
   metadata = parse_require_paths(metadata)
+  puts "xxx parsed require paths, metadata now #{metadata}"
   metadata = parse_gem_runtime_dependencies(metadata)
+  puts "xxx parsed gem runtime deps, metadata now #{metadata}"
   metadata
 end
 
@@ -62,21 +65,36 @@ def parse_gem_runtime_dependencies(metadata)
 def expand_src_dirs(metadata)
   # Files and required paths can include a directory which gemspec
   # cannot handle. This will convert directories to individual files
+  pkg = metadata['package']
+  puts "xxx expand_src_dirs pkg #{pkg}"
   srcs = metadata['raw_srcs']
   new_srcs = []
   srcs.each do |src|
     src_path = src['src_path']
     dest_path = src['dest_path']
+    short_path = src['short_path']
+    puts "xxx expand_src_dirs src_path #{src_path} dest_path #{dest_path} short_path #{short_path}"
     if File.directory?(src_path)
       Dir.glob("#{src_path}/**/*") do |f|
         # expand the directory, replacing each src path with its dest path
-        new_srcs << f.gsub(src_path, dest_path) if File.file?(f)
+        if File.file?(f) 
+          puts "xxx expand_src_dirs f [1] #{f}"
+          if f.start_with?(src_path+"/")
+            f[0, src_path.length] = dest_path
+          end
+          if f.start_with?(pkg+"/")
+            f[0, pkg.length+1] = ""
+          end
+          puts "xxx expand_src_dirs [2] f #{f}"
+          new_srcs << f
+        end
       end
     elsif File.file?(src_path)
       new_srcs << dest_path
     end
   end
   metadata['srcs'] = new_srcs
+  puts "xxx expand_src_dirs srcs #{srcs} new_srcs #{new_srcs}"
   metadata
 end
 

diff --git a/ruby/private/tools/paths.bzl b/ruby/private/tools/paths.bzl
@@ -1,12 +1,19 @@
-def strip_short_path(path, strip_paths):
-    """ Given a short_path string will iterate over the
-    list of strip_paths and remove any matches returning a
-    new path string with no leading slash.
+def shorten_for_package(f, pkg):
+    """Remove `pkg` from the beginning of `f.short_path`.
+    If the result is a bare filename (with no further dir levels), return it.
+    Otherwise, remove one more dir level.
+    If that results in an empty path, return ".".
     """
-    if not strip_paths:
-        return path
-
-    for strip_path in strip_paths:
-        if path.startswith(strip_path):
-            return path.replace(strip_path, "").lstrip("/")
+    print("xxx shorten_for_package({}, {})".format(f.short_path, pkg))
+    path = f.short_path
+    if path.startswith(pkg+"/"):
+        path = path[len(pkg)+1:]
+    slash = path.find("/")
+    if slash >= 0:
+        print("xxx shorten_for_package [1] -> {}/{}".format(pkg, path[slash+1:]))
+        return pkg+"/"+path[slash+1:]
+    if f.is_directory:
+        print("xxx shorten_for_package [2] -> {}".format(pkg))
+        return pkg
+    print("xxx shorten_for_package [3] -> {}".format(path))
     return path