fix: use runfiles symlinks for venv symlink creation to reduce action count (#3402)

When the venv files are materialized, it can result in many symlink
actions
being created. Rather than register them as regular symlink actions,
batch them
into the runfiles object, which can probably handle large numbers of
them more
efficiently.

Work towards #3401

Author: rickeylev

python/private/py_executable.bzl

@@ -356,7 +356,7 @@ def _create_executable(
             [stage2_bootstrap] + (
                 venv.files_without_interpreter if venv else []
             ),
-        )
+        ).merge(venv.lib_runfiles)
         zip_main = _create_zip_main(
             ctx,
             stage2_bootstrap = stage2_bootstrap,
@@ -606,7 +606,7 @@ def _create_venv(ctx, output_prefix, imports, runtime_details):
     }
     venv_app_files = create_venv_app_files(ctx, ctx.attr.deps, venv_dir_map)
 
-    files_without_interpreter = [pth, site_init] + venv_app_files
+    files_without_interpreter = [pth, site_init] + venv_app_files.venv_files
     if pyvenv_cfg:
         files_without_interpreter.append(pyvenv_cfg)
 
@@ -629,6 +629,11 @@ def _create_venv(ctx, output_prefix, imports, runtime_details):
                 venv,
             ),
         ),
+        # venv files for user library dependencies (files that are specific
+        # to the executable bootstrap and python runtime aren't here).
+        lib_runfiles = ctx.runfiles(
+            symlinks = venv_app_files.runfiles_symlinks,
+        ),
     )
 
 def _map_each_identity(v):

python/private/venv_runfiles.bzl

@@ -30,7 +30,12 @@ def create_venv_app_files(ctx, deps, venv_dir_map):
             paths within the current ctx's venv (e.g. `_foo.venv/bin`).
 
     Returns:
-        {type}`list[File]` of the files that were created.
+        {type}`struct` with the following attributes:
+        * {type}`list[File]` `venv_files` additional files created for
+          the venv.
+        * {type}`dict[str, File]` `runfiles_symlinks` map intended for
+          the `runfiles.symlinks` argument. A map of main-repo
+          relative paths to File.
     """
 
     # maps venv-relative path to the runfiles path it should point to
@@ -44,16 +49,16 @@ def create_venv_app_files(ctx, deps, venv_dir_map):
 
     link_map = build_link_map(ctx, entries)
     venv_files = []
+    runfiles_symlinks = {}
+
     for kind, kind_map in link_map.items():
         base = venv_dir_map[kind]
         for venv_path, link_to in kind_map.items():
             bin_venv_path = paths.join(base, venv_path)
             if is_file(link_to):
-                if link_to.is_directory:
-                    venv_link = ctx.actions.declare_directory(bin_venv_path)
-                else:
-                    venv_link = ctx.actions.declare_file(bin_venv_path)
-                ctx.actions.symlink(output = venv_link, target_file = link_to)
+                symlink_from = "{}/{}".format(ctx.label.package, bin_venv_path)
+                runfiles_symlinks[symlink_from] = link_to
+
             else:
                 venv_link = ctx.actions.declare_symlink(bin_venv_path)
                 venv_link_rf_path = runfiles_root_path(ctx, venv_link.short_path)
@@ -64,9 +69,12 @@ def create_venv_app_files(ctx, deps, venv_dir_map):
                     to = link_to,
                 )
                 ctx.actions.symlink(output = venv_link, target_path = rel_path)
-            venv_files.append(venv_link)
+                venv_files.append(venv_link)
 
-    return venv_files
+    return struct(
+        venv_files = venv_files,
+        runfiles_symlinks = runfiles_symlinks,
+    )
 
 # Visible for testing
 def build_link_map(ctx, entries):