From a982e4037b37317de2088e3edf0d5f8c217cd3ad Mon Sep 17 00:00:00 2001 From: Toby Harradine Date: Mon, 27 Oct 2025 14:19:10 +1100 Subject: [PATCH 1/4] perf: improve analysis performance of `py_binary` and `py_test` --- python/private/py_executable.bzl | 38 +++++++++++++------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 1a5ad4c3c6..03dceb7cb5 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -380,9 +380,8 @@ def _create_executable( _create_zip_file( ctx, output = zip_file, - original_nonzip_executable = executable, zip_main = zip_main, - runfiles = runfiles_details.default_runfiles.merge(extra_runfiles), + runfiles = runfiles_details.runfiles_without_exe.merge(extra_runfiles), ) extra_files_to_build = [] @@ -803,7 +802,7 @@ def _create_windows_exe_launcher( use_default_shell_env = True, ) -def _create_zip_file(ctx, *, output, original_nonzip_executable, zip_main, runfiles): +def _create_zip_file(ctx, *, output, zip_main, runfiles): """Create a Python zipapp (zip with __main__.py entry point).""" workspace_name = ctx.workspace_name legacy_external_runfiles = _py_builtins.get_legacy_external_runfiles(ctx) @@ -819,17 +818,17 @@ def _create_zip_file(ctx, *, output, original_nonzip_executable, zip_main, runfi _get_zip_runfiles_path("__init__.py", workspace_name, legacy_external_runfiles), ), ) - for path in runfiles.empty_filenames.to_list(): - manifest.add("{}=".format(_get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles))) + + def map_empty_filenames(path): + return "{}=".format(_get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles)) + + manifest.add_all(runfiles.empty_filenames, map_each = map_empty_filenames, allow_closure = True) def map_zip_runfiles(file): - if file != original_nonzip_executable and file != output: - return "{}={}".format( - _get_zip_runfiles_path(file.short_path, workspace_name, legacy_external_runfiles), - file.path, - ) - else: - return None + return "{}={}".format( + _get_zip_runfiles_path(file.short_path, workspace_name, legacy_external_runfiles), + file.path, + ) manifest.add_all(runfiles.files, map_each = map_zip_runfiles, allow_closure = True) @@ -850,13 +849,6 @@ def _create_zip_file(ctx, *, output, original_nonzip_executable, zip_main, runfi )) inputs.append(zip_repo_mapping_manifest) - for artifact in runfiles.files.to_list(): - # Don't include the original executable because it isn't used by the - # zip file, so no need to build it for the action. - # Don't include the zipfile itself because it's an output. - if artifact != original_nonzip_executable and artifact != output: - inputs.append(artifact) - zip_cli_args = ctx.actions.args() zip_cli_args.add("cC") zip_cli_args.add(output) @@ -864,7 +856,7 @@ def _create_zip_file(ctx, *, output, original_nonzip_executable, zip_main, runfi ctx.actions.run( executable = ctx.executable._zipper, arguments = [zip_cli_args, manifest], - inputs = depset(inputs), + inputs = depset(inputs, transitive = [runfiles.files]), outputs = [output], use_default_shell_env = True, mnemonic = "PythonZipper", @@ -873,13 +865,13 @@ def _create_zip_file(ctx, *, output, original_nonzip_executable, zip_main, runfi def _get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles): if legacy_external_runfiles and path.startswith(_EXTERNAL_PATH_PREFIX): - zip_runfiles_path = paths.relativize(path, _EXTERNAL_PATH_PREFIX) + zip_runfiles_path = path.removeprefix(_EXTERNAL_PATH_PREFIX) else: # NOTE: External runfiles (artifacts in other repos) will have a leading # path component of "../" so that they refer outside the main workspace - # directory and into the runfiles root. By normalizing, we simplify e.g. + # directory and into the runfiles root. So we simplify it, e.g. # "workspace/../foo/bar" to simply "foo/bar". - zip_runfiles_path = paths.normalize("{}/{}".format(workspace_name, path)) + zip_runfiles_path = "{}/{}".format(workspace_name, path) if not path.startswith("../") else path[3:] return "{}/{}".format(_ZIP_RUNFILES_DIRECTORY_NAME, zip_runfiles_path) def _create_executable_zip_file( From cf4ddb901a3e32321d0739624c7db915f3f4feb2 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 9 Nov 2025 15:57:09 -0800 Subject: [PATCH 2/4] use + instead of format(); pass empty_filenames lambda --- python/private/py_executable.bzl | 34 +++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 03dceb7cb5..97207a2a8b 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -819,15 +819,25 @@ def _create_zip_file(ctx, *, output, zip_main, runfiles): ), ) - def map_empty_filenames(path): - return "{}=".format(_get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles)) - - manifest.add_all(runfiles.empty_filenames, map_each = map_empty_filenames, allow_closure = True) + def map_zip_empty_filenames(list_paths_cb): + return [ + _get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles) + "=" + for path in list_paths_cb().to_list() + ] + + manifest.add_all( + # NOTE: Accessing runfiles.empty_filenames implicitly flattens the runfiles. + # Smuggle a lambda in via a list to defer that flattening. + [lambda: runfiles.empty_filenames], + map_each = map_zip_empty_filenames, + allow_closure = True + ) def map_zip_runfiles(file): - return "{}={}".format( - _get_zip_runfiles_path(file.short_path, workspace_name, legacy_external_runfiles), - file.path, + return ( + # NOTE: Use "+" for performance + _get_zip_runfiles_path(file.short_path, workspace_name, legacy_external_runfiles) + + "=" + file.path ) manifest.add_all(runfiles.files, map_each = map_zip_runfiles, allow_closure = True) @@ -864,6 +874,7 @@ def _create_zip_file(ctx, *, output, zip_main, runfiles): ) def _get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles): + maybe_workspace = "" if legacy_external_runfiles and path.startswith(_EXTERNAL_PATH_PREFIX): zip_runfiles_path = path.removeprefix(_EXTERNAL_PATH_PREFIX) else: @@ -871,8 +882,13 @@ def _get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles): # path component of "../" so that they refer outside the main workspace # directory and into the runfiles root. So we simplify it, e.g. # "workspace/../foo/bar" to simply "foo/bar". - zip_runfiles_path = "{}/{}".format(workspace_name, path) if not path.startswith("../") else path[3:] - return "{}/{}".format(_ZIP_RUNFILES_DIRECTORY_NAME, zip_runfiles_path) + if path.startswith("../"): + zip_runfiles_path = path[3:] + else: + zip_runfiles_path = path + maybe_workspace = workspace_name + "/" + # NOTE: Use "+" for performance + return _ZIP_RUNFILES_DIRECTORY_NAME + "/" + maybe_workspace + zip_runfiles_path def _create_executable_zip_file( ctx, From 6ec96b71929bd60edd5ca5e60a52db82cef57c26 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 9 Nov 2025 16:10:23 -0800 Subject: [PATCH 3/4] format code --- python/private/py_executable.bzl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 97207a2a8b..d5c0fa5388 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -830,14 +830,14 @@ def _create_zip_file(ctx, *, output, zip_main, runfiles): # Smuggle a lambda in via a list to defer that flattening. [lambda: runfiles.empty_filenames], map_each = map_zip_empty_filenames, - allow_closure = True + allow_closure = True, ) def map_zip_runfiles(file): return ( # NOTE: Use "+" for performance - _get_zip_runfiles_path(file.short_path, workspace_name, legacy_external_runfiles) - + "=" + file.path + _get_zip_runfiles_path(file.short_path, workspace_name, legacy_external_runfiles) + + "=" + file.path ) manifest.add_all(runfiles.files, map_each = map_zip_runfiles, allow_closure = True) @@ -887,6 +887,7 @@ def _get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles): else: zip_runfiles_path = path maybe_workspace = workspace_name + "/" + # NOTE: Use "+" for performance return _ZIP_RUNFILES_DIRECTORY_NAME + "/" + maybe_workspace + zip_runfiles_path From 76f1ee847b705eba632d2fea640d70cb5559cf1a Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 9 Nov 2025 16:16:21 -0800 Subject: [PATCH 4/4] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c472c08d8..d7403b4f1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,8 @@ END_UNRELEASED_TEMPLATE ([#3085](https://github.com/bazel-contrib/rules_python/issues/3085)). * (toolchains) local toolchains now tell the `sys.abiflags` value of the underlying runtime. +* (performance) 90% reduction in py_binary/py_test analysis phase cost. + ([#3381](https://github.com/bazel-contrib/rules_python/pull/3381)). {#v0-0-0-added} ### Added