Skip to content

Commit e25a655

Browse files
redsun82redsun82
committed
Swift: fix Linux compatibility problem and fetch resource dir from toolchain
This fetches the resource directory directly from the released toolchains, allowing us to stop prebuilding and assembling them. Moreover insertion of our resource directory is moved to the lua tracing configuration (solving a `TODO`) and enhanced. Now all options that start with the original resource directory (either explicit or implied) are redirected to our resource directory. This solves a problem where `-I <original resource dir>/some/path` was passed to the extractor and did not work. This works around the 5.9 linux compatibility problem by including the `PackageDescription` swift modules in the in-dist toolchain. Copying the toolchain and fixing the `-I` flag was not enough as for some reason compilation of `PackageDescription.swiftinterface` was causing a crash in the SIL pass. We work around that by pre-compiling those modules during the build and including `.swiftmodule` files in the resource directory. TODO (apart from testing): * the libraries included in the macOS toolchain are now fat (they were intel only before), occupying more space. We should see if we need to trim them down. * there might be other swiftinterface files causing problems on linux lurking around... * if we go with this, we can simplify and trim down the prebuilding we do leaving out the resource directory.
1 parent b9dfeb3 commit e25a655

File tree

7 files changed

+236
-82
lines changed

7 files changed

+236
-82
lines changed

‎swift/extractor/main.cpp‎

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -177,23 +177,14 @@ codeql::TrapDomain invocationTrapDomain(codeql::SwiftExtractorState& state) {
177177
return std::move(maybeDomain.value());
178178
}
179179

180-
codeql::SwiftExtractorConfiguration configure(int argc,
181-
char** argv,
182-
const std::string& resourceDir) {
180+
codeql::SwiftExtractorConfiguration configure(int argc, char** argv) {
183181
codeql::SwiftExtractorConfiguration configuration{};
184182
configuration.trapDir = getenv_or("CODEQL_EXTRACTOR_SWIFT_TRAP_DIR", "extractor-out/trap/swift");
185183
configuration.sourceArchiveDir =
186184
getenv_or("CODEQL_EXTRACTOR_SWIFT_SOURCE_ARCHIVE_DIR", "extractor-out/src");
187185
configuration.scratchDir =
188186
getenv_or("CODEQL_EXTRACTOR_SWIFT_SCRATCH_DIR", "extractor-out/working");
189187
configuration.frontendOptions.assign(argv + 1, argv + argc);
190-
// TODO: Should be moved to the tracer config
191-
for (int i = 0; i < argc - 1; i++) {
192-
if (std::string("-resource-dir") == configuration.frontendOptions[i]) {
193-
configuration.frontendOptions[i + 1] = resourceDir.c_str();
194-
break;
195-
}
196-
}
197188
return configuration;
198189
}
199190

@@ -226,9 +217,7 @@ int main(int argc, char** argv, char** envp) {
226217
INITIALIZE_LLVM();
227218
initializeSwiftModules();
228219

229-
std::string resourceDir = getenv_or("CODEQL_EXTRACTOR_SWIFT_ROOT", ".") + "/resource-dir/" +
230-
getenv_or("CODEQL_PLATFORM", ".");
231-
const auto configuration = configure(argc, argv, resourceDir);
220+
const auto configuration = configure(argc, argv);
232221
LOG_INFO("calling extractor with arguments \"{}\"", argDump(argc, argv));
233222
LOG_DEBUG("environment:\n{}\n", envDump(envp));
234223

‎swift/third_party/BUILD.swift-llvm-support.bazel‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,3 @@ pkg_files(
4141
strip_prefix = strip_prefix.from_pkg(),
4242
visibility = ["//visibility:public"],
4343
)
44-
45-
pkg_files(
46-
name = "swift-resource-dir",
47-
srcs = glob([
48-
"toolchain/lib/swift/**/*",
49-
]),
50-
strip_prefix = "toolchain/lib/swift",
51-
visibility = ["//visibility:public"],
52-
)

‎swift/third_party/BUILD.swift-toolchain-linux.bazel‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
load("@rules_pkg//:mappings.bzl", "pkg_filegroup", "pkg_files")
2+
3+
_strip_prefix = "usr/lib/swift"
4+
5+
_pm_interfaces_glob = "usr/lib/swift/pm/**/*.swiftinterface"
6+
7+
pkg_files(
8+
name = "resource-dir-original",
9+
srcs = glob(
10+
["usr/lib/swift/**/*"],
11+
exclude = [_pm_interfaces_glob],
12+
),
13+
strip_prefix = _strip_prefix,
14+
)
15+
16+
# pre-compile package manager interface files as compiling them hits a mysterious crash during SIL
17+
_pm_interface_files = [
18+
(
19+
f[len(_strip_prefix) + 1:f.rindex("/")],
20+
f[f.rindex("/") + 1:],
21+
f[f.rindex("/") + 1:f.rindex(".")] + ".swiftmodule",
22+
)
23+
for f in glob([_pm_interfaces_glob])
24+
]
25+
26+
[
27+
[
28+
genrule(
29+
name = "compile-%s" % interface,
30+
srcs = ["%s/%s/%s" % (_strip_prefix, dir, interface)],
31+
outs = [module],
32+
cmd = "$(location usr/bin/swift-frontend) -compile-module-from-interface $< -o $@ -I $$(dirname $<)",
33+
local = True,
34+
tools = ["usr/bin/swift-frontend"],
35+
),
36+
pkg_files(
37+
name = "pkg-%s" % module,
38+
srcs = [":compile-%s" % interface],
39+
prefix = dir,
40+
),
41+
]
42+
for dir, interface, module in _pm_interface_files
43+
]
44+
45+
pkg_filegroup(
46+
name = "resource-dir",
47+
srcs = [":resource-dir-original"] + [":pkg-%s" % module for _, _, module in _pm_interface_files],
48+
visibility = ["//visibility:public"],
49+
)

‎swift/third_party/BUILD.swift-toolchain-macos.bazel‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
load("@rules_pkg//:mappings.bzl", "pkg_files")
2+
3+
_strip_prefix = "usr/lib/swift"
4+
5+
pkg_files(
6+
name = "resource-dir",
7+
srcs = glob(
8+
["usr/lib/swift/**/*"],
9+
),
10+
strip_prefix = _strip_prefix,
11+
visibility = ["//visibility:public"],
12+
)

‎swift/third_party/load.bzl‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,92 @@ _swift_arch_map = {
1616
"macOS-X64": "darwin_x86_64",
1717
}
1818

19+
_swift_version = _swift_prebuilt_version.rpartition(".")[0]
20+
21+
_toolchain_info = {
22+
"linux": struct(
23+
platform = "ubuntu2204",
24+
suffix = "ubuntu22.04",
25+
extension = "tar.gz",
26+
sha = "bca015e9d727ca39385d7e5b5399f46302d54a02218d40d1c3063662ffc6b42f",
27+
),
28+
"macos": struct(
29+
platform = "xcode",
30+
suffix = "osx",
31+
extension = "pkg",
32+
sha = "3cf7a4b2f3efcfcb4fef42b6588a7b1c54f7b0f2d0a479f41c3e1620b045f48e",
33+
),
34+
}
35+
36+
def _get_toolchain_url(info):
37+
return "https://download.swift.org/%s/%s/%s/%s-%s.%s" % (
38+
_swift_version.lower(),
39+
info.platform,
40+
_swift_version,
41+
_swift_version,
42+
info.suffix,
43+
info.extension,
44+
)
45+
46+
def _toolchains(workspace_name):
47+
rules = {
48+
"tar.gz": http_archive,
49+
"pkg": _pkg_archive,
50+
}
51+
for arch, info in _toolchain_info.items():
52+
rule = rules[info.extension]
53+
rule(
54+
name = "swift_toolchain_%s" % arch,
55+
url = _get_toolchain_url(info),
56+
sha256 = info.sha,
57+
build_file = _build(workspace_name, "swift-toolchain-%s" % arch),
58+
strip_prefix = "%s-%s" % (_swift_version, info.suffix),
59+
)
60+
61+
def _run(repository_ctx, message, cmd, working_directory = "."):
62+
repository_ctx.report_progress(message)
63+
res = repository_ctx.execute(
64+
["bash", "-c", cmd],
65+
working_directory = working_directory,
66+
)
67+
if res.return_code != 0:
68+
fail(message)
69+
70+
def _pkg_archive_impl(repository_ctx):
71+
archive = "file.pkg"
72+
url = repository_ctx.attr.url
73+
dir = "%s-package.pkg" % repository_ctx.attr.strip_prefix
74+
repository_ctx.report_progress("downloading %s" % url)
75+
res = repository_ctx.download(
76+
url,
77+
output = archive,
78+
sha256 = repository_ctx.attr.sha256,
79+
)
80+
if not repository_ctx.attr.sha256:
81+
print("please set sha256 = %s" % repr(res.sha256))
82+
_run(repository_ctx, "extracting %s" % dir, "xar -xf %s" % archive)
83+
repository_ctx.delete(archive)
84+
_run(
85+
repository_ctx,
86+
"extracting Payload from %s" % dir,
87+
"cat %s/Payload | gunzip -dc | cpio -i" % dir,
88+
)
89+
repository_ctx.delete(dir)
90+
repository_ctx.symlink(repository_ctx.attr.build_file, "BUILD")
91+
repository_ctx.file("WORKSPACE")
92+
93+
_pkg_archive = repository_rule(
94+
implementation = _pkg_archive_impl,
95+
attrs = {
96+
"url": attr.string(mandatory = True),
97+
"sha256": attr.string(),
98+
"strip_prefix": attr.string(),
99+
"build_file": attr.label(mandatory = True),
100+
},
101+
)
102+
103+
# TODO apply the same mechanism to the macOS toolchain (needs some work as the toolchain is a pkg archive
104+
19105
def _github_archive(*, name, repository, commit, build_file = None, sha256 = None):
20106
github_name = repository[repository.index("/") + 1:]
21107
maybe(
@@ -53,6 +139,8 @@ def load_dependencies(workspace_name):
53139
],
54140
)
55141

142+
_toolchains(workspace_name)
143+
56144
_github_archive(
57145
name = "picosha2",
58146
build_file = _build(workspace_name, "picosha2"),

‎swift/third_party/swift-llvm-support/BUILD.bazel‎

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,25 @@
11
package(default_visibility = ["//swift:__subpackages__"])
22

3-
#TODO we will be introducing universal binaries at a later stage, when we have both architectures prebuilt for macOS
4-
# for the moment, we make arm build an x86_64 binary
5-
_arch_override = {
6-
"darwin_arm64": "darwin_x86_64",
7-
}
3+
alias(
4+
name = "swift-llvm-support",
5+
actual = select({
6+
"@bazel_tools//src/conditions:linux": "@swift_prebuilt_linux//:swift-llvm-support",
7+
"@bazel_tools//src/conditions:darwin": "@swift_prebuilt_darwin_x86_64//:swift-llvm-support",
8+
}),
9+
)
810

9-
[
10-
alias(
11-
name = name,
12-
actual = select({
13-
"@bazel_tools//src/conditions:%s" % arch: "@swift_prebuilt_%s//:%s" % (
14-
_arch_override.get(arch, arch),
15-
name,
16-
)
17-
for arch in ("linux", "darwin_x86_64", "darwin_arm64")
18-
}),
19-
)
20-
for name in ("swift-llvm-support", "swift-test-sdk", "swift-resource-dir")
21-
]
11+
alias(
12+
name = "swift-test-sdk",
13+
actual = select({
14+
"@bazel_tools//src/conditions:linux": "@swift_prebuilt_linux//:swift-test-sdk",
15+
"@bazel_tools//src/conditions:darwin": "@swift_prebuilt_darwin_x86_64//:swift-test-sdk",
16+
}),
17+
)
18+
19+
alias(
20+
name = "swift-resource-dir",
21+
actual = select({
22+
"@bazel_tools//src/conditions:linux": "@swift_toolchain_linux//:resource-dir",
23+
"@bazel_tools//src/conditions:darwin": "@swift_toolchain_macos//:resource-dir",
24+
}),
25+
)

0 commit comments

Comments
 (0)