Merge pull request #38507 from drexin/wip-llvm-build-script

[Build] Make LLVM a build-script product

Author: drexin

utils/build-script

@@ -417,7 +417,6 @@ def apply_default_arguments(toolchain, args):
         args.extra_cmake_options.append(
             '-DSWIFT_DARWIN_MODULE_ARCHS:STRING={}'.format(
                 args.swift_darwin_module_archs))
-
     if (args.infer_cross_compile_hosts_on_darwin and
             platform.system() == "Darwin"):
         args.cross_compile_hosts = _infer_cross_compile_hosts_on_darwin(

utils/build_swift/build_swift/driver_arguments.py

@@ -519,6 +519,9 @@ def create_argument_parser():
     option('--clang-profile-instr-use', store_path,
            help='profile file to use for clang PGO')
 
+    option('--swift-profile-instr-use', store_path,
+           help='profile file to use for clang PGO while building swift')
+
     option('--llvm-max-parallel-lto-link-jobs', store_int,
            default=defaults.LLVM_MAX_PARALLEL_LTO_LINK_JOBS,
            metavar='COUNT',
@@ -650,6 +653,9 @@ def create_argument_parser():
     option(['--back-deploy-concurrency'], toggle_true('build_backdeployconcurrency'),
            help='build back-deployment support for concurrency')
 
+    option('--install-llvm', toggle_true,
+           help='install llvm')
+
     option(['--install-back-deploy-concurrency'],
            toggle_true('install_backdeployconcurrency'),
            help='install back-deployment support libraries for concurrency')
@@ -762,6 +768,11 @@ def create_argument_parser():
            default=True,
            help='skip building clang-tools-extra as part of llvm')
 
+    option('--skip-build-compiler-rt',
+           toggle_false('build_compiler_rt'),
+           default=True,
+           help='skip building compiler-rt as part of llvm')
+
     # -------------------------------------------------------------------------
     in_group('Extra actions to perform before or in addition to building')
 
@@ -1111,6 +1122,9 @@ def create_argument_parser():
                 'This can be useful to reduce build times when e.g. '
                 'tests do not need to run')
 
+    option('--build-toolchain-only', toggle_true,
+           help='only build the necessary tools to build an external toolchain')
+
     # -------------------------------------------------------------------------
     in_group('Skip testing specified targets')
 
@@ -1211,6 +1225,9 @@ def create_argument_parser():
     # -------------------------------------------------------------------------
     in_group('Build settings specific for LLVM')
 
+    option('--llvm-enable-modules', toggle_true('llvm_enable_modules'),
+           help='enable building llvm using modules')
+
     option('--llvm-targets-to-build', store,
            default='X86;ARM;AArch64;PowerPC;SystemZ;Mips',
            help='LLVM target generators to build')
@@ -1230,6 +1247,9 @@ def create_argument_parser():
                 'Can be called multiple times '
                 'to add multiple such options.')
 
+    option('--no-llvm-include-tests', toggle_false('llvm_include_tests'),
+           help='do not generate testing targets for LLVM')
+
     # -------------------------------------------------------------------------
     in_group('Build settings for Android')
 
@@ -1294,6 +1314,9 @@ def create_argument_parser():
            help='skip building cmark')
     option('--skip-build-llvm', toggle_false('build_llvm'),
            help='skip building llvm')
+    option('--build-llvm', toggle_true('_build_llvm'),
+           default=True,
+           help='build llvm and clang')
     option('--skip-build-swift', toggle_false('build_swift'),
            help='skip building swift')
     option('--skip-build-libxml2', toggle_false('build_libxml2'),

utils/build_swift/tests/expected_options.py

@@ -51,12 +51,14 @@
     'build_backdeployconcurrency': False,
     'build_benchmarks': True,
     'build_clang_tools_extra': True,
+    'build_compiler_rt': True,
     'build_cygwin': True,
     'build_external_benchmarks': False,
     'build_foundation': False,
     'build_cmark': True,
     'build_swift': True,
     'build_llvm': True,
+    '_build_llvm': True,
     'build_freebsd': True,
     'build_ios': True,
     'build_ios_device': False,
@@ -100,6 +102,7 @@
     'test_indexstoredb_sanitize_all': False,
     'test_sourcekitlsp_sanitize_all': False,
     'build_sourcekitlsp': False,
+    'install_llvm': False,
     'install_swiftpm': False,
     'install_swiftsyntax': False,
     'install_swift_driver': False,
@@ -111,6 +114,7 @@
     'install_skstresstester': False,
     'install_swiftevolve': False,
     'build_toolchainbenchmarks': False,
+    'build_toolchain_only': False,
     'build_tvos': True,
     'build_tvos_device': False,
     'build_tvos_simulator': False,
@@ -197,6 +201,8 @@
     'lldb_build_with_xcode': '0',
     'llvm_assertions': True,
     'llvm_build_variant': 'Debug',
+    'llvm_enable_modules': False,
+    'llvm_include_tests': True,
     'llvm_ninja_targets': [],
     'llvm_ninja_targets_for_cross_compile_hosts': [],
     'llvm_max_parallel_lto_link_jobs':
@@ -226,6 +232,7 @@
     'swift_darwin_module_archs': None,
     'swift_darwin_supported_archs': None,
     'swift_freestanding_is_darwin': False,
+    'swift_profile_instr_use': None,
     'swift_stdlib_assertions': True,
     'swift_stdlib_build_variant': 'Debug',
     'swift_tools_ld64_lto_codegen_only_for_supporting_targets': False,
@@ -496,6 +503,8 @@ class BuildScriptImplOption(_BaseOption):
     SetOption('--skip-test-early-swift-driver',
               dest='test_early_swift_driver', value=False),
 
+    SetFalseOption('--no-llvm-include-tests', dest='llvm_include_tests'),
+
     SetTrueOption('--back-deploy-concurrency', dest='build_backdeployconcurrency'),
     SetTrueOption('--install-back-deploy-concurrency',
                   dest='install_backdeployconcurrency'),
@@ -548,6 +557,7 @@ class BuildScriptImplOption(_BaseOption):
     EnableOption('--build-swift-static-stdlib'),
     EnableOption('--build-swift-stdlib-unittest-extra'),
     EnableOption('--build-swift-stdlib-static-print'),
+    EnableOption('--build-toolchain-only'),
     EnableOption('--build-swift-private-stdlib'),
     EnableOption('--build-swift-stdlib-unicode-data'),
     EnableOption('--cross-compile-append-host-target-to-destdir'),
@@ -579,6 +589,7 @@ class BuildScriptImplOption(_BaseOption):
     EnableOption('--sourcekit-lsp', dest='build_sourcekitlsp'),
     EnableOption('--test-sourcekit-lsp-sanitize-all',
                  dest='test_sourcekitlsp_sanitize_all'),
+    EnableOption('--install-llvm', dest='install_llvm'),
     EnableOption('--install-swiftsyntax', dest='install_swiftsyntax'),
     EnableOption('--swiftsyntax-verify-generated-files',
                  dest='swiftsyntax_verify_generated_files'),
@@ -607,6 +618,8 @@ class BuildScriptImplOption(_BaseOption):
     EnableOption('--xctest', dest='build_xctest'),
     EnableOption('--swift-disable-dead-stripping'),
     EnableOption('--clean-early-swift-driver', dest='clean_early_swift_driver'),
+    EnableOption('--llvm-enable-modules'),
+    EnableOption('--build-llvm', dest='_build_llvm'),
 
     DisableOption('--skip-build-cmark', dest='build_cmark'),
     DisableOption('--skip-build-llvm', dest='build_llvm'),
@@ -679,6 +692,7 @@ class BuildScriptImplOption(_BaseOption):
     DisableOption('--skip-build-libxml2', dest='build_libxml2'),
     DisableOption('--skip-build-zlib', dest='build_zlib'),
     DisableOption('--skip-build-curl', dest='build_curl'),
+    DisableOption('--skip-build-compiler-rt', dest='build_compiler_rt'),
 
     ChoicesOption('--compiler-vendor',
                   choices=['none', 'apple']),
@@ -725,6 +739,7 @@ class BuildScriptImplOption(_BaseOption):
     PathOption('--symbols-package'),
     PathOption('--cmake-c-launcher'),
     PathOption('--cmake-cxx-launcher'),
+    PathOption('--swift-profile-instr-use'),
 
     IntOption('--benchmark-num-o-iterations'),
     IntOption('--benchmark-num-onone-iterations'),

utils/swift_build_support/swift_build_support/build_script_invocation.py

@@ -534,7 +534,7 @@ def compute_host_specific_variables(self):
                 "SWIFT_TEST_TARGETS": " ".join(
                     config.swift_test_run_targets),
                 "SWIFT_FLAGS": config.swift_flags,
-                "SWIFT_TARGET_CMAKE_OPTIONS": config.cmake_options,
+                "SWIFT_TARGET_CMAKE_OPTIONS": " ".join(config.cmake_options),
             }
 
         return options
@@ -571,6 +571,12 @@ def compute_product_pipelines(self):
         builder.add_product(products.CMark,
                             is_enabled=self.args.build_cmark)
 
+        # If --skip-build-llvm is passed in, LLVM cannot be completely disabled, as
+        # Swift still needs a few LLVM targets like tblgen to be built for it to be
+        # configured. Instead, handle this in the product for now.
+        builder.add_product(products.LLVM,
+                            is_enabled=True)
+
         builder.add_product(products.LibXML2,
                             is_enabled=self.args.build_libxml2)
 
@@ -588,11 +594,6 @@ def compute_product_pipelines(self):
         # test, install like a normal build-script product.
         builder.begin_impl_pipeline(should_run_epilogue_operations=False)
 
-        # If --skip-build-llvm is passed in, LLVM cannot be completely disabled, as
-        # Swift still needs a few LLVM targets like tblgen to be built for it to be
-        # configured. Instead, handle this in build-script-impl for now.
-        builder.add_impl_product(products.LLVM,
-                                 is_enabled=True)
         builder.add_impl_product(products.LibCXX,
                                  is_enabled=self.args.build_libcxx)
         builder.add_impl_product(products.LibICU,

utils/swift_build_support/swift_build_support/cmake.py

@@ -47,6 +47,13 @@ def define(self, var, value):
                              (var, value, type(value)))
         self._options.append('-D%s=%s' % (var, value))
 
+    def undefine(self, var):
+        """Utility to undefine cmake options in this object.
+
+        opts.undefine("FOO")       # -> -UFOO
+        """
+        self._options.append('-U%s' % var)
+
     def extend(self, tuples_or_options):
         if isinstance(tuples_or_options, CMakeOptions):
             self += tuples_or_options

utils/swift_build_support/swift_build_support/host_specific_configuration.py

@@ -15,6 +15,7 @@
 from argparse import ArgumentError
 
 from . import compiler_stage
+from .cmake import CMakeOptions
 from .targets import StdlibDeploymentTarget
 
 
@@ -83,7 +84,7 @@ def __init__(self, host_target, args, stage_dependent_args=None):
         self.swift_benchmark_build_targets = []
         self.swift_benchmark_run_targets = []
         self.swift_flags = ''
-        self.cmake_options = ''
+        self.cmake_options = CMakeOptions()
         for deployment_target_name in stdlib_targets_to_configure:
             # Get the target object.
             deployment_target = StdlibDeploymentTarget.get_target_for_name(