Add _MatchingEngine and _StringProcessing modules.

rxwei · rxwei · commit 65bffd7ad7cc · 2021-11-19T09:27:33.000-08:00
These modules are part of the experimental declarative string processing feature. If accepted to the Standard Library, _StringProcessing will be available via implicit import just like _Concurrency, though _MatchingEngine will still be hidden as an implementation detail.

`_MatchingEngine` will contain the general-purpose pattern matching engine ISA, bytecode, and executor. `_StringProcessing` will contain regular expression and pattern matching APIs whose implementation depends on the matching engine..

Also consolidates frontend flag `-enable-experimental-regex` as `-enable-experimental-string-processing`.

Resolves rdar://85478647.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -445,6 +445,10 @@ option(SWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED
   "Enable experimental distributed actors and functions"
   FALSE)
 
+option(SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING
+  "Enable experimental string processing"
+  FALSE)
+
 option(SWIFT_ENABLE_DISPATCH
   "Enable use of libdispatch"
   TRUE)
@@ -997,6 +1001,7 @@ if(SWIFT_BUILD_STDLIB OR SWIFT_BUILD_SDK_OVERLAY)
   message(STATUS "Differentiable Programming Support: ${SWIFT_ENABLE_EXPERIMENTAL_DIFFERENTIABLE_PROGRAMMING}")
   message(STATUS "Concurrency Support: ${SWIFT_ENABLE_EXPERIMENTAL_CONCURRENCY}")
   message(STATUS "Distributed Support: ${SWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED}")
+  message(STATUS "String Processing Support: ${SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING}")
   message(STATUS "")
 else()
   message(STATUS "Not building Swift standard library, SDK overlays, and runtime")
diff --git a/include/swift/Basic/LangOptions.h b/include/swift/Basic/LangOptions.h
@@ -145,7 +145,7 @@ namespace swift {
     version::Version PackageDescriptionVersion;
 
     /// Enable experimental string processing
-    bool EnableExperimentalRegex = false;
+    bool EnableExperimentalStringProcessing = false;
 
     /// Disable API availability checking.
     bool DisableAvailabilityChecking = false;
diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td
@@ -475,8 +475,8 @@ def disable_deserialization_recovery :
   Flag<["-"], "disable-deserialization-recovery">,
   HelpText<"Don't attempt to recover from missing xrefs (etc) in swiftmodules">;
 
-def enable_experimental_regex :
-  Flag<["-"], "enable-experimental-regex">,
+def enable_experimental_string_processing :
+  Flag<["-"], "enable-experimental-string-processing">,
   HelpText<"Enable experimental string processing">;
 
 def disable_availability_checking : Flag<["-"],
diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp
@@ -473,8 +473,8 @@ static bool ParseLangArgs(LangOptions &Opts, ArgList &Args,
   }
 
   // Experimental string processing
-  Opts.EnableExperimentalRegex |=
-      Args.hasArg(OPT_enable_experimental_regex);
+  Opts.EnableExperimentalStringProcessing |=
+      Args.hasArg(OPT_enable_experimental_string_processing);
 
   Opts.DisableAvailabilityChecking |=
       Args.hasArg(OPT_disable_availability_checking);
diff --git a/lib/Parse/Lexer.cpp b/lib/Parse/Lexer.cpp
@@ -1807,7 +1807,7 @@ void Lexer::diagnoseSingleQuoteStringLiteral(const char *TokStart,
   auto startLoc = Lexer::getSourceLoc(TokStart);
   auto endLoc = Lexer::getSourceLoc(TokEnd);
 
-  if (LangOpts.EnableExperimentalRegex) {
+  if (LangOpts.EnableExperimentalStringProcessing) {
     if (parseRegexStrawperson) {
       auto copy = std::string(TokStart, TokEnd-TokStart);
       auto msg = parseRegexStrawperson(copy.c_str());
diff --git a/stdlib/cmake/modules/SwiftSource.cmake b/stdlib/cmake/modules/SwiftSource.cmake
@@ -285,6 +285,10 @@ function(_add_target_variant_swift_compile_flags
     list(APPEND result "-D" "SWIFT_ENABLE_EXPERIMENTAL_DIFFERENTIABLE_PROGRAMMING")
   endif()
 
+  if(SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING)
+    list(APPEND result "-D" "SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING")
+  endif()
+
   if(SWIFT_STDLIB_OS_VERSIONING)
     list(APPEND result "-D" "SWIFT_RUNTIME_OS_VERSIONING")
   endif()
diff --git a/stdlib/public/CMakeLists.txt b/stdlib/public/CMakeLists.txt
@@ -115,6 +115,11 @@ if(SWIFT_BUILD_STDLIB)
   if(SWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED)
     add_subdirectory(Distributed)
   endif()
+
+  if(SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING)
+    add_subdirectory(MatchingEngine)
+    add_subdirectory(StringProcessing)
+  endif()
 endif()
 
 if(SWIFT_BUILD_STDLIB OR SWIFT_BUILD_REMOTE_MIRROR)
diff --git a/stdlib/public/MatchingEngine/CMakeLists.txt b/stdlib/public/MatchingEngine/CMakeLists.txt
@@ -0,0 +1,38 @@
+#===--- CMakeLists.txt - Pattern matching engine support library -----------===#
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2021 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+#===------------------------------------------------------------------------===#
+
+set(swift_matching_engine_link_libraries
+  swiftCore)
+
+
+add_swift_target_library(swift_MatchingEngine ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} IS_STDLIB
+  MatchingEngine.swift
+
+  SWIFT_MODULE_DEPENDS_LINUX Glibc
+  SWIFT_MODULE_DEPENDS_FREEBSD Glibc
+  SWIFT_MODULE_DEPENDS_OPENBSD Glibc
+  SWIFT_MODULE_DEPENDS_CYGWIN Glibc
+  SWIFT_MODULE_DEPENDS_HAIKU Glibc
+  SWIFT_MODULE_DEPENDS_WINDOWS CRT
+
+  LINK_LIBRARIES ${swift_matching_engine_link_libraries}
+
+  C_COMPILE_FLAGS
+    -Dswift_MatchingEngine_EXPORTS
+  SWIFT_COMPILE_FLAGS
+    ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS}
+    -parse-stdlib
+    -Xfrontend -enable-experimental-string-processing
+  LINK_FLAGS "${SWIFT_RUNTIME_SWIFT_LINK_FLAGS}"
+
+  INSTALL_IN_COMPONENT stdlib
+)
diff --git a/stdlib/public/MatchingEngine/MatchingEngine.swift b/stdlib/public/MatchingEngine/MatchingEngine.swift
@@ -0,0 +1,7 @@
+import Swift
+
+public struct DummyMatchingEngine {
+  public init() {
+    fatalError("Unimplemented")
+  }
+}
diff --git a/stdlib/public/StringProcessing/CMakeLists.txt b/stdlib/public/StringProcessing/CMakeLists.txt
@@ -0,0 +1,40 @@
+#===--- CMakeLists.txt - String processing support library -----------------===#
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2021 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+#===------------------------------------------------------------------------===#
+
+set(swift_string_processing_link_libraries
+  swiftCore
+  swift_MatchingEngine)
+
+
+add_swift_target_library(swift_StringProcessing ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} IS_STDLIB
+  Regex.swift
+
+  SWIFT_MODULE_DEPENDS_LINUX Glibc
+  SWIFT_MODULE_DEPENDS_FREEBSD Glibc
+  SWIFT_MODULE_DEPENDS_OPENBSD Glibc
+  SWIFT_MODULE_DEPENDS_CYGWIN Glibc
+  SWIFT_MODULE_DEPENDS_HAIKU Glibc
+  SWIFT_MODULE_DEPENDS_WINDOWS CRT
+
+  LINK_LIBRARIES ${swift_string_processing_link_libraries}
+
+  C_COMPILE_FLAGS
+    -Dswift_StringProcessing_EXPORTS
+  SWIFT_COMPILE_FLAGS
+    ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS}
+    -parse-stdlib
+    -Xfrontend -enable-experimental-string-processing
+  LINK_FLAGS "${SWIFT_RUNTIME_SWIFT_LINK_FLAGS}"
+
+  SWIFT_MODULE_DEPENDS _MatchingEngine
+  INSTALL_IN_COMPONENT stdlib
+)
diff --git a/stdlib/public/StringProcessing/Regex.swift b/stdlib/public/StringProcessing/Regex.swift
@@ -0,0 +1,8 @@
+import Swift
+import _MatchingEngine
+
+public struct DummyRegex<Match> {
+  public init() {
+    fatalError("Unimplemented")
+  }
+}
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -193,6 +193,7 @@ normalize_boolean_spelling(SWIFT_ENABLE_EXPERIMENTAL_DIFFERENTIABLE_PROGRAMMING)
 normalize_boolean_spelling(SWIFT_ENABLE_EXPERIMENTAL_CONCURRENCY)
 normalize_boolean_spelling(SWIFT_BACK_DEPLOY_CONCURRENCY)
 normalize_boolean_spelling(SWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED)
+normalize_boolean_spelling(SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING)
 normalize_boolean_spelling(SWIFT_ENABLE_MACCATALYST)
 normalize_boolean_spelling(SWIFT_RUN_TESTS_WITH_HOST_COMPILER)
 normalize_boolean_spelling(SWIFT_RUNTIME_ENABLE_LEAK_CHECKER)
@@ -385,6 +386,10 @@ foreach(SDK ${SWIFT_SDKS})
           list(APPEND LIT_ARGS "--param" "distributed")
         endif()
 
+        if(SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING)
+          list(APPEND LIT_ARGS "--param" "string_processing")
+        endif()
+
         foreach(test_subset ${TEST_SUBSETS})
           set(directories)
           set(dependencies ${test_dependencies})
diff --git a/test/ExperimentalRegex/Parse/Strawperson.swift b/test/ExperimentalRegex/Parse/Strawperson.swift
@@ -1,4 +1,4 @@
-// RUN: %target-typecheck-verify-swift -enable-experimental-regex
+// RUN: %target-typecheck-verify-swift -enable-experimental-string-processing
 // REQUIRES: libswift
 
 var s = 'abc'
diff --git a/test/StringProcessing/lit.local.cfg b/test/StringProcessing/lit.local.cfg
@@ -0,0 +1,2 @@
+if 'string_processing' not in config.available_features:
+    config.unsupported = False
diff --git a/test/StringProcessing/string_processing_dummy.swift b/test/StringProcessing/string_processing_dummy.swift
@@ -0,0 +1,11 @@
+// RUN: %target-swift-frontend -enable-experimental-string-processing -parse-as-library -emit-sil -verify %s
+
+// REQUIRES: string_processing
+
+import _MatchingEngine
+import _StringProcessing
+
+func foo() {
+  _ = DummyMatchingEngine()
+  _ = DummyRegex<Substring>()
+}
diff --git a/test/lit.cfg b/test/lit.cfg
@@ -445,6 +445,10 @@ differentiable_programming = lit_config.params.get('differentiable_programming',
 if differentiable_programming is not None:
     config.available_features.add('differentiable_programming')
 
+string_processing = lit_config.params.get('string_processing', None)
+if string_processing is not None:
+    config.available_features.add('string_processing')
+
 config.available_features.add('lld_lto')
 
 test_options = os.environ.get('SWIFT_TEST_OPTIONS')
@@ -1711,6 +1715,13 @@ config.substitutions.append(('%distributed_module', distributed_module))
 config.substitutions.append(('%/distributed_module',
                              '/'.join(os.path.normpath(distributed_module).split(os.sep))))
 
+# Add 'string_processing_module' as the path to the _StringProcessing .swiftmodule file
+string_processing_module = os.path.join(stdlib_dir, "_StringProcessing.swiftmodule",
+                                   target_specific_module_triple + ".swiftmodule")
+config.substitutions.append(('%string_processing_module', string_processing_module))
+config.substitutions.append(('%/string_processing_module',
+                             '/'.join(os.path.normpath(string_processing_module).split(os.sep))))
+
 # Different OS's require different prefixes for the environment variables to be
 # propagated to the calling contexts.
 # In order to make tests OS-agnostic, names of environment variables should be
diff --git a/test/lit.site.cfg.in b/test/lit.site.cfg.in
@@ -138,6 +138,8 @@ if "@SWIFT_BACK_DEPLOY_CONCURRENCY@" == "TRUE":
     config.available_features.add('back_deploy_concurrency')
 if "@SWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED@" == "TRUE":
     config.available_features.add('distributed')
+if "@SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING@" == "TRUE":
+    config.available_features.add('string_processing')
 
 # Let the main config do the real work.
 if config.test_exec_root is None:
diff --git a/tools/swift-refactor/swift-refactor.cpp b/tools/swift-refactor/swift-refactor.cpp
@@ -128,6 +128,10 @@ static llvm::cl::opt<bool> EnableExperimentalConcurrency(
     "enable-experimental-concurrency",
     llvm::cl::desc("Whether to enable experimental concurrency or not"));
 
+static llvm::cl::opt<bool> EnableExperimentalStringProcessing(
+    "enable-experimental-string-processing",
+    llvm::cl::desc("Whether to enable experimental string processing or not"));
+
 static llvm::cl::opt<std::string>
     SDK("sdk", llvm::cl::desc("Path to the SDK to build against"));
 
@@ -311,6 +315,9 @@ int main(int argc, char *argv[]) {
   if (options::EnableExperimentalConcurrency)
     Invocation.getLangOptions().EnableExperimentalConcurrency = true;
 
+  if (options::EnableExperimentalStringProcessing)
+    Invocation.getLangOptions().EnableExperimentalStringProcessing = true;
+
   for (auto FileName : options::InputFilenames)
     Invocation.getFrontendOptions().InputsAndOutputs.addInputFile(FileName);
   Invocation.setModuleName(options::ModuleName);
diff --git a/utils/build_swift/build_swift/driver_arguments.py b/utils/build_swift/build_swift/driver_arguments.py
@@ -1188,8 +1188,7 @@ def create_argument_parser():
 
     option('--enable-experimental-differentiable-programming', toggle_true,
            default=True,
-           help='Enable experimental Swift differentiable programming language'
-                ' features.')
+           help='Enable experimental Swift differentiable programming.')
 
     option('--enable-experimental-concurrency', toggle_true,
            default=True,
@@ -1199,6 +1198,10 @@ def create_argument_parser():
            default=True,
            help='Enable experimental Swift distributed actors.')
 
+    option('--enable-experimental-string-processing', toggle_true,
+           default=True,
+           help='Enable experimental Swift string processing.')
+
     # -------------------------------------------------------------------------
     in_group('Unsupported options')
 
diff --git a/utils/build_swift/tests/expected_options.py b/utils/build_swift/tests/expected_options.py
@@ -149,6 +149,7 @@
     'enable_experimental_differentiable_programming': True,
     'enable_experimental_concurrency': True,
     'enable_experimental_distributed': True,
+    'enable_experimental_string_processing': True,
     'enable_lsan': False,
     'enable_sanitize_coverage': False,
     'disable_guaranteed_normal_arguments': False,
@@ -540,6 +541,7 @@ class BuildScriptImplOption(_BaseOption):
     EnableOption('--enable-experimental-differentiable-programming'),
     EnableOption('--enable-experimental-concurrency'),
     EnableOption('--enable-experimental-distributed'),
+    EnableOption('--enable-experimental-string-processing'),
     EnableOption('--enable-lsan'),
     EnableOption('--enable-sanitize-coverage'),
     EnableOption('--enable-tsan'),
diff --git a/utils/refactor-check-compiles.py b/utils/refactor-check-compiles.py
@@ -33,6 +33,7 @@ def parse_args():
          - swift-refactor
          - temp-dir
          - enable-experimental-concurrency (sent to both)
+         - enable-experimental-string-processing (sent to both)
          - I (sent to both)
          - sdk (sent to both)
          - target (sent to both)
@@ -74,6 +75,14 @@ def parse_args():
         swift-frontend
         '''
     )
+    parser.add_argument(
+        '-enable-experimental-string-processing',
+        action='store_true',
+        help='''
+        Whether to enable experimental string processing in both swift-refactor
+        and swift-frontend
+        '''
+    )
     parser.add_argument(
         '-I',
         action='append',
@@ -103,6 +112,8 @@ def main():
     extra_both_args = []
     if args.enable_experimental_concurrency:
         extra_both_args.append('-enable-experimental-concurrency')
+    if args.enable_experimental_string_processing:
+        extra_both_args.append('-enable-experimental-string-processing')
     if args.I:
         for path in args.I:
             extra_both_args += ['-I', path]
diff --git a/utils/swift-api-dump.py b/utils/swift-api-dump.py
@@ -334,6 +334,8 @@ def main():
         extra_args = extra_args + ['-enable-experimental-concurrency']
     if args.enable_experimental_distributed:
         extra_args = extra_args + ['-enable-experimental-distributed']
+    if args.enable_experimental_string_processing:
+        extra_args = extra_args + ['-enable-experimental-string-processing']
     if args.swift_version:
         extra_args = extra_args + ['-swift-version', '%s' % args.swift_version]
 
diff --git a/utils/swift_build_support/swift_build_support/products/swift.py b/utils/swift_build_support/swift_build_support/products/swift.py
@@ -56,6 +56,9 @@ def __init__(self, args, toolchain, source_dir, build_dir):
         # Add experimental distributed flag.
         self.cmake_options.extend(self._enable_experimental_distributed)
 
+        # Add experimental string processing flag.
+        self.cmake_options.extend(self._enable_experimental_string_processing)
+
     @classmethod
     def is_build_script_impl_product(cls):
         """is_build_script_impl_product -> bool
@@ -155,6 +158,11 @@ def _enable_experimental_distributed(self):
         return [('SWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED:BOOL',
                  self.args.enable_experimental_distributed)]
 
+    @property
+    def _enable_experimental_string_processing(self):
+        return [('SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING:BOOL',
+                 self.args.enable_experimental_string_processing)]
+
     @classmethod
     def get_dependencies(cls):
         return [cmark.CMark,
diff --git a/utils/swift_build_support/tests/products/test_swift.py b/utils/swift_build_support/tests/products/test_swift.py
diff --git a/validation-test/lit.site.cfg.in b/validation-test/lit.site.cfg.in

Original file line number	Diff line number	Diff line change
`@@ -473,8 +473,8 @@ static bool ParseLangArgs(LangOptions &Opts, ArgList &Args,`
`473`	`473`	`}`
`474`	`474`
`475`	`475`	`// Experimental string processing`
`476`		`- Opts.EnableExperimentalRegex \|=`
`477`		`- Args.hasArg(OPT_enable_experimental_regex);`
	`476`	`+ Opts.EnableExperimentalStringProcessing \|=`
	`477`	`+ Args.hasArg(OPT_enable_experimental_string_processing);`
`478`	`478`
`479`	`479`	`Opts.DisableAvailabilityChecking \|=`
`480`	`480`	`Args.hasArg(OPT_disable_availability_checking);`