Merge pull request #36646 from ahoppen/pr/verify-syntax-tree-build-swift

[libSyntax] Several fixes for libSyntax parsing

Author: ahoppen

include/swift/Parse/Token.h

@@ -130,8 +130,8 @@ class Token {
   }
 
   bool isContextualKeyword(StringRef ContextKW) const {
-    return is(tok::identifier) && !isEscapedIdentifier() &&
-           Text == ContextKW;
+    return isAny(tok::identifier, tok::contextual_keyword) &&
+           !isEscapedIdentifier() && Text == ContextKW;
   }
 
   /// Return true if this is a contextual keyword that could be the start of a

lib/Parse/ParseDecl.cpp

@@ -1510,7 +1510,12 @@ void Parser::parseAllAvailabilityMacroArguments() {
   AvailabilityMacroMap Map;
 
   SourceManager &SM = Context.SourceMgr;
-  const LangOptions &LangOpts = Context.LangOpts;
+  LangOptions LangOpts = Context.LangOpts;
+  // The sub-parser is not actually parsing the source file but the LangOpts
+  // AvailabilityMacros. No point creating a libSyntax tree for it. In fact, the
+  // creation of a libSyntax tree would always fail because the
+  // AvailibilityMacro is not valid Swift source code.
+  LangOpts.BuildSyntaxTree = false;
 
   for (StringRef macro: LangOpts.AvailabilityMacros) {
 
@@ -2145,7 +2150,7 @@ bool Parser::parseNewDeclAttribute(DeclAttributes &Attributes, SourceLoc AtLoc,
     bool SuppressLaterDiags = false;
     if (parseList(tok::r_paren, LeftLoc, RightLoc, false,
                   diag::originally_defined_in_missing_rparen,
-                  SyntaxKind::Unknown, [&]() -> ParserStatus {
+                  SyntaxKind::AvailabilitySpecList, [&]() -> ParserStatus {
       SWIFT_DEFER {
         if (NK != NextSegmentKind::PlatformVersion) {
           NK = (NextSegmentKind)((uint8_t)NK + (uint8_t)1);
@@ -2154,6 +2159,8 @@ bool Parser::parseNewDeclAttribute(DeclAttributes &Attributes, SourceLoc AtLoc,
       switch (NK) {
       // Parse 'module: "original_module_name"'.
       case NextSegmentKind::ModuleName: {
+        SyntaxParsingContext argumentContext(SyntaxContext,
+                                             SyntaxKind::AvailabilityLabeledArgument);
         // Parse 'module' ':'.
         if (!Tok.is(tok::identifier) || Tok.getText() != "module" ||
             !peekToken().is(tok::colon)) {
@@ -2182,6 +2189,8 @@ bool Parser::parseNewDeclAttribute(DeclAttributes &Attributes, SourceLoc AtLoc,
       }
       // Parse 'OSX 13.13'.
       case NextSegmentKind::PlatformVersion: {
+        SyntaxParsingContext argumentContext(SyntaxContext,
+                                             SyntaxKind::AvailabilityVersionRestriction);
         if ((Tok.is(tok::identifier) || Tok.is(tok::oper_binary_spaced)) &&
             (peekToken().isAny(tok::integer_literal, tok::floating_literal) ||
              peekAvailabilityMacroName())) {
@@ -4421,6 +4430,7 @@ Parser::parseDecl(ParseDeclOptions Flags,
 
     if (shouldParseExperimentalConcurrency() &&
         Tok.isContextualKeyword("actor") && peekToken().is(tok::identifier)) {
+      Tok.setKind(tok::contextual_keyword);
       DeclParsingContext.setCreateSyntax(SyntaxKind::ClassDecl);
       DeclResult = parseDeclClass(Flags, Attributes);
       break;
@@ -5007,6 +5017,9 @@ ParserStatus Parser::parseDeclItem(bool &PreviousHadSemi,
       .fixItInsert(endOfPrevious, ";");
   }
 
+  SyntaxParsingContext DeclContext(SyntaxContext,
+                                   SyntaxKind::MemberDeclListItem);
+  
   if (Tok.isAny(tok::pound_sourceLocation, tok::pound_line)) {
     auto LineDirectiveStatus = parseLineDirective(Tok.is(tok::pound_line));
     if (LineDirectiveStatus.isErrorOrHasCompletion())
@@ -5015,8 +5028,6 @@ ParserStatus Parser::parseDeclItem(bool &PreviousHadSemi,
   }
 
   ParserResult<Decl> Result;
-  SyntaxParsingContext DeclContext(SyntaxContext,
-                                   SyntaxKind::MemberDeclListItem);
   if (loadCurrentSyntaxNodeFromCache()) {
     return ParserStatus();
   }
@@ -7540,7 +7551,6 @@ ParserResult<ClassDecl> Parser::parseDeclClass(ParseDeclOptions Flags,
   // part of
   SourceLoc ClassLoc;
   if (isExplicitActorDecl) {
-    assert(Tok.is(tok::identifier) && Tok.isContextualKeyword("actor"));
     ClassLoc = consumeToken();
   } else {
     ClassLoc = consumeToken(tok::kw_class);

lib/Parse/Parser.cpp

@@ -1050,6 +1050,8 @@ static SyntaxKind getListElementKind(SyntaxKind ListKind) {
     return SyntaxKind::TupleTypeElement;
   case SyntaxKind::TuplePatternElementList:
     return SyntaxKind::TuplePatternElement;
+  case SyntaxKind::AvailabilitySpecList:
+    return SyntaxKind::AvailabilityArgument;
   default:
     return SyntaxKind::Unknown;
   }

test/Syntax/round_trip_concurrency.swift

@@ -0,0 +1,81 @@
+// RUN: %empty-directory(%t)
+// RUN: %round-trip-syntax-test --swift-syntax-test %swift-syntax-test --file %s
+
+@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *)
+actor Counter {
+  private var value = 0
+  private let scratchBuffer: UnsafeMutableBufferPointer<Int>
+
+  init(maxCount: Int) {
+    scratchBuffer = .allocate(capacity: maxCount)
+    scratchBuffer.initialize(repeating: 0)
+  }
+
+  func next() -> Int {
+    let current = value
+
+    // Make sure we haven't produced this value before
+    assert(scratchBuffer[current] == 0)
+    scratchBuffer[current] = 1
+
+    value = value + 1
+    return current
+  }
+}
+
+
+@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *)
+func worker(identity: Int, counters: [Counter], numIterations: Int) async {
+  for i in 0..<numIterations {
+    let counterIndex = Int.random(in: 0 ..< counters.count)
+    let counter = counters[counterIndex]
+    let nextValue = await counter.next()
+    print("Worker \(identity) calling counter \(counterIndex) produced \(nextValue)")
+  }
+}
+
+@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *)
+func runTest(numCounters: Int, numWorkers: Int, numIterations: Int) async {
+  // Create counter actors.
+  var counters: [Counter] = []
+  for i in 0..<numCounters {
+    counters.append(Counter(maxCount: numWorkers * numIterations))
+  }
+
+  // Create a bunch of worker threads.
+  var workers: [Task.Handle<Void, Error>] = []
+  for i in 0..<numWorkers {
+    workers.append(
+      detach { [counters] in
+        await Task.sleep(UInt64.random(in: 0..<100) * 1_000_000)
+        await worker(identity: i, counters: counters, numIterations: numIterations)
+      }
+    )
+  }
+
+  // Wait until all of the workers have finished.
+  for worker in workers {
+    try! await worker.get()
+  }
+
+  print("DONE!")
+}
+
+@available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *)
+@main struct Main {
+  static func main() async {
+    // Useful for debugging: specify counter/worker/iteration counts
+    let args = CommandLine.arguments
+    let counters = args.count >= 2 ? Int(args[1])! : 10
+    let workers = args.count >= 3 ? Int(args[2])! : 100
+    let iterations = args.count >= 4 ? Int(args[3])! : 1000
+    print("counters: \(counters), workers: \(workers), iterations: \(iterations)")
+    await runTest(numCounters: counters, numWorkers: workers, numIterations: iterations)
+  }
+}
+
+struct X3 {
+  subscript(_ i : Int) -> Int {
+    get async throws {}
+  }
+}

utils/gyb_syntax_support/AvailabilityNodes.py

@@ -70,7 +70,7 @@
                    restricted or 'swift' if the availability should be
                    restricted based on a Swift version.
                    '''),
-             Child('Version', kind='VersionTuple'),
+             Child('Version', kind='VersionTuple', is_optional=True),
          ]),
 
     # version-tuple -> integer-literal

utils/gyb_syntax_support/DeclNodes.py

@@ -533,6 +533,15 @@
                       '_read', '_modify'
                    ]),
              Child('Parameter', kind='AccessorParameter', is_optional=True),
+             Child('AsyncKeyword', kind='IdentifierToken',
+                   classification='Keyword',
+                   text_choices=['async'], is_optional=True),
+             Child('ThrowsKeyword', kind='Token',
+                   is_optional=True,
+                   token_choices=[
+                       'ThrowsToken',
+                       'RethrowsToken',
+                   ]),
              Child('Body', kind='CodeBlock', is_optional=True),
          ]),
 

utils/gyb_syntax_support/ExprNodes.py

@@ -543,7 +543,8 @@
              Child('RootExpr', kind='Expr', is_optional=True,
                    node_choices=[
                        Child('IdentifierExpr', kind='IdentifierExpr'),
-                       Child('SpecializeExpr', kind='SpecializeExpr')
+                       Child('SpecializeExpr', kind='SpecializeExpr'),
+                       Child('OptionalChainingExpr', kind='OptionalChainingExpr'),
                    ]),
              Child('Expression', kind='Expr'),
          ]),