Merge pull request #2925 from rintaro/syntax-data-arena

[Syntax] Use BumpPtrAllocator for Syntax node internals

Author: rintaro

CodeGeneration/Sources/generate-swift-syntax/templates/swiftsyntax/SyntaxRewriterFile.swift

@@ -42,13 +42,6 @@ let syntaxRewriterFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     )
 
-    DeclSyntax(
-      """
-      /// 'Syntax' object factory recycling 'Syntax.Info' instances.
-      private let nodeFactory: SyntaxNodeFactory = SyntaxNodeFactory()
-      """
-    )
-
     DeclSyntax(
       """
       public init(viewMode: SyntaxTreeViewMode = .sourceAccurate) {
@@ -330,11 +323,11 @@ let syntaxRewriterFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
         // with 'Syntax'
         var rewrittens: ContiguousArray<RetainedSyntaxArena> = []
 
-        for case let (child?, info) in RawSyntaxChildren(node) where viewMode.shouldTraverse(node: child) {
+        for case let childDataRef? in node.layoutBuffer where viewMode.shouldTraverse(node: childDataRef.pointee.raw) {
 
           // Build the Syntax node to rewrite
-          var childNode = visitImpl(nodeFactory.create(parent: node, raw: child, absoluteInfo: info))
-          if childNode.raw.id != child.id {
+          let childNode = visitImpl(Syntax(arena: node.arena, dataRef: childDataRef))
+          if childNode.raw.id != childDataRef.pointee.raw.id {
             // The node was rewritten, let's handle it
 
             if newLayout.baseAddress == nil {
@@ -345,13 +338,10 @@ let syntaxRewriterFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
             }
 
             // Update the rewritten child.
-            newLayout[Int(info.indexInParent)] = childNode.raw
+            newLayout[Int(childDataRef.pointee.absoluteInfo.layoutIndexInParent)] = childNode.raw
             // Retain the syntax arena of the new node until it's wrapped with Syntax node.
             rewrittens.append(childNode.raw.arenaReference.retained)
           }
-
-          // Recycle 'childNode.info'
-          nodeFactory.dispose(&childNode)
         }
 
         if newLayout.baseAddress != nil {

CodeGeneration/Sources/generate-swift-syntax/templates/swiftsyntax/SyntaxVisitorFile.swift

@@ -32,13 +32,6 @@ let syntaxVisitorFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   try! ClassDeclSyntax("open class SyntaxVisitor") {
     DeclSyntax("public let viewMode: SyntaxTreeViewMode")
 
-    DeclSyntax(
-      """
-      /// 'Syntax' object factory recycling 'Syntax.Info' instances.
-      private let nodeFactory: SyntaxNodeFactory = SyntaxNodeFactory()
-      """
-    )
-
     DeclSyntax(
       """
       public init(viewMode: SyntaxTreeViewMode) {
@@ -221,10 +214,8 @@ let syntaxVisitorFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
     DeclSyntax(
       """
       private func visitChildren(_ node: Syntax) {
-        for case let (child?, info) in RawSyntaxChildren(node) where viewMode.shouldTraverse(node: child) {
-          var childNode = nodeFactory.create(parent: node, raw: child, absoluteInfo: info)
-          dispatchVisit(childNode)
-          nodeFactory.dispose(&childNode)
+        for case let childDataRef? in node.layoutBuffer where viewMode.shouldTraverse(node: childDataRef.pointee.raw) {
+          dispatchVisit(Syntax(arena: node.arena, dataRef: childDataRef))
         }
       }
       """

Release Notes/602.md

@@ -20,6 +20,12 @@
   - Migration steps: Do not use `SyntaxArena` or `ParsingSyntaxArena` directly.
   - Notes: Although the type itself was `public`, most initializers were already SPI and there was no way to retrive them from existing types via public API.
 
+- `SyntaxChildrenIndex` is no longer `ExpressibleByNilLiteral`
+  - Description: `nil` used to represent the end index. However, due to a change in the internal structure, the end index must now be retrieved from the collection.
+  - Pull Request: https://github.com/swiftlang/swift-syntax/pull/2925
+  - Migration steps: Use `SyntaxChildren.endIndex` instead.
+  - Notes: `ExpressibleByNilLiteral` was a mistake. In general, `Collection.Index` should only be created and managed by the collection itself. For example, `Collection.index(after:)` exists, but `Index.advanced(by:)` does not.
+
 ## Template
 
 - *Affected API or two word description*

Sources/SwiftSyntax/AbsoluteRawSyntax.swift

@@ -2,57 +2,60 @@
 //
 // This source file is part of the Swift.org open source project
 //
-// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See https://swift.org/LICENSE.txt for license information
 // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
 //
 //===----------------------------------------------------------------------===//
 
-struct AbsoluteSyntaxPosition: Sendable {
-  /// The UTF-8 offset of the syntax node in the source file
-  let offset: UInt32
-  let indexInParent: UInt32
-
-  func advancedBySibling(_ raw: RawSyntax?) -> AbsoluteSyntaxPosition {
-    let newOffset = self.offset + UInt32(truncatingIfNeeded: raw?.totalLength.utf8Length ?? 0)
-    let newIndexInParent = self.indexInParent + 1
-    return .init(offset: newOffset, indexInParent: newIndexInParent)
-  }
-
-  func advancedToFirstChild() -> AbsoluteSyntaxPosition {
-    return .init(offset: self.offset, indexInParent: 0)
-  }
-
-  static var forRoot: AbsoluteSyntaxPosition {
-    return .init(offset: 0, indexInParent: 0)
-  }
-}
-
 /// `AbsoluteSyntaxInfo` represents the information that relates a `RawSyntax`
 /// to a source file tree, like its absolute source offset.
 struct AbsoluteSyntaxInfo: Sendable {
-  let position: AbsoluteSyntaxPosition
-  let nodeId: SyntaxIdentifier
+  /// The UTF-8 offset at which the syntax node’s leading trivia start in the source file.
+  let offset: UInt32
+
+  /// Index in parent's layout. Note that this counts `nil` children.
+  let layoutIndexInParent: UInt32
 
-  /// The UTF-8 offset of the syntax node in the source file
-  var offset: UInt32 { return position.offset }
-  var indexInParent: UInt32 { return position.indexInParent }
+  /// Index of the node when traversing the syntax tree using a depth-first traversal.
+  /// This skips `nil` children in the parent's layout.
+  let indexInTree: UInt32
 
   func advancedBySibling(_ raw: RawSyntax?) -> AbsoluteSyntaxInfo {
-    let newPosition = position.advancedBySibling(raw)
-    let newNodeId = nodeId.advancedBySibling(raw)
-    return .init(position: newPosition, nodeId: newNodeId)
+    if let raw {
+      // '&+' operations are safe because we have the preconditions in 'forRoot(_:)'.
+      return AbsoluteSyntaxInfo(
+        offset: offset &+ UInt32(truncatingIfNeeded: raw.totalLength.utf8Length),
+        layoutIndexInParent: layoutIndexInParent &+ 1,
+        indexInTree: indexInTree &+ UInt32(truncatingIfNeeded: raw.totalNodes)
+      )
+    } else {
+      return AbsoluteSyntaxInfo(
+        offset: offset,
+        layoutIndexInParent: layoutIndexInParent &+ 1,
+        indexInTree: indexInTree
+      )
+    }
   }
 
   func advancedToFirstChild() -> AbsoluteSyntaxInfo {
-    let newPosition = position.advancedToFirstChild()
-    let newNodeId = nodeId.advancedToFirstChild()
-    return .init(position: newPosition, nodeId: newNodeId)
+    return AbsoluteSyntaxInfo(
+      offset: offset,
+      layoutIndexInParent: 0,
+      indexInTree: indexInTree &+ 1
+    )
   }
 
   static func forRoot(_ raw: RawSyntax) -> AbsoluteSyntaxInfo {
-    return .init(position: .forRoot, nodeId: .forRoot(raw))
+    // These checks ensure the safety of the unchecked arithmetic operations in 'advancedBySibling(_:)'.
+    precondition(raw.totalLength.utf8Length <= UInt32.max, "too long")
+    precondition(raw.totalNodes <= UInt32.max, "too many nodes")
+    return AbsoluteSyntaxInfo(
+      offset: 0,
+      layoutIndexInParent: 0,
+      indexInTree: 0
+    )
   }
 }

Sources/SwiftSyntax/AbsoluteSyntaxInfo.swift

@@ -8,7 +8,6 @@
 
 add_swift_syntax_library(SwiftSyntax
   AbsolutePosition.swift
-  AbsoluteRawSyntax.swift
   AbsoluteSyntaxInfo.swift
   Assert.swift
   BumpPtrAllocator.swift
@@ -31,7 +30,6 @@ add_swift_syntax_library(SwiftSyntax
   SyntaxCollection.swift
   SyntaxHashable.swift
   SyntaxIdentifier.swift
-  SyntaxNodeFactory.swift
   SyntaxNodeStructure.swift
   SyntaxProtocol.swift
   SyntaxText.swift

Sources/SwiftSyntax/CMakeLists.txt

@@ -13,9 +13,9 @@
 // See `MemoryLayoutTest.swift`.
 @_spi(Testing) public enum SyntaxMemoryLayout: Sendable {
   public struct Value: Equatable, Sendable {
-    var size: Int
-    var stride: Int
-    var alignment: Int
+    public let size: Int
+    public let stride: Int
+    public let alignment: Int
 
     public init(size: Int, stride: Int, alignment: Int) {
       self.size = size

Sources/SwiftSyntax/MemoryLayout.swift

@@ -186,10 +186,10 @@ extension RawSyntaxData.ParsedToken {
 
 extension RawSyntaxData.MaterializedToken {
   var leadingTrivia: RawTriviaPieceBuffer {
-    triviaPieces[..<Int(numLeadingTrivia)]
+    RawTriviaPieceBuffer(rebasing: triviaPieces[..<Int(numLeadingTrivia)])
   }
   var trailingTrivia: RawTriviaPieceBuffer {
-    triviaPieces[Int(numLeadingTrivia)...]
+    RawTriviaPieceBuffer(rebasing: triviaPieces[Int(numLeadingTrivia)...])
   }
 }
 
@@ -954,7 +954,7 @@ extension RawSyntax {
 extension RawSyntax: Identifiable {
   public struct ID: Hashable, @unchecked Sendable {
     /// The pointer to the start of the `RawSyntax` node.
-    private var pointer: UnsafeRawPointer
+    fileprivate var pointer: UnsafeRawPointer
     fileprivate init(_ raw: RawSyntax) {
       self.pointer = raw.pointer.unsafeRawPointer
     }
@@ -965,6 +965,13 @@ extension RawSyntax: Identifiable {
   }
 }
 
+extension UInt {
+  /// Convert `RawSymtax.ID` to `UInt`. Lossless.
+  init(rawID: RawSyntax.ID) {
+    self.init(bitPattern: rawID.pointer)
+  }
+}
+
 /// See `SyntaxMemoryLayout`.
 let RawSyntaxDataMemoryLayouts: [String: SyntaxMemoryLayout.Value] = [
   "RawSyntaxData": .init(RawSyntaxData.self),