Merge pull request swiftlang#2933 from swiftlang/revert-2925-syntax-data-arena

Revert "[Syntax] Use BumpPtrAllocator for Syntax node internals"

Author: rintaro

CodeGeneration/Sources/generate-swift-syntax/templates/swiftsyntax/SyntaxRewriterFile.swift

@@ -42,6 +42,13 @@ let syntaxRewriterFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
     )
 
+    DeclSyntax(
+      """
+      /// 'Syntax' object factory recycling 'Syntax.Info' instances.
+      private let nodeFactory: SyntaxNodeFactory = SyntaxNodeFactory()
+      """
+    )
+
     DeclSyntax(
       """
       public init(viewMode: SyntaxTreeViewMode = .sourceAccurate) {
@@ -323,11 +330,11 @@ let syntaxRewriterFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
         // with 'Syntax'
         var rewrittens: ContiguousArray<RetainedSyntaxArena> = []
 
-        for case let childDataRef? in node.layoutBuffer where viewMode.shouldTraverse(node: childDataRef.pointee.raw) {
+        for case let (child?, info) in RawSyntaxChildren(node) where viewMode.shouldTraverse(node: child) {
 
           // Build the Syntax node to rewrite
-          let childNode = visitImpl(Syntax(arena: node.arena, dataRef: childDataRef))
-          if childNode.raw.id != childDataRef.pointee.raw.id {
+          var childNode = visitImpl(nodeFactory.create(parent: node, raw: child, absoluteInfo: info))
+          if childNode.raw.id != child.id {
             // The node was rewritten, let's handle it
 
             if newLayout.baseAddress == nil {
@@ -338,10 +345,13 @@ let syntaxRewriterFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
             }
 
             // Update the rewritten child.
-            newLayout[Int(childDataRef.pointee.absoluteInfo.layoutIndexInParent)] = childNode.raw
+            newLayout[Int(info.indexInParent)] = childNode.raw
             // Retain the syntax arena of the new node until it's wrapped with Syntax node.
             rewrittens.append(childNode.raw.arenaReference.retained)
           }
+
+          // Recycle 'childNode.info'
+          nodeFactory.dispose(&childNode)
         }
 
         if newLayout.baseAddress != nil {

CodeGeneration/Sources/generate-swift-syntax/templates/swiftsyntax/SyntaxVisitorFile.swift

@@ -32,6 +32,13 @@ let syntaxVisitorFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   try! ClassDeclSyntax("open class SyntaxVisitor") {
     DeclSyntax("public let viewMode: SyntaxTreeViewMode")
 
+    DeclSyntax(
+      """
+      /// 'Syntax' object factory recycling 'Syntax.Info' instances.
+      private let nodeFactory: SyntaxNodeFactory = SyntaxNodeFactory()
+      """
+    )
+
     DeclSyntax(
       """
       public init(viewMode: SyntaxTreeViewMode) {
@@ -214,8 +221,10 @@ let syntaxVisitorFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
     DeclSyntax(
       """
       private func visitChildren(_ node: Syntax) {
-        for case let childDataRef? in node.layoutBuffer where viewMode.shouldTraverse(node: childDataRef.pointee.raw) {
-          dispatchVisit(Syntax(arena: node.arena, dataRef: childDataRef))
+        for case let (child?, info) in RawSyntaxChildren(node) where viewMode.shouldTraverse(node: child) {
+          var childNode = nodeFactory.create(parent: node, raw: child, absoluteInfo: info)
+          dispatchVisit(childNode)
+          nodeFactory.dispose(&childNode)
         }
       }
       """

Release Notes/602.md

@@ -20,12 +20,6 @@
   - Migration steps: Do not use `SyntaxArena` or `ParsingSyntaxArena` directly.
   - Notes: Although the type itself was `public`, most initializers were already SPI and there was no way to retrive them from existing types via public API.
 
-- `SyntaxChildrenIndex` is no longer `ExpressibleByNilLiteral`
-  - Description: `nil` used to represent the end index. However, due to a change in the internal structure, the end index must now be retrieved from the collection.
-  - Pull Request: https://github.com/swiftlang/swift-syntax/pull/2925
-  - Migration steps: Use `SyntaxChildren.endIndex` instead.
-  - Notes: `ExpressibleByNilLiteral` was a mistake. In general, `Collection.Index` should only be created and managed by the collection itself. For example, `Collection.index(after:)` exists, but `Index.advanced(by:)` does not.
-
 ## Template
 
 - *Affected API or two word description*

Sources/SwiftSyntax/AbsoluteRawSyntax.swift

@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+struct AbsoluteRawSyntax: Sendable {
+  let raw: RawSyntax
+  let info: AbsoluteSyntaxInfo
+
+  /// Returns first `present` child.
+  func firstChild(viewMode: SyntaxTreeViewMode) -> AbsoluteRawSyntax? {
+    guard let layoutView = raw.layoutView else { return nil }
+    var curInfo = info.advancedToFirstChild()
+    for childOpt in layoutView.children {
+      if let child = childOpt, viewMode.shouldTraverse(node: child) {
+        return AbsoluteRawSyntax(raw: child, info: curInfo)
+      }
+      curInfo = curInfo.advancedBySibling(childOpt)
+    }
+    return nil
+  }
+
+  /// Returns next `present` sibling.
+  func nextSibling(parent: AbsoluteRawSyntax, viewMode: SyntaxTreeViewMode) -> AbsoluteRawSyntax? {
+    var curInfo = info.advancedBySibling(raw)
+    for siblingOpt in parent.raw.layoutView!.children.dropFirst(Int(info.indexInParent + 1)) {
+      if let sibling = siblingOpt, viewMode.shouldTraverse(node: sibling) {
+        return AbsoluteRawSyntax(raw: sibling, info: curInfo)
+      }
+      curInfo = curInfo.advancedBySibling(siblingOpt)
+    }
+    return nil
+  }
+
+  func replacingSelf(_ newRaw: RawSyntax, newRootId: UInt) -> AbsoluteRawSyntax {
+    let nodeId = SyntaxIdentifier(rootId: newRootId, indexInTree: info.nodeId.indexInTree)
+    let newInfo = AbsoluteSyntaxInfo(position: info.position, nodeId: nodeId)
+    return .init(raw: newRaw, info: newInfo)
+  }
+}

Sources/SwiftSyntax/AbsoluteSyntaxInfo.swift

@@ -2,60 +2,57 @@
 //
 // This source file is part of the Swift.org open source project
 //
-// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See https://swift.org/LICENSE.txt for license information
 // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
 //
 //===----------------------------------------------------------------------===//
 
+struct AbsoluteSyntaxPosition: Sendable {
+  /// The UTF-8 offset of the syntax node in the source file
+  let offset: UInt32
+  let indexInParent: UInt32
+
+  func advancedBySibling(_ raw: RawSyntax?) -> AbsoluteSyntaxPosition {
+    let newOffset = self.offset + UInt32(truncatingIfNeeded: raw?.totalLength.utf8Length ?? 0)
+    let newIndexInParent = self.indexInParent + 1
+    return .init(offset: newOffset, indexInParent: newIndexInParent)
+  }
+
+  func advancedToFirstChild() -> AbsoluteSyntaxPosition {
+    return .init(offset: self.offset, indexInParent: 0)
+  }
+
+  static var forRoot: AbsoluteSyntaxPosition {
+    return .init(offset: 0, indexInParent: 0)
+  }
+}
+
 /// `AbsoluteSyntaxInfo` represents the information that relates a `RawSyntax`
 /// to a source file tree, like its absolute source offset.
 struct AbsoluteSyntaxInfo: Sendable {
-  /// The UTF-8 offset at which the syntax node’s leading trivia start in the source file.
-  let offset: UInt32
-
-  /// Index in parent's layout. Note that this counts `nil` children.
-  let layoutIndexInParent: UInt32
+  let position: AbsoluteSyntaxPosition
+  let nodeId: SyntaxIdentifier
 
-  /// Index of the node when traversing the syntax tree using a depth-first traversal.
-  /// This skips `nil` children in the parent's layout.
-  let indexInTree: UInt32
+  /// The UTF-8 offset of the syntax node in the source file
+  var offset: UInt32 { return position.offset }
+  var indexInParent: UInt32 { return position.indexInParent }
 
   func advancedBySibling(_ raw: RawSyntax?) -> AbsoluteSyntaxInfo {
-    if let raw {
-      // '&+' operations are safe because we have the preconditions in 'forRoot(_:)'.
-      return AbsoluteSyntaxInfo(
-        offset: offset &+ UInt32(truncatingIfNeeded: raw.totalLength.utf8Length),
-        layoutIndexInParent: layoutIndexInParent &+ 1,
-        indexInTree: indexInTree &+ UInt32(truncatingIfNeeded: raw.totalNodes)
-      )
-    } else {
-      return AbsoluteSyntaxInfo(
-        offset: offset,
-        layoutIndexInParent: layoutIndexInParent &+ 1,
-        indexInTree: indexInTree
-      )
-    }
+    let newPosition = position.advancedBySibling(raw)
+    let newNodeId = nodeId.advancedBySibling(raw)
+    return .init(position: newPosition, nodeId: newNodeId)
   }
 
   func advancedToFirstChild() -> AbsoluteSyntaxInfo {
-    return AbsoluteSyntaxInfo(
-      offset: offset,
-      layoutIndexInParent: 0,
-      indexInTree: indexInTree &+ 1
-    )
+    let newPosition = position.advancedToFirstChild()
+    let newNodeId = nodeId.advancedToFirstChild()
+    return .init(position: newPosition, nodeId: newNodeId)
   }
 
   static func forRoot(_ raw: RawSyntax) -> AbsoluteSyntaxInfo {
-    // These checks ensure the safety of the unchecked arithmetic operations in 'advancedBySibling(_:)'.
-    precondition(raw.totalLength.utf8Length <= UInt32.max, "too long")
-    precondition(raw.totalNodes <= UInt32.max, "too many nodes")
-    return AbsoluteSyntaxInfo(
-      offset: 0,
-      layoutIndexInParent: 0,
-      indexInTree: 0
-    )
+    return .init(position: .forRoot, nodeId: .forRoot(raw))
   }
 }

Sources/SwiftSyntax/CMakeLists.txt

@@ -8,6 +8,7 @@
 
 add_swift_syntax_library(SwiftSyntax
   AbsolutePosition.swift
+  AbsoluteRawSyntax.swift
   AbsoluteSyntaxInfo.swift
   Assert.swift
   BumpPtrAllocator.swift
@@ -30,6 +31,7 @@ add_swift_syntax_library(SwiftSyntax
   SyntaxCollection.swift
   SyntaxHashable.swift
   SyntaxIdentifier.swift
+  SyntaxNodeFactory.swift
   SyntaxNodeStructure.swift
   SyntaxProtocol.swift
   SyntaxText.swift

Sources/SwiftSyntax/MemoryLayout.swift

@@ -13,9 +13,9 @@
 // See `MemoryLayoutTest.swift`.
 @_spi(Testing) public enum SyntaxMemoryLayout: Sendable {
   public struct Value: Equatable, Sendable {
-    public let size: Int
-    public let stride: Int
-    public let alignment: Int
+    var size: Int
+    var stride: Int
+    var alignment: Int
 
     public init(size: Int, stride: Int, alignment: Int) {
       self.size = size

Sources/SwiftSyntax/Raw/RawSyntax.swift

@@ -186,10 +186,10 @@ extension RawSyntaxData.ParsedToken {
 
 extension RawSyntaxData.MaterializedToken {
   var leadingTrivia: RawTriviaPieceBuffer {
-    RawTriviaPieceBuffer(rebasing: triviaPieces[..<Int(numLeadingTrivia)])
+    triviaPieces[..<Int(numLeadingTrivia)]
   }
   var trailingTrivia: RawTriviaPieceBuffer {
-    RawTriviaPieceBuffer(rebasing: triviaPieces[Int(numLeadingTrivia)...])
+    triviaPieces[Int(numLeadingTrivia)...]
   }
 }
 
@@ -954,7 +954,7 @@ extension RawSyntax {
 extension RawSyntax: Identifiable {
   public struct ID: Hashable, @unchecked Sendable {
     /// The pointer to the start of the `RawSyntax` node.
-    fileprivate var pointer: UnsafeRawPointer
+    private var pointer: UnsafeRawPointer
     fileprivate init(_ raw: RawSyntax) {
       self.pointer = raw.pointer.unsafeRawPointer
     }
@@ -965,13 +965,6 @@ extension RawSyntax: Identifiable {
   }
 }
 
-extension UInt {
-  /// Convert `RawSymtax.ID` to `UInt`. Lossless.
-  init(rawID: RawSyntax.ID) {
-    self.init(bitPattern: rawID.pointer)
-  }
-}
-
 /// See `SyntaxMemoryLayout`.
 let RawSyntaxDataMemoryLayouts: [String: SyntaxMemoryLayout.Value] = [
   "RawSyntaxData": .init(RawSyntaxData.self),