implement handling of extensions to external types as enabled by swiftlang/swift#59047 (#369)

- introduce transformation generating internal Extended Types Symbol Graph Format from
   the Extension Block Symbol Graph Format emmitted by the compiler
 - register symbols and relationships used by Extended Types Symbol Graph Format
 - adapt SymbolGraphLoader to automatically detect if input Symbol Graph Files use the
   Extension Block Symbol Graph Format and apply the ExtendedTypesFormatTransformation
   in case
 - improve Swift title token parsing to correctly identify Symbol titles of nested types,
   which contain "." infixes

added tests:
 - test detection of the Extension Block Symbol Graph Format and application of the
   ExtendedTypesFormatTransformation in SymbolGraphLoader
 - test the ExtendedTypesFormatTransformation
 - extend tests for Swift title token parsing with test-cases containing "." infixes

Author: theMomax

Package.resolved

@@ -79,7 +79,9 @@ let package = Package(
         // Test utility library
         .target(
             name: "SwiftDocCTestUtilities",
-            dependencies: []),
+            dependencies: [
+                "SymbolKit"
+            ]),
 
         // Command-line tool
         .executableTarget(

Package.swift

@@ -185,15 +185,15 @@ extension DocumentationCoverageOptions.KindFilterOptions {
         /// Converts given ``DocumentationNode.Kind`` to corresponding `BitFlagRepresentation` if possible. Returns `nil` if the given Kind is not representable.
         fileprivate init?(kind: DocumentationNode.Kind) {
             switch kind {
-            case .module: // 1
+            case .module, .extendedModule: // 1
                 self = .module
-            case .class: // 2
+            case .class, .extendedClass: // 2
                 self = .class
-            case .structure: // 3
+            case .structure, .extendedStructure: // 3
                 self = .structure
-            case .enumeration: // 4
+            case .enumeration, .extendedEnumeration: // 4
                 self = .enumeration
-            case .protocol: // 5
+            case .protocol, .extendedProtocol: // 5
                 self = .protocol
             case .typeAlias: // 6
                 self = .typeAlias

Sources/SwiftDocC/Coverage/DocumentationCoverageOptions.swift

@@ -76,7 +76,7 @@ public class FileSystemRenderNodeProvider: RenderNodeProvider {
 
 extension RenderNode {
     private static let typesThatShouldNotUseNavigatorTitle: Set<NavigatorIndex.PageType> = [
-        .framework, .class, .structure, .enumeration, .protocol, .typeAlias, .associatedType
+        .framework, .class, .structure, .enumeration, .protocol, .typeAlias, .associatedType, .extension
     ]
 
     /// Returns a navigator title preferring the fragments inside the metadata, if applicable.

Sources/SwiftDocC/Indexing/Navigator/NavigatorIndex+Ext.swift

@@ -243,7 +243,12 @@ extension CoverageDataEntry {
                 .protocol,
                 .typeAlias,
                 .associatedType,
-                .typeDef:
+                .typeDef,
+                .extendedClass,
+                .extendedStructure,
+                .extendedEnumeration,
+                .extendedProtocol,
+                .unknownExtendedType:
                 self = .types
             case .localVariable,
                 .instanceProperty,
@@ -256,7 +261,7 @@ extension CoverageDataEntry {
                 .typeSubscript,
                 .instanceSubscript:
                 self = .members
-            case .function, .module, .globalVariable, .operator:
+            case .function, .module, .globalVariable, .operator, .extendedModule:
                 self = .globals
             case let kind where SummaryCategory.allKnownNonSymbolKindNames.contains(kind.name):
                 self = .nonSymbol
@@ -297,46 +302,46 @@ extension CoverageDataEntry {
             context: DocumentationContext
         ) throws {
             switch documentationNode.kind {
-            case DocumentationNode.Kind.class:
+            case .class, .extendedClass:
                 self = try .class(
                     memberStats: KindSpecificData.extractChildStats(
                         documentationNode: documentationNode,
                         context: context))
-            case DocumentationNode.Kind.enumeration:
+            case .enumeration, .extendedEnumeration:
                 self = try .enumeration(
                     memberStats: KindSpecificData.extractChildStats(
                         documentationNode: documentationNode,
                         context: context))
-            case DocumentationNode.Kind.structure:
+            case .structure, .extendedStructure:
                 self = try .structure(
                     memberStats:  KindSpecificData.extractChildStats(
                         documentationNode: documentationNode,
                         context: context))
-            case DocumentationNode.Kind.protocol:
-                self = try .enumeration(
+            case .protocol, .extendedProtocol:
+                self = try .protocol(
                     memberStats: KindSpecificData.extractChildStats(
                         documentationNode: documentationNode,
                         context: context))
 
-            case DocumentationNode.Kind.instanceMethod:
+            case .instanceMethod:
                 self = try .instanceMethod(
                     parameterStats: CoverageDataEntry.KindSpecificData.extractFunctionSignatureStats(
                         documentationNode: documentationNode,
                         context: context
                         , fieldName: "method parameters"))
-            case DocumentationNode.Kind.operator:
+            case .operator:
                 self = try .`operator`(
                     parameterStats: CoverageDataEntry.KindSpecificData.extractFunctionSignatureStats(
                         documentationNode: documentationNode,
                         context: context,
                         fieldName: "operator parameters"))
-            case DocumentationNode.Kind.function:
+            case .function:
                 self = try .`operator`(
                     parameterStats: CoverageDataEntry.KindSpecificData.extractFunctionSignatureStats(
                         documentationNode: documentationNode,
                         context: context,
                         fieldName: "function parameters"))
-            case DocumentationNode.Kind.initializer:
+            case .initializer:
                 self = try .`operator`(
                     parameterStats: CoverageDataEntry.KindSpecificData.extractFunctionSignatureStats(
                         documentationNode: documentationNode,

Sources/SwiftDocC/Infrastructure/CoverageDataEntry.swift

@@ -277,6 +277,9 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
     public var externalMetadata = ExternalMetadata()
 
 
+    /// The decoder used in the `SymbolGraphLoader`
+    var decoder: JSONDecoder = JSONDecoder()
+    
     /// Initializes a documentation context with a given `dataProvider` and registers all the documentation bundles that it provides.
     ///
     /// - Parameter dataProvider: The data provider to register bundles from.
@@ -1035,7 +1038,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
     private func parentChildRelationship(from edge: SymbolGraph.Relationship) -> (ResolvedTopicReference, ResolvedTopicReference)? {
         // Filter only parent <-> child edges
         switch edge.kind {
-        case .memberOf, .requirementOf:
+        case .memberOf, .requirementOf, .declaredIn, .inContextOf:
             guard let parentRef = symbolIndex[edge.target]?.reference, let childRef = symbolIndex[edge.source]?.reference else {
             return nil
             }
@@ -1933,7 +1936,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
         discoveryGroup.async(queue: discoveryQueue) { [unowned self] in
             symbolGraphLoader = SymbolGraphLoader(bundle: bundle, dataProvider: self.dataProvider)
             do {
-                try symbolGraphLoader.loadAll()
+                try symbolGraphLoader.loadAll(using: decoder)
                 if LinkResolutionMigrationConfiguration.shouldSetUpHierarchyBasedLinkResolver {
                     let pathHierarchy = PathHierarchy(symbolGraphLoader: symbolGraphLoader, bundleName: urlReadablePath(bundle.displayName), knownDisambiguatedPathComponents: knownDisambiguatedSymbolPathComponents)
                     hierarchyBasedResolver = PathHierarchyBasedLinkResolver(pathHierarchy: pathHierarchy)

Sources/SwiftDocC/Infrastructure/DocumentationContext.swift

@@ -66,6 +66,18 @@ extension ExternalSymbolResolver {
             symbolKind = .var
         case .module:
             symbolKind = .module
+        case .extendedModule:
+            symbolKind = .extendedModule
+        case .extendedStructure:
+            symbolKind = .extendedStructure
+        case .extendedClass:
+            symbolKind = .extendedClass
+        case .extendedEnumeration:
+            symbolKind = .extendedEnumeration
+        case .extendedProtocol:
+            symbolKind = .extendedProtocol
+        case .unknownExtendedType:
+            symbolKind = .unknownExtendedType
 
         // There shouldn't be any reason for a symbol graph file to reference one of these kinds outside of the symbol graph itself.
         // Return `.class` as the symbol kind (acting as "any symbol") so that the render reference gets a "symbol" role.

Sources/SwiftDocC/Infrastructure/External Data/ExternalSymbolResolver+SymbolKind.swift

@@ -495,10 +495,13 @@ final class DocumentationCacheBasedLinkResolver {
         // therefore for the currently processed symbol to be a child of a re-written symbol it needs to have
         // at least 3 components. It's a fair optimization to make since graphs will include a lot of root level symbols.
         guard reference.pathComponents.count > 3,
-            // Fetch the symbol's parent
-            let parentReference = try symbolsURLHierarchy.parent(of: reference),
-            // If the parent path matches the current reference path, bail out
-            parentReference.pathComponents != reference.pathComponents.dropLast()
+                // Fetch the symbol's parent
+                let parentReference = try symbolsURLHierarchy.parent(of: reference),
+                // If the parent path matches the current reference path, bail out
+                parentReference.pathComponents != reference.pathComponents.dropLast(),
+                // If the parent is not from the same module (because we're dealing with a
+                // default implementation of an external protocol), bail out
+                parentReference.pathComponents[..<3] == reference.pathComponents[..<3]
         else { return reference }
 
         // Build an up to date reference path for the current node based on the parent path

Sources/SwiftDocC/Infrastructure/Link Resolution/DocumentationCacheBasedLinkResolver.swift

@@ -0,0 +1,40 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See https://swift.org/LICENSE.txt for license information
+ See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+*/
+
+import SymbolKit
+
+extension SymbolGraph.Symbol.AccessControl: Comparable {
+    private var level: Int? {
+        switch self {
+        case .private:
+            return 0
+        case .filePrivate:
+            return 1
+        case .internal:
+            return 2
+        case .public:
+            return 3
+        case .open:
+            return 4
+        default:
+            assertionFailure("Unknown AccessControl case was used in comparison.")
+            return nil
+        }
+    }
+    
+    public static func < (lhs: SymbolGraph.Symbol.AccessControl, rhs: SymbolGraph.Symbol.AccessControl) -> Bool {
+        guard let lhs = lhs.level,
+              let rhs = rhs.level else {
+            return false
+        }
+        
+        return lhs < rhs
+    }
+}

Sources/SwiftDocC/Infrastructure/Symbol Graph/AccessControl+Comparable.swift

@@ -0,0 +1,104 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See https://swift.org/LICENSE.txt for license information
+ See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+*/
+
+import SymbolKit
+
+// MARK: Custom Relationship Kind Identifiers
+
+extension SymbolGraph.Relationship.Kind {
+    /// This relationship connects top-level extended type symbols the
+    /// respective extended module symbol.
+    static let declaredIn = Self(rawValue: "declaredIn")
+    
+    /// This relationship markes a parent-child hierarchy between a nested
+    /// extended type symbol and its parent extended type symbol. It mirrors the
+    /// `memberOf` relationship between the two respective original type symbols.
+    static let inContextOf = Self(rawValue: "inContextOf")
+}
+
+// MARK: Custom Symbol Kind Identifiers
+
+extension SymbolGraph.Symbol.KindIdentifier {
+    static let extendedProtocol = Self(rawValue: "protocol.extension")
+    
+    static let extendedStructure = Self(rawValue: "struct.extension")
+    
+    static let extendedClass = Self(rawValue: "class.extension")
+    
+    static let extendedEnumeration = Self(rawValue: "enum.extension")
+    
+    static let unknownExtendedType = Self(rawValue: "unknown.extension")
+    
+    static let extendedModule = Self(rawValue: "module.extension")
+    
+    init?(extending other: Self) {
+        switch other {
+        case .struct:
+            self = .extendedStructure
+        case .protocol:
+            self = .extendedProtocol
+        case .class:
+            self = .extendedClass
+        case .enum:
+            self = .extendedEnumeration
+        case .module:
+            self = .extendedModule
+        default:
+            return nil
+        }
+    }
+    
+    static func extendedType(for extensionBlock: SymbolGraph.Symbol) -> Self? {
+        guard let extensionMixin = extensionBlock.mixins[SymbolGraph.Symbol.Swift.Extension.mixinKey] as? SymbolGraph.Symbol.Swift.Extension else {
+            return nil
+        }
+        
+        guard let typeKind = extensionMixin.typeKind else {
+            return nil
+        }
+        
+        return Self(extending: typeKind)
+    }
+}
+
+extension SymbolGraph.Symbol.Kind {
+    static func extendedType(for extensionBlock: SymbolGraph.Symbol) -> Self {
+        let id = SymbolGraph.Symbol.KindIdentifier.extendedType(for: extensionBlock)
+        switch id {
+        case .some(.extendedProtocol):
+            return Self(parsedIdentifier: .extendedProtocol, displayName: "Extended Protocol")
+        case .some(.extendedStructure):
+            return Self(parsedIdentifier: .extendedStructure, displayName: "Extended Structure")
+        case .some(.extendedClass):
+            return Self(parsedIdentifier: .extendedClass, displayName: "Extended Class")
+        case .some(.extendedEnumeration):
+            return Self(parsedIdentifier: .extendedEnumeration, displayName: "Extended Enumeration")
+        default:
+            return unknownExtendedType
+        }
+    }
+    
+    static let unknownExtendedType = Self(parsedIdentifier: .unknownExtendedType, displayName: "Extended Type")
+}
+
+
+// MARK: Swift AccessControl Levels
+
+extension SymbolGraph.Symbol.AccessControl {
+    static let `private` = Self(rawValue: "private")
+    
+    static let filePrivate = Self(rawValue: "fileprivate")
+    
+    static let `internal` = Self(rawValue: "internal")
+    
+    static let `public` = Self(rawValue: "public")
+    
+    static let open = Self(rawValue: "open")
+}