Add interface for lookup up of externally stored type descriptors

Currently, TypeRefBuilder knows how to parse type descriptors from
reflection metadata. We aim to store the same information that lives
in type descriptors externally, as an effort to support embedded Swift
debugging. In order to take advantage of all of the existing type
information parsing code that exists today in remote mirrors, add an
interface for external lookup of type descriptors, and replace all the
usages of type descriptors with their generic counterpart (this patch
only adds support for builtin descriptors, but follow up patches will
add support for other descriptor types).

Author: augusto2112

include/swift/RemoteInspection/DescriptorFinder.h

@@ -0,0 +1,53 @@
+//===--------------- DescriptorFinder.h -------------------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_REFLECTION_DESCRIPTOR_FINDER_H
+#define SWIFT_REFLECTION_DESCRIPTOR_FINDER_H
+
+#include "llvm/ADT/StringRef.h"
+
+namespace swift {
+namespace reflection {
+
+class TypeRef;
+
+/// An abstract interface for a builtin type descriptor.
+struct BuiltinTypeDescriptorBase {
+  const uint32_t Size;
+  const uint32_t Alignment;
+  const uint32_t Stride;
+  const uint32_t NumExtraInhabitants;
+  const bool IsBitwiseTakable;
+
+  BuiltinTypeDescriptorBase(uint32_t Size, uint32_t Alignment, uint32_t Stride,
+                            uint32_t NumExtraInhabitants, bool IsBitwiseTakable)
+      : Size(Size), Alignment(Alignment), Stride(Stride),
+        NumExtraInhabitants(NumExtraInhabitants),
+        IsBitwiseTakable(IsBitwiseTakable) {}
+
+  virtual ~BuiltinTypeDescriptorBase(){};
+
+  virtual llvm::StringRef getMangledTypeName() = 0;
+};
+
+/// Interface for finding type descriptors. Implementors may provide descriptors
+/// that live inside or outside reflection metadata.
+struct DescriptorFinder {
+  virtual ~DescriptorFinder(){};
+
+  virtual std::unique_ptr<BuiltinTypeDescriptorBase>
+  getBuiltinTypeDescriptor(const TypeRef *TR) = 0;
+};
+
+} // namespace reflection
+} // namespace swift
+#endif

include/swift/RemoteInspection/ReflectionContext.h

@@ -30,6 +30,7 @@
 #include "swift/Concurrency/Actor.h"
 #include "swift/Remote/MemoryReader.h"
 #include "swift/Remote/MetadataReader.h"
+#include "swift/RemoteInspection/DescriptorFinder.h"
 #include "swift/RemoteInspection/GenericMetadataCacheEntry.h"
 #include "swift/RemoteInspection/Records.h"
 #include "swift/RemoteInspection/RuntimeInternals.h"
@@ -214,8 +215,9 @@ class ReflectionContext
 
   explicit ReflectionContext(
       std::shared_ptr<MemoryReader> reader,
-      remote::ExternalTypeRefCache *externalCache = nullptr)
-      : super(std::move(reader), *this, externalCache) {}
+      remote::ExternalTypeRefCache *externalCache = nullptr,
+      reflection::DescriptorFinder *descriptorFinder = nullptr)
+      : super(std::move(reader), *this, externalCache, descriptorFinder) {}
 
   ReflectionContext(const ReflectionContext &other) = delete;
   ReflectionContext &operator=(const ReflectionContext &other) = delete;

include/swift/RemoteInspection/TypeLowering.h

@@ -23,6 +23,7 @@
 #include "llvm/Support/Casting.h"
 #include "swift/Remote/MetadataReader.h"
 #include "swift/Remote/TypeInfoProvider.h"
+#include "swift/RemoteInspection/DescriptorFinder.h"
 
 #include <memory>
 
@@ -174,7 +175,7 @@ class BuiltinTypeInfo : public TypeInfo {
 
 public:
   explicit BuiltinTypeInfo(TypeRefBuilder &builder,
-                           RemoteRef<BuiltinTypeDescriptor> descriptor);
+                           BuiltinTypeDescriptorBase &descriptor);
 
   /// Construct an empty builtin type info.
   BuiltinTypeInfo()

include/swift/RemoteInspection/TypeRefBuilder.h

@@ -20,6 +20,7 @@
 
 #include "swift/Remote/ExternalTypeRefCache.h"
 #include "swift/Remote/MetadataReader.h"
+#include "swift/RemoteInspection/DescriptorFinder.h"
 #include "swift/RemoteInspection/MetadataSourceBuilder.h"
 #include "swift/RemoteInspection/Records.h"
 #include "swift/RemoteInspection/TypeLowering.h"
@@ -409,10 +410,9 @@ class TypeRefBuilder {
 
   TypeConverter TC;
 
-#define TYPEREF(Id, Parent)                                                    \
-  std::unordered_map<TypeRefID, const Id##TypeRef *, TypeRefID::Hash,          \
-                     TypeRefID::Equal>                                         \
-      Id##TypeRefs;
+#define TYPEREF(Id, Parent) \
+  std::unordered_map<TypeRefID, const Id##TypeRef *, \
+                     TypeRefID::Hash, TypeRefID::Equal> Id##TypeRefs;
 #include "swift/RemoteInspection/TypeRefs.def"
 
 public:
@@ -453,7 +453,10 @@ class TypeRefBuilder {
     }
   };
 
-  struct ReflectionTypeDescriptorFinder {
+  /// The default descriptor finder implementation that find descriptors from
+  /// reflection metadata.
+  struct ReflectionTypeDescriptorFinder
+      : public swift::reflection::DescriptorFinder {
     ReflectionTypeDescriptorFinder(TypeRefBuilder &Builder,
                                    remote::ExternalTypeRefCache *externalCache)
         : Builder(Builder), ExternalTypeRefCache(externalCache) {}
@@ -476,8 +479,8 @@ class TypeRefBuilder {
     /// Load unsubstituted field types for a nominal type.
     RemoteRef<FieldDescriptor> getFieldTypeInfo(const TypeRef *TR);
 
-    /// Get the primitive type lowering for a builtin type.
-    RemoteRef<BuiltinTypeDescriptor> getBuiltinTypeInfo(const TypeRef *TR);
+    std::unique_ptr<BuiltinTypeDescriptorBase>
+    getBuiltinTypeDescriptor(const TypeRef *TR) override;
 
     /// Get the raw capture descriptor for a remote capture descriptor
     /// address.
@@ -506,7 +509,11 @@ class TypeRefBuilder {
     llvm::Optional<std::string> normalizeReflectionName(RemoteRef<char> name);
 
   private:
+    /// Get the primitive type lowering for a builtin type.
+    RemoteRef<BuiltinTypeDescriptor> getBuiltinTypeInfo(const TypeRef *TR);
+
     void populateFieldTypeInfoCacheWithReflectionAtIndex(size_t Index);
+
     llvm::Optional<RemoteRef<FieldDescriptor>>
     findFieldDescriptorAtIndex(size_t Index, const std::string &MangledName);
 
@@ -548,6 +555,8 @@ class TypeRefBuilder {
 
     TypeRefBuilder &Builder;
 
+    /// The external typeref cache for looking up field descriptor locators in
+    /// an external file.
     remote::ExternalTypeRefCache *ExternalTypeRefCache = nullptr;
 
   public:
@@ -844,7 +853,6 @@ class TypeRefBuilder {
 
   BuiltTypeDecl createTypeDecl(std::string &&mangledName, bool &typeAlias) {
     return {{(mangledName)}};
-    ;
   }
 
   BuiltProtocolDecl createProtocolDecl(Node *node) {
@@ -1409,8 +1417,21 @@ class TypeRefBuilder {
   using IntVariableReader =
       std::function<llvm::Optional<uint64_t>(std::string, unsigned)>;
 
+  /// The external type descriptor finder injected into this TypeRefBuilder, for
+  /// lookup of descriptors outside of metadata.
+  DescriptorFinder *EDF;
+
+  /// The type descriptor finder that looks up descriptors from metadata.
   ReflectionTypeDescriptorFinder RDF;
 
+  /// Returns the descriptor finders in the order that they should be consulted
+  /// in.
+  llvm::SmallVector<DescriptorFinder *, 2> getDescriptorFinders() {
+    if (EDF)
+      return {EDF, &RDF};
+    return {&RDF};
+  }
+
   // These fields are captured from the MetadataReader template passed into the
   // TypeRefBuilder struct, to isolate its template-ness from the rest of
   // TypeRefBuilder.
@@ -1428,8 +1449,9 @@ class TypeRefBuilder {
 public:
   template <typename Runtime>
   TypeRefBuilder(remote::MetadataReader<Runtime, TypeRefBuilder> &reader,
-                 remote::ExternalTypeRefCache *externalCache = nullptr)
-      : TC(*this), RDF(*this, externalCache),
+                 remote::ExternalTypeRefCache *externalCache = nullptr,
+                 DescriptorFinder *externalDescriptorFinder = nullptr)
+      : TC(*this), EDF(externalDescriptorFinder), RDF(*this, externalCache),
         PointerSize(sizeof(typename Runtime::StoredPointer)),
         TypeRefDemangler([this, &reader](RemoteRef<char> string,
                                          bool useOpaqueTypeSymbolicReferences)
@@ -1519,10 +1541,11 @@ class TypeRefBuilder {
                         remote::TypeInfoProvider *ExternalTypeInfo,
                         std::vector<FieldTypeInfo> &Fields);
 
-  /// Get the primitive type lowering for a builtin type.
-  RemoteRef<BuiltinTypeDescriptor> getBuiltinTypeInfo(const TypeRef *TR) {
-    return RDF.getBuiltinTypeInfo(TR);
-  }
+  /// Get the generic interface version of a builtin type descriptor. This
+  /// descriptor may originate from reflection metadata or from an external
+  /// source.
+  std::unique_ptr<BuiltinTypeDescriptorBase>
+  getBuiltinTypeDescriptor(const TypeRef *TR);
 
   /// Get the raw capture descriptor for a remote capture descriptor
   /// address.
@@ -1542,6 +1565,9 @@ class TypeRefBuilder {
   }
 
 private:
+  /// Get the primitive type lowering for a builtin type.
+  RemoteRef<BuiltinTypeDescriptor> getBuiltinTypeInfo(const TypeRef *TR);
+
   llvm::Optional<uint64_t> multiPayloadEnumPointerMask;
 
 public:

stdlib/public/RemoteInspection/TypeLowering.cpp

@@ -230,16 +230,12 @@ void TypeInfo::dump(std::ostream &stream, unsigned Indent) const {
 }
 
 BuiltinTypeInfo::BuiltinTypeInfo(TypeRefBuilder &builder,
-                                 RemoteRef<BuiltinTypeDescriptor> descriptor)
-    : TypeInfo(TypeInfoKind::Builtin,
-               descriptor->Size,
-               descriptor->getAlignment(),
-               descriptor->Stride,
-               descriptor->NumExtraInhabitants,
-               descriptor->isBitwiseTakable()),
-      Name(builder.getTypeRefString(
-              builder.readTypeRef(descriptor, descriptor->TypeName)))
-{}
+                                 BuiltinTypeDescriptorBase &descriptor)
+    : TypeInfo(TypeInfoKind::Builtin, descriptor.Size,
+               descriptor.Alignment, descriptor.Stride,
+               descriptor.NumExtraInhabitants,
+               descriptor.IsBitwiseTakable),
+      Name(descriptor.getMangledTypeName()) {}
 
 bool BuiltinTypeInfo::readExtraInhabitantIndex(
     remote::MemoryReader &reader, remote::RemoteAddress address,
@@ -1641,9 +1637,8 @@ const RecordTypeInfo *RecordTypeInfoBuilder::build() {
       Kind, Fields);
 }
 
-const ReferenceTypeInfo *
-TypeConverter::getReferenceTypeInfo(ReferenceKind Kind,
-                                    ReferenceCounting Refcounting) {
+const ReferenceTypeInfo *TypeConverter::getReferenceTypeInfo(
+    ReferenceKind Kind, ReferenceCounting Refcounting) {
   auto key = std::make_pair(unsigned(Kind), unsigned(Refcounting));
   auto found = ReferenceCache.find(key);
   if (found != ReferenceCache.end())
@@ -1664,7 +1659,7 @@ TypeConverter::getReferenceTypeInfo(ReferenceKind Kind,
   //
   // Weak references do not have any extra inhabitants.
 
-  auto BuiltinTI = Builder.getBuiltinTypeInfo(TR);
+  auto BuiltinTI = Builder.getBuiltinTypeDescriptor(TR);
   if (BuiltinTI == nullptr) {
     DEBUG_LOG(fprintf(stderr, "No TypeInfo for reference type: "); TR->dump());
     return nullptr;
@@ -1689,7 +1684,7 @@ TypeConverter::getReferenceTypeInfo(ReferenceKind Kind,
   }
 
   auto *TI = makeTypeInfo<ReferenceTypeInfo>(BuiltinTI->Size,
-                                             BuiltinTI->getAlignment(),
+                                             BuiltinTI->Alignment,
                                              BuiltinTI->Stride,
                                              numExtraInhabitants,
                                              bitwiseTakable,
@@ -1705,13 +1700,14 @@ TypeConverter::getThinFunctionTypeInfo() {
   if (ThinFunctionTI != nullptr)
     return ThinFunctionTI;
 
-  auto descriptor = getBuilder().getBuiltinTypeInfo(getThinFunctionTypeRef());
+  auto descriptor =
+      getBuilder().getBuiltinTypeDescriptor(getThinFunctionTypeRef());
   if (descriptor == nullptr) {
     DEBUG_LOG(fprintf(stderr, "No TypeInfo for function type\n"));
     return nullptr;
   }
 
-  ThinFunctionTI = makeTypeInfo<BuiltinTypeInfo>(getBuilder(), descriptor);
+  ThinFunctionTI = makeTypeInfo<BuiltinTypeInfo>(getBuilder(), *descriptor.get());
 
   return ThinFunctionTI;
 }
@@ -1739,13 +1735,14 @@ TypeConverter::getAnyMetatypeTypeInfo() {
   if (AnyMetatypeTI != nullptr)
     return AnyMetatypeTI;
 
-  auto descriptor = getBuilder().getBuiltinTypeInfo(getAnyMetatypeTypeRef());
+  auto descriptor =
+      getBuilder().getBuiltinTypeDescriptor(getAnyMetatypeTypeRef());
   if (descriptor == nullptr) {
     DEBUG_LOG(fprintf(stderr, "No TypeInfo for metatype type\n"));
     return nullptr;
   }
 
-  AnyMetatypeTI = makeTypeInfo<BuiltinTypeInfo>(getBuilder(), descriptor);
+  AnyMetatypeTI = makeTypeInfo<BuiltinTypeInfo>(getBuilder(), *descriptor.get());
 
   return AnyMetatypeTI;
 }
@@ -2245,7 +2242,7 @@ class EnumTypeInfoBuilder {
 
     // Do we have a fixed layout?
     // TODO: Test whether a missing FixedDescriptor is actually relevant.
-    auto FixedDescriptor = TC.getBuilder().getBuiltinTypeInfo(TR);
+    auto FixedDescriptor = TC.getBuilder().getBuiltinTypeDescriptor(TR);
     if (!FixedDescriptor || GenericPayloadCases > 0) {
       // This is a "dynamic multi-payload enum".  For example,
       // this occurs with:
@@ -2283,9 +2280,9 @@ class EnumTypeInfoBuilder {
     //  * Has at least two cases with non-zero payload size
     //  * Has a descriptor stored as BuiltinTypeInfo
     Size = FixedDescriptor->Size;
-    Alignment = FixedDescriptor->getAlignment();
+    Alignment = FixedDescriptor->Alignment;
     NumExtraInhabitants = FixedDescriptor->NumExtraInhabitants;
-    BitwiseTakable = FixedDescriptor->isBitwiseTakable();
+    BitwiseTakable = FixedDescriptor->IsBitwiseTakable;
     unsigned Stride = ((Size + Alignment - 1) & ~(Alignment - 1));
     if (Stride == 0)
       Stride = 1;
@@ -2395,12 +2392,12 @@ class LowerType
 
     /// Otherwise, get the fixed layout information from reflection
     /// metadata.
-    auto descriptor = TC.getBuilder().getBuiltinTypeInfo(B);
+    auto descriptor = TC.getBuilder().getBuiltinTypeDescriptor(B);
     if (descriptor == nullptr) {
       DEBUG_LOG(fprintf(stderr, "No TypeInfo for builtin type: "); B->dump());
       return nullptr;
     }
-    return TC.makeTypeInfo<BuiltinTypeInfo>(TC.getBuilder(), descriptor);
+    return TC.makeTypeInfo<BuiltinTypeInfo>(TC.getBuilder(), *descriptor.get());
   }
 
   const TypeInfo *visitAnyNominalTypeRef(const TypeRef *TR) {
@@ -2424,15 +2421,15 @@ class LowerType
       // descriptor to see if we at least know its size
       // and alignment.
       if (auto ImportedTypeDescriptor =
-              TC.getBuilder().getBuiltinTypeInfo(TR)) {
+              TC.getBuilder().getBuiltinTypeDescriptor(TR)) {
         // This might be an external type we treat as opaque (like C structs),
         // the external type info provider might have better type information,
         // so ask it first.
         if (auto External = QueryExternalTypeInfoProvider())
           return External;
 
         return TC.makeTypeInfo<BuiltinTypeInfo>(TC.getBuilder(),
-                                                ImportedTypeDescriptor);
+                                                *ImportedTypeDescriptor.get());
       }
 
       if (FD == nullptr) {