RequirementMachine: Skip protocol type aliases that contain unbound dependent member types

In the below, 'Self.A.A' is not a type parameter; rather, since
'Self.A' is concretely known to be 'S', we resolve it as 'S.A',
which performs a name lookup and finds the concrete type alias 'A':

    public struct S {
      public typealias A = Int
    }

    public protocol P {
      typealias A = S
    }

    public struct G<T> {}

    public protocol Q: P {
      typealias B = G<Self.A.A>
    }

This is fine, but such a type alias should not participate in
the rewrite system. Let's exclude them like any other invalid
requirement.

The type alias itself is not an error; however, it is an error
to use it from a 'where' clause requirement. This is not
diagnosed yet, though.

Fixes rdar://136686001.

Author: slavapestov

lib/AST/RequirementMachine/HomotopyReduction.cpp

@@ -611,10 +611,11 @@ GenericSignatureErrors RewriteSystem::getErrors() const {
     if (!isInMinimizationDomain(rule.getLHS().getRootProtocol()))
       continue;
 
-    if (!rule.isRedundant() &&
-        !rule.isProtocolTypeAliasRule() &&
-        rule.containsNameSymbols())
-      result |= GenericSignatureErrorFlags::HasInvalidRequirements;
+    if (!rule.isRedundant()) {
+      if (!rule.isProtocolTypeAliasRule() &&
+          rule.containsNameSymbols())
+        result |= GenericSignatureErrorFlags::HasInvalidRequirements;
+    }
 
     if (rule.isRecursive())
       result |= GenericSignatureErrorFlags::HasInvalidRequirements;
@@ -624,12 +625,9 @@ GenericSignatureErrors RewriteSystem::getErrors() const {
         if (property->getKind() == Symbol::Kind::ConcreteConformance)
           result |= GenericSignatureErrorFlags::HasConcreteConformances;
 
-        if (property->hasSubstitutions()) {
-          for (auto t : property->getSubstitutions()) {
-            if (t.containsNameSymbols())
-              result |= GenericSignatureErrorFlags::HasInvalidRequirements;
-          }
-        }
+        if (property->hasSubstitutions() &&
+            property->containsNameSymbols())
+          result |= GenericSignatureErrorFlags::HasInvalidRequirements;
       }
     }
   }
@@ -661,10 +659,20 @@ RewriteSystem::getMinimizedProtocolRules() const {
     if (!isInMinimizationDomain(proto))
       continue;
 
-    if (rule.isProtocolTypeAliasRule())
+    if (rule.isProtocolTypeAliasRule()) {
+      if (auto property = rule.isPropertyRule()) {
+        if (property->containsNameSymbols())
+          continue;
+      } else if (rule.getRHS().containsNameSymbols()) {
+        continue;
+      }
       rules[proto].TypeAliases.push_back(ruleID);
-    else if (!rule.containsNameSymbols())
+    } else {
+      if (rule.containsNameSymbols())
+        continue;
+
       rules[proto].Requirements.push_back(ruleID);
+    }
   }
 
   return rules;

lib/AST/RequirementMachine/Symbol.cpp

@@ -686,6 +686,15 @@ Symbol Symbol::transformConcreteSubstitutions(
   return withConcreteSubstitutions(substitutions, ctx);
 }
 
+bool Symbol::containsNameSymbols() const {
+  for (auto t : getSubstitutions()) {
+    if (t.containsNameSymbols())
+      return true;
+  }
+
+  return false;
+}
+
 /// Print the symbol using our mnemonic representation.
 void Symbol::dump(llvm::raw_ostream &out) const {
   llvm::DenseMap<CanType, Identifier> substitutionNames;

lib/AST/RequirementMachine/Symbol.h

@@ -242,6 +242,8 @@ class Symbol final {
       const MutableTerm &prefix,
       RewriteContext &ctx) const;
 
+  bool containsNameSymbols() const;
+
   void dump(llvm::raw_ostream &out) const;
 
   friend bool operator==(Symbol lhs, Symbol rhs) {

test/Generics/rdar136686001.swift

@@ -0,0 +1,23 @@
+// RUN: %empty-directory(%t)
+// RUN: %target-swift-frontend -emit-module %s -emit-module-path %t/out.swiftmodule
+// RUN: %target-swift-frontend -typecheck %s -debug-generic-signatures 2>&1 | %FileCheck %s
+
+public struct S {
+  public typealias A = Int
+}
+
+public protocol P {
+  typealias A = S
+}
+
+public struct G<T> {}
+
+public protocol Q: P {
+  typealias B = G<Self.A.A>
+}
+
+// FIXME: This should be diagnosed as an error.
+
+// CHECK-LABEL: rdar136686001.(file).f@
+// CHECK-NEXT: Generic signature: <T, U where T : Q>
+public func f<T: Q, U>(_: T, _: U) where T.B == U {}