Merge pull request #7546 from abeln/dotty-explicit-nulls-typesystem

Dotty with explicit nulls (and flow typing)

Author: odersky

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -425,6 +425,9 @@ object Contexts {
     def useColors: Boolean =
       base.settings.color.value == "always"
 
+    /** Is the explicit nulls option set? */
+    def explicitNulls: Boolean = base.settings.YexplicitNulls.value
+
     protected def init(outer: Context, origin: Context): this.type = {
       util.Stats.record("Context.fresh")
       _outer = outer

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -4,7 +4,7 @@ package core
 
 import scala.annotation.{threadUnsafe => tu}
 import Types._, Contexts._, Symbols._, SymDenotations._, StdNames._, Names._
-import Flags._, Scopes._, Decorators._, NameOps._, Periods._
+import Flags._, Scopes._, Decorators._, NameOps._, Periods._, NullOpsDecorator._
 import unpickleScala2.Scala2Unpickler.ensureConstructor
 import scala.collection.mutable
 import collection.mutable
@@ -269,7 +269,7 @@ class Definitions {
     @tu lazy val Any_asInstanceOf: TermSymbol = enterT1ParameterlessMethod(AnyClass, nme.asInstanceOf_, _.paramRefs(0), Final)
     @tu lazy val Any_typeTest: TermSymbol     = enterT1ParameterlessMethod(AnyClass, nme.isInstanceOfPM, _ => BooleanType, Final | Synthetic | Artifact)
     @tu lazy val Any_typeCast: TermSymbol     = enterT1ParameterlessMethod(AnyClass, nme.asInstanceOfPM, _.paramRefs(0), Final | Synthetic | Artifact | StableRealizable)
-      // generated by pattern matcher, eliminated by erasure
+      // generated by pattern matcher and explicit nulls, eliminated by erasure
 
     /** def getClass[A >: this.type](): Class[? <: A] */
     @tu lazy val Any_getClass: TermSymbol =
@@ -347,11 +347,29 @@ class Definitions {
     ScalaPackageClass, tpnme.Nothing, AbstractFinal, List(AnyClass.typeRef))
   def NothingType: TypeRef = NothingClass.typeRef
   @tu lazy val RuntimeNothingModuleRef: TermRef = ctx.requiredModuleRef("scala.runtime.Nothing")
-  @tu lazy val NullClass: ClassSymbol = enterCompleteClassSymbol(
-    ScalaPackageClass, tpnme.Null, AbstractFinal, List(ObjectClass.typeRef))
+  @tu lazy val NullClass: ClassSymbol = {
+    val parent = if (ctx.explicitNulls) AnyType else ObjectType
+    enterCompleteClassSymbol(ScalaPackageClass, tpnme.Null, AbstractFinal, parent :: Nil)
+  }
   def NullType: TypeRef = NullClass.typeRef
   @tu lazy val RuntimeNullModuleRef: TermRef = ctx.requiredModuleRef("scala.runtime.Null")
 
+  /** An alias for null values that originate in Java code.
+   *  This type gets special treatment in the Typer. Specifically, `JavaNull` can be selected through:
+   *  e.g.
+   *  ```
+   *  // x: String|Null
+   *  x.length // error: `Null` has no `length` field
+   *  // x2: String|JavaNull
+   *  x2.length // allowed by the Typer, but unsound (might throw NPE)
+   *  ```
+   */
+  lazy val JavaNullAlias: TypeSymbol = {
+    assert(ctx.explicitNulls)
+    enterAliasType(tpnme.JavaNull, NullType)
+  }
+  def JavaNullAliasType: TypeRef = JavaNullAlias.typeRef
+
   @tu lazy val ImplicitScrutineeTypeSym =
     newSymbol(ScalaPackageClass, tpnme.IMPLICITkw, EmptyFlags, TypeBounds.empty).entered
   def ImplicitScrutineeTypeRef: TypeRef = ImplicitScrutineeTypeSym.typeRef
@@ -441,12 +459,12 @@ class Definitions {
     @tu lazy val Boolean_|| : Symbol = BooleanClass.requiredMethod(nme.ZOR)
     @tu lazy val Boolean_== : Symbol =
       BooleanClass.info.member(nme.EQ).suchThat(_.info.firstParamTypes match {
-        case List(pt) => (pt isRef BooleanClass)
+        case List(pt) => pt.isRef(BooleanClass)
         case _ => false
       }).symbol
     @tu lazy val Boolean_!= : Symbol =
       BooleanClass.info.member(nme.NE).suchThat(_.info.firstParamTypes match {
-        case List(pt) => (pt isRef BooleanClass)
+        case List(pt) => pt.isRef(BooleanClass)
         case _ => false
       }).symbol
 
@@ -509,7 +527,7 @@ class Definitions {
   @tu lazy val StringModule: Symbol = StringClass.linkedClass
     @tu lazy val String_+ : TermSymbol = enterMethod(StringClass, nme.raw.PLUS, methOfAny(StringType), Final)
     @tu lazy val String_valueOf_Object: Symbol = StringModule.info.member(nme.valueOf).suchThat(_.info.firstParamTypes match {
-      case List(pt) => (pt isRef AnyClass) || (pt isRef ObjectClass)
+      case List(pt) => pt.isRef(AnyClass) || pt.isRef(ObjectClass)
       case _ => false
     }).symbol
 
@@ -520,12 +538,16 @@ class Definitions {
   @tu lazy val BoxedNumberClass: ClassSymbol          = ctx.requiredClass("java.lang.Number")
   @tu lazy val ClassCastExceptionClass: ClassSymbol   = ctx.requiredClass("java.lang.ClassCastException")
     @tu lazy val ClassCastExceptionClass_stringConstructor: TermSymbol  = ClassCastExceptionClass.info.member(nme.CONSTRUCTOR).suchThat(_.info.firstParamTypes match {
-      case List(pt) => (pt isRef StringClass)
+      case List(pt) =>
+        val pt1 = if (ctx.explicitNulls) pt.stripNull() else pt
+        pt1.isRef(StringClass)
       case _ => false
     }).symbol.asTerm
   @tu lazy val ArithmeticExceptionClass: ClassSymbol  = ctx.requiredClass("java.lang.ArithmeticException")
     @tu lazy val ArithmeticExceptionClass_stringConstructor: TermSymbol  = ArithmeticExceptionClass.info.member(nme.CONSTRUCTOR).suchThat(_.info.firstParamTypes match {
-      case List(pt) => (pt isRef StringClass)
+      case List(pt) =>
+        val pt1 = if (ctx.explicitNulls) pt.stripNull() else pt
+        pt1.isRef(StringClass)
       case _ => false
     }).symbol.asTerm
 
@@ -793,6 +815,31 @@ class Definitions {
   @tu lazy val InfixAnnot: ClassSymbol = ctx.requiredClass("scala.annotation.infix")
   @tu lazy val AlphaAnnot: ClassSymbol = ctx.requiredClass("scala.annotation.alpha")
 
+  // A list of annotations that are commonly used to indicate that a field/method argument or return
+  // type is not null. These annotations are used by the nullification logic in JavaNullInterop to
+  // improve the precision of type nullification.
+  // We don't require that any of these annotations be present in the class path, but we want to
+  // create Symbols for the ones that are present, so they can be checked during nullification.
+  @tu lazy val NotNullAnnots: List[ClassSymbol] = ctx.getClassesIfDefined(
+    "javax.annotation.Nonnull" ::
+    "javax.validation.constraints.NotNull" ::
+    "androidx.annotation.NonNull" ::
+    "android.support.annotation.NonNull" ::
+    "android.annotation.NonNull" ::
+    "com.android.annotations.NonNull" ::
+    "org.eclipse.jdt.annotation.NonNull" ::
+    "edu.umd.cs.findbugs.annotations.NonNull" ::
+    "org.checkerframework.checker.nullness.qual.NonNull" ::
+    "org.checkerframework.checker.nullness.compatqual.NonNullDecl" ::
+    "org.jetbrains.annotations.NotNull" ::
+    "org.springframework.lang.NonNull" ::
+    "org.springframework.lang.NonNullApi" ::
+    "org.springframework.lang.NonNullFields" ::
+    "lombok.NonNull" ::
+    "reactor.util.annotation.NonNull" ::
+    "reactor.util.annotation.NonNullApi" ::
+    "io.reactivex.annotations.NonNull" :: Nil map PreNamedString)
+
   // convenient one-parameter method types
   def methOfAny(tp: Type): MethodType = MethodType(List(AnyType), tp)
   def methOfAnyVal(tp: Type): MethodType = MethodType(List(AnyValType), tp)
@@ -845,7 +892,7 @@ class Definitions {
       if (ctx.erasedTypes) JavaArrayType(elem)
       else ArrayType.appliedTo(elem :: Nil)
     def unapply(tp: Type)(implicit ctx: Context): Option[Type] = tp.dealias match {
-      case AppliedType(at, arg :: Nil) if at isRef ArrayType.symbol => Some(arg)
+      case AppliedType(at, arg :: Nil) if at.isRef(ArrayType.symbol) => Some(arg)
       case _ => None
     }
   }
@@ -957,8 +1004,16 @@ class Definitions {
       name.drop(prefix.length).forall(_.isDigit))
 
   def isBottomClass(cls: Symbol): Boolean =
-    cls == NothingClass || cls == NullClass
+    if (ctx.explicitNulls && !ctx.phase.erasedTypes) cls == NothingClass
+    else isBottomClassAfterErasure(cls)
+
+  def isBottomClassAfterErasure(cls: Symbol): Boolean = cls == NothingClass || cls == NullClass
+
   def isBottomType(tp: Type): Boolean =
+    if (ctx.explicitNulls && !ctx.phase.erasedTypes) tp.derivesFrom(NothingClass)
+    else isBottomTypeAfterErasure(tp)
+
+  def isBottomTypeAfterErasure(tp: Type): Boolean =
     tp.derivesFrom(NothingClass) || tp.derivesFrom(NullClass)
 
   /** Is a function class.
@@ -1089,7 +1144,7 @@ class Definitions {
 
   def isTupleType(tp: Type)(implicit ctx: Context): Boolean = {
     val arity = tp.dealias.argInfos.length
-    arity <= MaxTupleArity && TupleType(arity) != null && (tp isRef TupleType(arity).symbol)
+    arity <= MaxTupleArity && TupleType(arity) != null && tp.isRef(TupleType(arity).symbol)
   }
 
   def tupleType(elems: List[Type]): Type = {
@@ -1302,18 +1357,22 @@ class Definitions {
   // ----- Initialization ---------------------------------------------------
 
   /** Lists core classes that don't have underlying bytecode, but are synthesized on-the-fly in every reflection universe */
-  @tu lazy val syntheticScalaClasses: List[TypeSymbol] = List(
-    AnyClass,
-    AnyRefAlias,
-    AnyKindClass,
-    andType,
-    orType,
-    RepeatedParamClass,
-    ByNameParamClass2x,
-    AnyValClass,
-    NullClass,
-    NothingClass,
-    SingletonClass)
+  @tu lazy val syntheticScalaClasses: List[TypeSymbol] = {
+    val synth = List(
+      AnyClass,
+      AnyRefAlias,
+      AnyKindClass,
+      andType,
+      orType,
+      RepeatedParamClass,
+      ByNameParamClass2x,
+      AnyValClass,
+      NullClass,
+      NothingClass,
+      SingletonClass)
+
+    if (ctx.explicitNulls) synth :+ JavaNullAlias else synth
+  }
 
   @tu lazy val syntheticCoreClasses: List[Symbol] = syntheticScalaClasses ++ List(
     EmptyPackageVal,

compiler/src/dotty/tools/dotc/core/Flags.scala

@@ -452,7 +452,6 @@ object Flags {
   val AfterLoadFlags: FlagSet = commonFlags(
     FromStartFlags, AccessFlags, Final, AccessorOrSealed, LazyOrTrait, SelfName, JavaDefined)
 
-
   /** A value that's unstable unless complemented with a Stable flag */
   val UnstableValueFlags: FlagSet = Mutable | Method
 

compiler/src/dotty/tools/dotc/core/JavaNullInterop.scala

@@ -0,0 +1,150 @@
+package dotty.tools.dotc.core
+
+import dotty.tools.dotc.core.Contexts.Context
+import dotty.tools.dotc.core.Flags.JavaDefined
+import dotty.tools.dotc.core.StdNames.{jnme, nme}
+import dotty.tools.dotc.core.Symbols._
+import dotty.tools.dotc.core.Types._
+import NullOpsDecorator._
+
+/** This module defines methods to interpret types of Java symbols, which are implicitly nullable in Java,
+ *  as Scala types, which are explicitly nullable.
+ *
+ *  The transformation is (conceptually) a function `n` that adheres to the following rules:
+ *    (1) n(T)              = T|JavaNull              if T is a reference type
+ *    (2) n(T)              = T                       if T is a value type
+ *    (3) n(C[T])           = C[T]|JavaNull           if C is Java-defined
+ *    (4) n(C[T])           = C[n(T)]|JavaNull        if C is Scala-defined
+ *    (5) n(A|B)            = n(A)|n(B)|JavaNull
+ *    (6) n(A&B)            = n(A) & n(B)
+ *    (7) n((A1, ..., Am)R) = (n(A1), ..., n(Am))n(R) for a method with arguments (A1, ..., Am) and return type R
+ *    (8) n(T)              = T                       otherwise
+ *
+ *   Treatment of generics (rules 3 and 4):
+ *     - if `C` is Java-defined, then `n(C[T]) = C[T]|JavaNull`. That is, we don't recurse
+ *       on the type argument, and only add JavaNull on the outside. This is because
+ *       `C` itself will be nullified, and in particular so will be usages of `C`'s type argument within C's body.
+ *       e.g. calling `get` on a `java.util.List[String]` already returns `String|Null` and not `String`, so
+ *       we don't need to write `java.util.List[String|Null]`.
+ *     - if `C` is Scala-defined, however, then we want `n(C[T]) = C[n(T)]|JavaNull`. This is because
+ *       `C` won't be nullified, so we need to indicate that its type argument is nullable.
+ *
+ *   Notice that since the transformation is only applied to types attached to Java symbols, it doesn't need
+ *   to handle the full spectrum of Scala types. Additionally, some kinds of symbols like constructors and
+ *   enum instances get special treatment.
+ */
+object JavaNullInterop {
+
+  /** Transforms the type `tp` of Java member `sym` to be explicitly nullable.
+   *  `tp` is needed because the type inside `sym` might not be set when this method is called.
+   *
+   *  e.g. given a Java method
+   *  String foo(String arg) { return arg; }
+   *
+   *  After calling `nullifyMember`, Scala will see the method as
+   *
+   *  def foo(arg: String|JavaNull): String|JavaNull
+   *
+   *  This nullability function uses `JavaNull` instead of vanilla `Null`, for usability.
+   *  This means that we can select on the return of `foo`:
+   *
+   *  val len = foo("hello").length
+   *
+   *  But the selection can throw an NPE if the returned value is `null`.
+   */
+  def nullifyMember(sym: Symbol, tp: Type, isEnumValueDef: Boolean)(implicit ctx: Context): Type = {
+    assert(ctx.explicitNulls)
+    assert(sym.is(JavaDefined), "can only nullify java-defined members")
+
+    // Some special cases when nullifying the type
+    if (isEnumValueDef || sym.name == nme.TYPE_)
+      // Don't nullify the `TYPE` field in every class and Java enum instances
+      tp
+    else if (sym.name == nme.toString_ || sym.isConstructor || hasNotNullAnnot(sym))
+      // Don't nullify the return type of the `toString` method.
+      // Don't nullify the return type of constructors.
+      // Don't nullify the return type of methods with a not-null annotation.
+      nullifyExceptReturnType(tp)
+    else
+      // Otherwise, nullify everything
+      nullifyType(tp)
+  }
+
+  private def hasNotNullAnnot(sym: Symbol)(implicit ctx: Context): Boolean =
+    ctx.definitions.NotNullAnnots.exists(nna => sym.unforcedAnnotation(nna).isDefined)
+
+  /** If tp is a MethodType, the parameters and the inside of return type are nullified,
+   *  but the result return type is not nullable.
+   *  If tp is a type of a field, the inside of the type is nullified,
+   *  but the result type is not nullable.
+   */
+  private def nullifyExceptReturnType(tp: Type)(implicit ctx: Context): Type =
+    new JavaNullMap(true)(ctx)(tp)
+
+  /** Nullifies a Java type by adding `| JavaNull` in the relevant places. */
+  private def nullifyType(tp: Type)(implicit ctx: Context): Type =
+    new JavaNullMap(false)(ctx)(tp)
+
+  /** A type map that implements the nullification function on types. Given a Java-sourced type, this adds `| JavaNull`
+   *  in the right places to make the nulls explicit in Scala.
+   *
+   *  @param outermostLevelAlreadyNullable whether this type is already nullable at the outermost level.
+   *                                       For example, `Array[String]|JavaNull` is already nullable at the
+   *                                       outermost level, but `Array[String|JavaNull]` isn't.
+   *                                       If this parameter is set to true, then the types of fields, and the return
+   *                                       types of methods will not be nullified.
+   *                                       This is useful for e.g. constructors, and also so that `A & B` is nullified
+   *                                       to `(A & B) | JavaNull`, instead of `(A|JavaNull & B|JavaNull) | JavaNull`.
+   */
+  private class JavaNullMap(var outermostLevelAlreadyNullable: Boolean)(implicit ctx: Context) extends TypeMap {
+    /** Should we nullify `tp` at the outermost level? */
+    def needsNull(tp: Type): Boolean =
+      !outermostLevelAlreadyNullable && (tp match {
+        case tp: TypeRef =>
+          // We don't modify value types because they're non-nullable even in Java.
+          !tp.symbol.isValueClass &&
+          // We don't modify `Any` because it's already nullable.
+          !tp.isRef(defn.AnyClass) &&
+          // We don't nullify Java varargs at the top level.
+          // Example: if `setNames` is a Java method with signature `void setNames(String... names)`,
+          // then its Scala signature will be `def setNames(names: (String|JavaNull)*): Unit`.
+          // This is because `setNames(null)` passes as argument a single-element array containing the value `null`,
+          // and not a `null` array.
+          !tp.isRef(defn.RepeatedParamClass)
+        case _ => true
+      })
+
+    override def apply(tp: Type): Type = tp match {
+      case tp: TypeRef if needsNull(tp) => OrJavaNull(tp)
+      case appTp @ AppliedType(tycon, targs) =>
+        val oldOutermostNullable = outermostLevelAlreadyNullable
+        // We don't make the outmost levels of type arguements nullable if tycon is Java-defined.
+        // This is because Java classes are _all_ nullified, so both `java.util.List[String]` and
+        // `java.util.List[String|Null]` contain nullable elements.
+        outermostLevelAlreadyNullable = tp.classSymbol.is(JavaDefined)
+        val targs2 = targs map this
+        outermostLevelAlreadyNullable = oldOutermostNullable
+        val appTp2 = derivedAppliedType(appTp, tycon, targs2)
+        if (needsNull(tycon)) OrJavaNull(appTp2) else appTp2
+      case ptp: PolyType =>
+        derivedLambdaType(ptp)(ptp.paramInfos, this(ptp.resType))
+      case mtp: MethodType =>
+        val oldOutermostNullable = outermostLevelAlreadyNullable
+        outermostLevelAlreadyNullable = false
+        val paramInfos2 = mtp.paramInfos map this
+        outermostLevelAlreadyNullable = oldOutermostNullable
+        derivedLambdaType(mtp)(paramInfos2, this(mtp.resType))
+      case tp: TypeAlias => mapOver(tp)
+      case tp: AndType =>
+        // nullify(A & B) = (nullify(A) & nullify(B)) | JavaNull, but take care not to add
+        // duplicate `JavaNull`s at the outermost level inside `A` and `B`.
+        outermostLevelAlreadyNullable = true
+        OrJavaNull(derivedAndType(tp, this(tp.tp1), this(tp.tp2)))
+      case tp: TypeParamRef if needsNull(tp) => OrJavaNull(tp)
+      // In all other cases, return the type unchanged.
+      // In particular, if the type is a ConstantType, then we don't nullify it because it is the
+      // type of a final non-nullable field.
+      case _ => tp
+    }
+  }
+}