Fix HashMap and HashSet remove criterion

Make criterion when to fill a hole in a HashMap or HashSet remove more robust.
The old criterion relied on fill factor always being less than 0.5.
The new criterion works for arbitrary fill factors.

Author: odersky

compiler/src/dotty/tools/dotc/util/GenericHashMap.scala

@@ -111,9 +111,11 @@ abstract class GenericHashMap[Key, Value]
           k = keyAt(idx)
           k != null
         do
+          val eidx = index(hash(k))
           if isDense
-            || index(hole - index(hash(k))) < limit * 2
-               // hash(k) is then logically at or before hole; can be moved forward to fill hole
+            || index(eidx - (hole + 2)) > index(idx - (hole + 2))
+               // entry `e` at `idx` can move unless `index(hash(e))` is in
+               // the (ring-)interval [hole + 2 .. idx]
           then
             setKey(hole, k)
             setValue(hole, valueAt(idx))

compiler/src/dotty/tools/dotc/util/HashSet.scala

@@ -109,9 +109,11 @@ class HashSet[T](initialCapacity: Int = 8, capacityMultiple: Int = 2) extends Mu
           e = entryAt(idx)
           e != null
         do
+          val eidx = index(hash(e))
           if isDense
-            || index(hole - index(hash(e))) < limit
-               // hash(k) is then logically at or before hole; can be moved forward to fill hole
+            || index(eidx - (hole + 1)) > index(idx - (hole + 1))
+               // entry `e` at `idx` can move unless `index(hash(e))` is in
+               // the (ring-)interval [hole + 1 .. idx]
           then
             setEntry(hole, e)
             hole = idx

tests/run-with-compiler/settest.scala

@@ -0,0 +1,71 @@
+trait Generator[+T]:
+  self =>
+  def generate: T
+  def map[S](f: T => S) = new Generator[S]:
+    def generate: S = f(self.generate)
+  def flatMap[S](f: T => Generator[S]) = new Generator[S]:
+    def generate: S = f(self.generate).generate
+
+object Generator:
+  val NumLimit = 300
+  val Iterations = 10000
+
+  given integers as Generator[Int]:
+    val rand = new java.util.Random
+    def generate = rand.nextInt()
+
+  given booleans as Generator[Boolean] =
+    integers.map(x => x > 0)
+
+  def range(end: Int): Generator[Int] =
+    integers.map(x => (x % end).abs)
+
+  enum Op:
+    case Lookup, Update, Remove
+  export Op._
+
+  given ops as Generator[Op] =
+    range(10).map {
+      case 0 | 1 | 2 | 3 => Lookup
+      case 4 | 5 | 6 | 7 => Update
+      case 8 | 9         => Remove
+    }
+
+  val nums: Generator[Integer] = range(NumLimit).map(Integer(_))
+
+@main def Test =
+  import Generator._
+
+  val set1 = dotty.tools.dotc.util.HashSet[Int]()
+  val set2 = scala.collection.mutable.HashSet[Int]()
+
+  def checkSame() =
+    assert(set1.size == set2.size)
+    for e <- set1.iterator do
+      assert(set2.contains(e))
+    for e <- set2.iterator do
+      assert(set1.contains(e))
+
+  def lookupTest(num: Integer) =
+    val res1 = set1.contains(num)
+    val res2 = set2.contains(num)
+    assert(res1 == res2)
+
+  def updateTest(num: Integer) =
+    lookupTest(num)
+    set1 += num
+    set2 += num
+    checkSame()
+
+  def removeTest(num: Integer) =
+    //println(s"test remove $num")
+    set1 -= num
+    set2 -= num
+    checkSame()
+
+  for i <- 0 until Iterations do
+    val num = nums.generate
+    Generator.ops.generate match
+      case Lookup => lookupTest(num)
+      case Update => updateTest(num)
+      case Remove => removeTest(num)