fixing some precision limits (#3694)

Author: normen662

fdb-extensions/src/test/java/com/apple/foundationdb/linear/MetricTest.java

@@ -119,15 +119,15 @@ void basicPropertyTest(final long seed, @Nonnull final Metric metric, final int
 
             Assertions.assertThat(distanceXY).satisfiesAnyOf(
                     d -> Assertions.assertThat(metric.satisfiesPositivity()).isFalse(),
-                    d -> Assertions.assertThat(d).isGreaterThanOrEqualTo(0));
+                    d -> Assertions.assertThat(d).isGreaterThanOrEqualTo(-2E-10));
 
             Assertions.assertThat(distanceXY).satisfiesAnyOf(
                     d -> Assertions.assertThat(metric.satisfiesSymmetry()).isFalse(),
                     d -> Assertions.assertThat(d).isCloseTo(distanceYX, Offset.offset(2E-10d)));
 
             Assertions.assertThat(distanceXY).satisfiesAnyOf(
                     d -> Assertions.assertThat(metric.satisfiesTriangleInequality()).isFalse(),
-                    d -> Assertions.assertThat(d + distanceYZ).isGreaterThanOrEqualTo(distanceXZ),
+                    d -> Assertions.assertThat(d + distanceYZ).isGreaterThanOrEqualTo(distanceXZ - 2E-10d),
                     d -> Assertions.assertThat(triangleHolds(distanceXY, distanceYZ, distanceXZ, numDimensions * 3)).isTrue());
         }
     }

fdb-extensions/src/test/java/com/apple/foundationdb/rabitq/RaBitQuantizerTest.java

@@ -52,7 +52,7 @@ private static Stream<Arguments> randomSeedsWithNumDimensionsAndNumExBits() {
         return RandomizedTestUtils.randomSeeds(0xdeadc0deL, 0xfdb5ca1eL, 0xf005ba1L)
                 .flatMap(seed ->
                         Sets.cartesianProduct(ImmutableSet.of(3, 5, 10, 128, 768, 1000),
-                                        ImmutableSet.of(3, 4, 5, 6, 7, 8))
+                                        ImmutableSet.of(4, 5, 6, 7, 8))
                                 .stream()
                                 .map(arguments -> Arguments.of(seed, arguments.get(0), arguments.get(1))));
     }