Retry with exponential backoff

This commit changes default retry policy from fixed backoff to exponential
backoff with random jitter. Retry policy is user when retrying transactions
executed via `Session#readTransaction()` and `Session#writeTransaction()`
methods. Exponential backoff with random jitter is a better policy because
it spreads out clusters of retries and progressively reduces load on
the database.

The only configuration option available is:
`ConfigBuilder#withMaxTransactionRetryTime()`
which sets a cap on how long particular transaction can be retried with
exponential backoff.

Author: lutovich

driver/src/main/java/org/neo4j/driver/internal/DriverFactory.java

@@ -28,10 +28,10 @@
 import org.neo4j.driver.internal.net.SocketConnector;
 import org.neo4j.driver.internal.net.pooling.PoolSettings;
 import org.neo4j.driver.internal.net.pooling.SocketConnectionPool;
+import org.neo4j.driver.internal.retry.ExponentialBackoff;
 import org.neo4j.driver.internal.retry.RetryDecision;
 import org.neo4j.driver.internal.retry.RetryLogic;
 import org.neo4j.driver.internal.retry.RetrySettings;
-import org.neo4j.driver.internal.retry.RetryWithDelay;
 import org.neo4j.driver.internal.security.SecurityPlan;
 import org.neo4j.driver.internal.spi.ConnectionPool;
 import org.neo4j.driver.internal.spi.ConnectionProvider;
@@ -57,7 +57,7 @@ public final Driver newInstance( URI uri, AuthToken authToken, RoutingSettings r
         BoltServerAddress address = BoltServerAddress.from( uri );
         SecurityPlan securityPlan = createSecurityPlan( address, config );
         ConnectionPool connectionPool = createConnectionPool( authToken, securityPlan, config );
-        RetryLogic<RetryDecision> retryLogic = RetryWithDelay.create( retrySettings, createClock() );
+        RetryLogic<RetryDecision> retryLogic = createRetryLogic( retrySettings );
 
         try
         {
@@ -195,6 +195,16 @@ protected SessionFactory createSessionFactory( ConnectionProvider connectionProv
         return new SessionFactoryImpl( connectionProvider, retryLogic, config );
     }
 
+    /**
+     * Creates new {@link RetryLogic<RetryDecision>}.
+     * <p>
+     * <b>This method is protected only for testing</b>
+     */
+    protected RetryLogic<RetryDecision> createRetryLogic( RetrySettings settings )
+    {
+        return ExponentialBackoff.create( settings, createClock() );
+    }
+
     private static SecurityPlan createSecurityPlan( BoltServerAddress address, Config config )
     {
         try

driver/src/main/java/org/neo4j/driver/internal/retry/ExponentialBackoff.java

@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2002-2017 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.neo4j.driver.internal.retry;
+
+import java.util.Objects;
+import java.util.concurrent.ThreadLocalRandom;
+
+import org.neo4j.driver.internal.util.Clock;
+import org.neo4j.driver.v1.exceptions.ServiceUnavailableException;
+import org.neo4j.driver.v1.exceptions.SessionExpiredException;
+import org.neo4j.driver.v1.exceptions.TransientException;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+
+public class ExponentialBackoff implements RetryLogic<ExponentialBackoffDecision>
+{
+    public static final long DEFAULT_MAX_RETRY_TIME_MS = SECONDS.toMillis( 30 );
+
+    private static final long INITIAL_RETRY_DELAY_MS = SECONDS.toMillis( 1 );
+    private static final double RETRY_DELAY_MULTIPLIER = 2.0;
+    private static final double RETRY_DELAY_JITTER_FACTOR = 0.2;
+
+    private final long maxRetryTimeMs;
+    private final long initialRetryDelayMs;
+    private final double multiplier;
+    private final double jitterFactor;
+    private final Clock clock;
+
+    ExponentialBackoff( long maxRetryTimeMs, long initialRetryDelayMs, double multiplier, double jitterFactor,
+            Clock clock )
+    {
+        this.maxRetryTimeMs = maxRetryTimeMs;
+        this.initialRetryDelayMs = initialRetryDelayMs;
+        this.multiplier = multiplier;
+        this.jitterFactor = jitterFactor;
+        this.clock = clock;
+
+        verifyAfterConstruction();
+    }
+
+    public static RetryLogic<RetryDecision> defaultRetryLogic()
+    {
+        return create( RetrySettings.DEFAULT, Clock.SYSTEM );
+    }
+
+    public static RetryLogic<RetryDecision> noRetries()
+    {
+        return create( new RetrySettings( 0 ), Clock.SYSTEM );
+    }
+
+    @SuppressWarnings( "unchecked" )
+    public static RetryLogic<RetryDecision> create( RetrySettings settings, Clock clock )
+    {
+        return (RetryLogic) new ExponentialBackoff( settings.maxRetryTimeMs(), INITIAL_RETRY_DELAY_MS,
+                RETRY_DELAY_MULTIPLIER, RETRY_DELAY_JITTER_FACTOR, clock );
+    }
+
+    @Override
+    public ExponentialBackoffDecision apply( Throwable error, ExponentialBackoffDecision previousDecision )
+    {
+        Objects.requireNonNull( error );
+        ExponentialBackoffDecision decision = decision( previousDecision );
+
+        long elapsedTimeMs = clock.millis() - decision.startTimestamp();
+        if ( elapsedTimeMs > maxRetryTimeMs || !canRetryOn( error ) )
+        {
+            return decision.stopRetrying();
+        }
+
+        long delayWithJitterMs = computeDelayWithJitter( decision.delay() );
+        sleep( delayWithJitterMs );
+
+        long nextDelayWithoutJitterMs = (long) (decision.delay() * multiplier);
+        return decision.withDelay( nextDelayWithoutJitterMs );
+    }
+
+    private ExponentialBackoffDecision decision( ExponentialBackoffDecision previous )
+    {
+        return previous == null ? new ExponentialBackoffDecision( clock.millis(), initialRetryDelayMs ) : previous;
+    }
+
+    private long computeDelayWithJitter( long delayMs )
+    {
+        long jitter = (long) (delayMs * jitterFactor);
+        long min = delayMs - jitter;
+        long max = delayMs + jitter;
+        if ( max < 0 )
+        {
+            // overflow detected, truncate min and max values
+            min -= 1;
+            max = min;
+        }
+        return ThreadLocalRandom.current().nextLong( min, max + 1 );
+    }
+
+    private void sleep( long delayMs )
+    {
+        try
+        {
+            clock.sleep( delayMs );
+        }
+        catch ( InterruptedException e )
+        {
+            Thread.currentThread().interrupt();
+            throw new IllegalStateException( "Retries interrupted", e );
+        }
+    }
+
+    private void verifyAfterConstruction()
+    {
+        if ( maxRetryTimeMs < 0 )
+        {
+            throw new IllegalArgumentException( "Max retry time should be >= 0: " + maxRetryTimeMs );
+        }
+        if ( initialRetryDelayMs < 0 )
+        {
+            throw new IllegalArgumentException( "Initial retry delay should >= 0: " + initialRetryDelayMs );
+        }
+        if ( multiplier < 1.0 )
+        {
+            throw new IllegalArgumentException( "Multiplier should be >= 1.0: " + multiplier );
+        }
+        if ( jitterFactor < 0 || jitterFactor > 1 )
+        {
+            throw new IllegalArgumentException( "Jitter factor should be in [0.0, 1.0]: " + jitterFactor );
+        }
+        if ( clock == null )
+        {
+            throw new IllegalArgumentException( "Clock should not be null" );
+        }
+    }
+
+    private static boolean canRetryOn( Throwable error )
+    {
+        return error instanceof SessionExpiredException ||
+               error instanceof ServiceUnavailableException ||
+               error instanceof TransientException;
+    }
+}

driver/src/main/java/org/neo4j/driver/internal/retry/RetryWithDelayDecision.java renamed to driver/src/main/java/org/neo4j/driver/internal/retry/ExponentialBackoffDecision.java

@@ -18,23 +18,16 @@
  */
 package org.neo4j.driver.internal.retry;
 
-class RetryWithDelayDecision implements RetryDecision
+public class ExponentialBackoffDecision implements RetryDecision
 {
-    private int attempt = 1;
+    private final long startTimestamp;
+    private long delay;
     private boolean shouldRetry = true;
 
-    RetryWithDelayDecision()
+    ExponentialBackoffDecision( long startTimestamp, long initialDelay )
     {
-    }
-
-    /**
-     * Create a decision with the specified attempt.
-     * <p>
-     * <b>Note:</b> only for testing.
-     */
-    RetryWithDelayDecision( int attempt )
-    {
-        this.attempt = attempt;
+        this.startTimestamp = startTimestamp;
+        this.delay = initialDelay;
     }
 
     @Override
@@ -43,22 +36,23 @@ public boolean shouldRetry()
         return shouldRetry;
     }
 
-    int attempt()
+    long startTimestamp()
+    {
+        return startTimestamp;
+    }
+
+    long delay()
     {
-        return attempt;
+        return delay;
     }
 
-    RetryWithDelayDecision incrementAttempt()
+    ExponentialBackoffDecision withDelay( long delay )
     {
-        attempt++;
-        if ( attempt == Integer.MAX_VALUE )
-        {
-            stopRetrying();
-        }
+        this.delay = delay;
         return this;
     }
 
-    RetryWithDelayDecision stopRetrying()
+    ExponentialBackoffDecision stopRetrying()
     {
         shouldRetry = false;
         return this;

driver/src/main/java/org/neo4j/driver/internal/retry/RetrySettings.java

@@ -20,26 +20,17 @@
 
 public final class RetrySettings
 {
-    public static final int DEFAULT_MAX_ATTEMPTS = 3;
-    public static final int DEFAULT_DELAY_MS = 2_000;
-    public static final RetrySettings DEFAULT = new RetrySettings( DEFAULT_MAX_ATTEMPTS, DEFAULT_DELAY_MS );
+    public static final RetrySettings DEFAULT = new RetrySettings( ExponentialBackoff.DEFAULT_MAX_RETRY_TIME_MS );
 
-    private final int maxAttempts;
-    private final long delayMs;
+    private final long maxRetryTimeMs;
 
-    public RetrySettings( int maxAttempts, long delayMs )
+    public RetrySettings( long maxRetryTimeMs )
     {
-        this.maxAttempts = maxAttempts;
-        this.delayMs = delayMs;
+        this.maxRetryTimeMs = maxRetryTimeMs;
     }
 
-    public int maxAttempts()
+    public long maxRetryTimeMs()
     {
-        return maxAttempts;
-    }
-
-    public long delayMs()
-    {
-        return delayMs;
+        return maxRetryTimeMs;
     }
 }