Retry prim_file:rename on eacces error

On Windows, when creating a snapshot (renaming a checkpoint),
occasionally prim_file:rename failed with `eacces`. It's not clear
why this happens, it appears that Windows may not release the file
handle immediately after closing the file so a close->rename in
a quick succession may sometimes return an error.
With this commit, we just retry after 20ms and so far, in our testing,
the error has never occurred on the second attempt (with a 10ms
delay, it still failed every now and then)

We could have separate macros for eagain and eaccess, but I went with
a share one. One retry seems reasonable in both cases.

Author: mkuratczyk

src/ra_file.erl

@@ -9,14 +9,14 @@
 
 -include("ra.hrl").
 
--define(HANDLE_EAGAIN(Op),
+-define(RETRY_ON_ERROR(Op),
     case Op of
-        {error, eagain} ->
-            ?DEBUG("EAGAIN during file operation, retrying once in 10ms...", []),
-            timer:sleep(10),
+        {error, E} when E =:= eagain orelse E =:= eacces ->
+            ?DEBUG("Error `~p` during file operation, retrying once in 20ms...", [E]),
+            timer:sleep(20),
             case Op of
                 {error, eagain} = Err ->
-                    ?DEBUG("EAGAIN again during file operation", []),
+                    ?DEBUG("Error `~p` again during file operation", [E]),
                     Err;
                 Res ->
                     Res
@@ -26,8 +26,12 @@
     end).
 
 -export([
-         sync/1
+         sync/1,
+         rename/2
         ]).
 
 sync(Fd) ->
-    ?HANDLE_EAGAIN(file:sync(Fd)).
+    ?RETRY_ON_ERROR(file:sync(Fd)).
+
+rename(Src, Dst) ->
+    ?RETRY_ON_ERROR(prim_file:rename(Src, Dst)).

src/ra_snapshot.erl

@@ -417,7 +417,7 @@ promote_checkpoint(PromotionIdx,
                                 %% sync the checkpoint before promoting it
                                 %% into a snapshot.
                                 ok = Mod:sync(Checkpoint),
-                                ok = prim_file:rename(Checkpoint, Snapshot),
+                                ok = ra_file:rename(Checkpoint, Snapshot),
                                 Self ! {ra_log_event,
                                         {snapshot_written,
                                          {Idx, Term}, snapshot}}