[FSSDK-9061] fix retries on odp event failures (#487)

- Add support for retries (up to 3 times) when ODP event dispatch fails on connection or server errors. ODP events are discarded after max retries.
- Check reachability before dispatching to avoid discarding all events in the queue when network is down.

Author: jaeopt

.github/workflows/swift.yml

@@ -47,8 +47,7 @@ jobs:
 
   unittests: 
     if: "${{ github.event.inputs.PREP == '' && github.event.inputs.RELEASE == '' }}" 
-    ##uses: optimizely/swift-sdk/.github/workflows/unit_tests.yml@master
-    uses: optimizely/swift-sdk/.github/workflows/unit_tests.yml@jae/empty-odp-action
+    uses: optimizely/swift-sdk/.github/workflows/unit_tests.yml@master
 
   prepare_for_release:
     runs-on: macos-12

Sources/Customization/DefaultEventDispatcher.swift

@@ -207,6 +207,13 @@ open class DefaultEventDispatcher: BackgroundingCallbacks, OPTEventDispatcher {
         let config = URLSessionConfiguration.ephemeral
         return URLSession(configuration: config)
     }
+    
+    // MARK: - Tests
+
+    open func close() {
+        self.flushEvents()
+        self.queueLock.sync {}
+    }
 
 }
 
@@ -258,11 +265,4 @@ extension DefaultEventDispatcher {
         timer.property = nil
     }
 
-    // MARK: - Tests
-
-    open func close() {
-        self.flushEvents()
-        self.queueLock.sync {}
-    }
-    
 }

Sources/ODP/OdpEventManager.swift

@@ -23,9 +23,11 @@ open class OdpEventManager {
 
     var maxQueueSize = 100
     let maxBatchEvents = 10
+    let maxFailureCount = 3
     let queueLock: DispatchQueue
     let eventQueue: DataStoreQueueStackImpl<OdpEvent>
 
+    let reachability = NetworkReachability(maxContiguousFails: 1)
     let logger = OPTLoggerFactory.getLogger()
 
     /// OdpEventManager init
@@ -124,6 +126,12 @@ open class OdpEventManager {
             return
         }
 
+        // check if network is down to avoid that all existing events in the queue get discarded when network is down
+        if reachability.shouldBlockNetworkAccess() {
+            logger.e(.eventDispatchFailed("NetworkReachability down for ODP events"))
+            return
+        }
+        
         queueLock.async {
             func removeStoredEvents(num: Int) {
                 if let removedItem = self.eventQueue.removeFirstItems(count: num), removedItem.count > 0 {
@@ -137,6 +145,7 @@ open class OdpEventManager {
             // sync group used to ensure that the sendEvent is synchronous.
             // used in flushEvents
             let sync = DispatchGroup()
+            var failureCount = 0
 
             while let events: [OdpEvent] = self.eventQueue.getFirstItems(count: self.maxBatchEvents) {
                 let numEvents = events.count
@@ -151,31 +160,36 @@ open class OdpEventManager {
                 self.apiMgr.sendOdpEvents(apiKey: odpApiKey,
                                           apiHost: odpApiHost,
                                           events: events) { error in
+                    if let error = error {
+                        self.logger.e(error.reason)
+                    }
+
                     odpError = error
                     sync.leave()  // our send is done.
                 }
                 sync.wait()  // wait for send completed
 
-                if let error = odpError {
-                    self.logger.e(error.reason)
+                // retry only for recoverable errors (connection failures or 5xx server errors)
+                if let error = odpError, case .odpEventFailed(_, let canRetry) = error, canRetry {
+                    failureCount += 1
 
-                    // retry only if needed (non-permanent)
-                    if case .odpEventFailed(_, let canRetry) = error {
-                        if canRetry {
-                            // keep the failed event queue so it can be re-sent later
-                            break
-                        } else {
-                            // permanent errors (400 response or invalid events, etc)
-                            // discard these events so that they do not block following valid events
-                        }
+                    if failureCount >= self.maxFailureCount {
+                        self.logger.e(.odpEventSendRetyFailed(failureCount))
+                        failureCount = 0
                     }
+                } else { // success or non-recoverable errors
+                    failureCount = 0
+                }
+                                
+                if failureCount == 0 {
+                    removeStoredEvents(num: numEvents)
                 }
 
-                removeStoredEvents(num: numEvents)
+                self.reachability.updateNumContiguousFails(isError: odpError != nil)
             }
         }
     }
-    
+        
     func reset() {
         _ = eventQueue.removeFirstItems(count: self.maxQueueSize)
     }

Sources/Utils/LogMessage.swift

@@ -63,6 +63,7 @@ enum LogMessage {
     case unrecognizedAttribute(_ key: String)
     case eventBatchFailed
     case eventSendRetyFailed(_ count: Int)
+    case odpEventSendRetyFailed(_ count: Int)
     case failedToConvertMapToString
     case failedToAssignValue
     case valueForKeyNotFound(_ key: String)
@@ -123,8 +124,9 @@ extension LogMessage: CustomStringConvertible {
         case .unrecognizedAttribute(let key):                                   message = "Unrecognized attribute (\(key)) provided. Pruning before sending event to Optimizely."
         case .eventBatchFailed:                                                 message = "Failed to batch events"
         case .eventSendRetyFailed(let count):                                   message = "Event dispatch retries failed (\(count)) times"
+        case .odpEventSendRetyFailed(let count):                                message = "ODP event dispatch retries failed (\(count)) times"
         case .failedToConvertMapToString:                                       message = "Provided map could not be converted to string."
-        case .failedToAssignValue:                                      message = "Value for path could not be assigned to provided type."
+        case .failedToAssignValue:                                              message = "Value for path could not be assigned to provided type."
         case .valueForKeyNotFound(let key):                                     message = "Value for JSON key (\(key)) not found."
         }