feat(persistQueryClient): persist error handling (#3556)

* refactor: remove type-fest as a dependency

only used for the Promisable type, which is easy to recreate

* feat(persistQueryClient): error handling strategies for persist plugins

* feat(persistQueryClient): error handling strategies for persist plugins

adapt tests

* make handlePersistError return null to stop retries

if null is returned, which is also the default strategy, the webstorage entry will be removed completely.

* test for default behaviour

* async version for persist error handling

to make sync and async compatible, persist version must also throw an error to abort

* make sure that async persister can accept sync error handlers

* undefined errorStrategy, or return undefined from it, will just not persist anymore

* rename to retry + documentation

* improve docs

Author: TkDodo

docs/src/pages/plugins/createAsyncStoragePersister.md

@@ -37,6 +37,10 @@ persistQueryClient({
 })
 ```
 
+## Retries
+
+Retries work the same as for a [WebStoragePersister](./createWebStoragePersister), except that they can also be asynchronous. You can also use all the predefined retry handlers.
+
 ## API
 
 ### `createAsyncStoragePersister`
@@ -62,6 +66,8 @@ interface CreateAsyncStoragePersisterOptions {
   serialize?: (client: PersistedClient) => string
   /** How to deserialize the data from storage */
   deserialize?: (cachedString: string) => PersistedClient
+  /** How to retry persistence on error **/
+  retry?: AsyncPersistRetryer
 }
 
 interface AsyncStorage {

docs/src/pages/plugins/createWebStoragePersister.md

@@ -34,6 +34,34 @@ persistQueryClient({
 })
 ```
 
+## Retries
+
+Persistence can fail, e.g. if the size exceeds the available space on the storage. Errors can be handled gracefully by providing a `retry` function to the persister.
+
+The retry function receives the `persistedClient` it tried to save, as well as the `error` and the `errorCount` as input. It is expected to return a _new_ `PersistedClient`, with which it tries to persist again. If _undefined_ is returned, there will be no further attempt to persist.
+
+```ts
+export type PersistRetryer = (props: {
+  persistedClient: PersistedClient
+  error: Error
+  errorCount: number
+}) => PersistedClient | undefined
+```
+
+### Predefined strategies
+
+Per default, no retry will occur. You can use one of the predefined strategies to handle retries. They can be imported `from 'react-query/persistQueryClient'`:
+
+- `removeOldestQuery`
+  - will return a new `PersistedClient` with the oldest query removed.
+
+```js
+const localStoragePersister = createWebStoragePersister({
+  storage: window.localStorage,
+  retry: removeOldestQuery
+})
+```
+
 ## API
 
 ### `createWebStoragePersister`
@@ -59,6 +87,8 @@ interface CreateWebStoragePersisterOptions {
   serialize?: (client: PersistedClient) => string
   /** How to deserialize the data from storage */
   deserialize?: (cachedString: string) => PersistedClient
+  /** How to retry persistence on error **/
+  retry?: PersistRetryer
 }
 ```
 
@@ -74,7 +104,7 @@ The default options are:
 ```
 
 #### `serialize` and `deserialize` options
-There is a limit to the amount of data which can be stored in `localStorage`. 
+There is a limit to the amount of data which can be stored in `localStorage`.
 If you need to store more data in `localStorage`, you can override the `serialize` and `deserialize` functions to compress and decrompress the data using a library like [lz-string](https://github.com/pieroxy/lz-string/).
 
 ```js

package.json

@@ -181,7 +181,6 @@
     "rollup-plugin-size": "^0.2.2",
     "rollup-plugin-terser": "^7.0.2",
     "rollup-plugin-visualizer": "^5.6.0",
-    "type-fest": "^0.21.0",
     "typescript": "4.5.3"
   },
   "bundlewatch": {

src/createAsyncStoragePersister/index.ts

@@ -1,15 +1,22 @@
-import { PersistedClient, Persister } from '../persistQueryClient'
+import { PersistedClient, Persister, Promisable } from '../persistQueryClient'
 import { asyncThrottle } from './asyncThrottle'
+import { noop } from '../core/utils'
 
 interface AsyncStorage {
   getItem: (key: string) => Promise<string | null>
   setItem: (key: string, value: string) => Promise<void>
   removeItem: (key: string) => Promise<void>
 }
 
+export type AsyncPersistRetryer = (props: {
+  persistedClient: PersistedClient
+  error: Error
+  errorCount: number
+}) => Promisable<PersistedClient | undefined>
+
 interface CreateAsyncStoragePersisterOptions {
   /** The storage client used for setting an retrieving items from cache */
-  storage: AsyncStorage
+  storage: AsyncStorage | undefined
   /** The key to use when storing the cache */
   key?: string
   /** To avoid spamming,
@@ -25,6 +32,8 @@ interface CreateAsyncStoragePersisterOptions {
    * @default `JSON.parse`
    */
   deserialize?: (cachedString: string) => PersistedClient
+
+  retry?: AsyncPersistRetryer
 }
 
 export const createAsyncStoragePersister = ({
@@ -33,21 +42,56 @@ export const createAsyncStoragePersister = ({
   throttleTime = 1000,
   serialize = JSON.stringify,
   deserialize = JSON.parse,
+  retry,
 }: CreateAsyncStoragePersisterOptions): Persister => {
-  return {
-    persistClient: asyncThrottle(
-      persistedClient => storage.setItem(key, serialize(persistedClient)),
-      { interval: throttleTime }
-    ),
-    restoreClient: async () => {
-      const cacheString = await storage.getItem(key)
-
-      if (!cacheString) {
-        return
+  if (typeof storage !== 'undefined') {
+    const trySave = async (
+      persistedClient: PersistedClient
+    ): Promise<Error | undefined> => {
+      try {
+        await storage.setItem(key, serialize(persistedClient))
+      } catch (error) {
+        return error as Error
       }
+    }
+
+    return {
+      persistClient: asyncThrottle(
+        async persistedClient => {
+          let client: PersistedClient | undefined = persistedClient
+          let error = await trySave(client)
+          let errorCount = 0
+          while (error && client) {
+            errorCount++
+            client = await retry?.({
+              persistedClient: client,
+              error,
+              errorCount,
+            })
+
+            if (client) {
+              error = await trySave(client)
+            }
+          }
+        },
+        { interval: throttleTime }
+      ),
+      restoreClient: async () => {
+        const cacheString = await storage.getItem(key)
 
-      return deserialize(cacheString) as PersistedClient
-    },
-    removeClient: () => storage.removeItem(key),
+        if (!cacheString) {
+          return
+        }
+
+        return deserialize(cacheString) as PersistedClient
+      },
+      removeClient: () => storage.removeItem(key),
+    }
+  }
+
+  return {
+    persistClient: noop,
+    restoreClient: noop,
+    removeClient: noop,
   }
 }

src/createWebStoragePersister/index.ts

@@ -1,9 +1,13 @@
 import { noop } from '../core/utils'
-import { PersistedClient, Persister } from '../persistQueryClient'
+import {
+  PersistedClient,
+  Persister,
+  PersistRetryer,
+} from '../persistQueryClient'
 
 interface CreateWebStoragePersisterOptions {
-  /** The storage client used for setting an retrieving items from cache */
-  storage: Storage
+  /** The storage client used for setting and retrieving items from cache */
+  storage: Storage | undefined
   /** The key to use when storing the cache */
   key?: string
   /** To avoid spamming,
@@ -19,6 +23,8 @@ interface CreateWebStoragePersisterOptions {
    * @default `JSON.parse`
    */
   deserialize?: (cachedString: string) => PersistedClient
+
+  retry?: PersistRetryer
 }
 
 export function createWebStoragePersister({
@@ -27,46 +33,31 @@ export function createWebStoragePersister({
   throttleTime = 1000,
   serialize = JSON.stringify,
   deserialize = JSON.parse,
+  retry,
 }: CreateWebStoragePersisterOptions): Persister {
-  //try to save data to storage
-  function trySave(persistedClient: PersistedClient) {
-    try {
-      storage.setItem(key, serialize(persistedClient))
-    } catch {
-      return false
-    }
-    return true
-  }
-
   if (typeof storage !== 'undefined') {
+    const trySave = (persistedClient: PersistedClient): Error | undefined => {
+      try {
+        storage.setItem(key, serialize(persistedClient))
+      } catch (error) {
+        return error as Error
+      }
+    }
     return {
       persistClient: throttle(persistedClient => {
-        if (trySave(persistedClient) !== true) {
-          const mutations = [...persistedClient.clientState.mutations]
-          const queries = [...persistedClient.clientState.queries]
-          const client: PersistedClient = {
-            ...persistedClient,
-            clientState: { mutations, queries },
-          }
-
-          // sort queries by dataUpdatedAt (oldest first)
-          const sortedQueries = [...queries].sort(
-            (a, b) => a.state.dataUpdatedAt - b.state.dataUpdatedAt
-          )
-          // clean old queries and try to save
-          while (sortedQueries.length > 0) {
-            const oldestData = sortedQueries.shift()
-            client.clientState.queries = queries.filter(q => q !== oldestData)
-            if (trySave(client)) {
-              return // save success
-            }
-          }
+        let client: PersistedClient | undefined = persistedClient
+        let error = trySave(client)
+        let errorCount = 0
+        while (error && client) {
+          errorCount++
+          client = retry?.({
+            persistedClient: client,
+            error,
+            errorCount,
+          })
 
-          // clean mutations and try to save
-          while (mutations.shift()) {
-            if (trySave(client)) {
-              return // save success
-            }
+          if (client) {
+            error = trySave(client)
           }
         }
       }, throttleTime),

src/createWebStoragePersister/tests/storageIsFull.test.ts

@@ -1,5 +1,6 @@
 import { dehydrate, MutationCache, QueryCache, QueryClient } from '../../core'
 import { createWebStoragePersister } from '../index'
+import { removeOldestQuery } from '../../persistQueryClient'
 import { sleep } from '../../tests/utils'
 
 function getMockStorage(limitSize?: number) {
@@ -11,7 +12,7 @@ function getMockStorage(limitSize?: number) {
     },
 
     setItem(key: string, value: string) {
-      if (limitSize) {
+      if (typeof limitSize !== 'undefined') {
         const currentSize = Array.from(dataSet.entries()).reduce(
           (n, d) => d[0].length + d[1].length + n,
           0
@@ -21,7 +22,7 @@ function getMockStorage(limitSize?: number) {
           limitSize
         ) {
           throw Error(
-            ` Failed to execute 'setItem' on 'Storage': Setting the value of '${key}' exceeded the quota.`
+            `Failed to execute 'setItem' on 'Storage': Setting the value of '${key}' exceeded the quota.`
           )
         }
       }
@@ -74,6 +75,7 @@ describe('createWebStoragePersister ', () => {
     const webStoragePersister = createWebStoragePersister({
       throttleTime: 0,
       storage,
+      retry: removeOldestQuery,
     })
 
     await queryClient.prefetchQuery(['A'], () => Promise.resolve('A'.repeat(N)))
@@ -121,55 +123,30 @@ describe('createWebStoragePersister ', () => {
       restoredClient2?.clientState.queries.find(q => q.queryKey[0] === 'B')
     ).toBeUndefined()
   })
-
-  test('should clean queries before mutations when storage full', async () => {
+  test('should clear storage as default error handling', async () => {
     const queryCache = new QueryCache()
     const mutationCache = new MutationCache()
     const queryClient = new QueryClient({ queryCache, mutationCache })
 
     const N = 2000
-    const storage = getMockStorage(N * 5) // can save 4 items;
+    const storage = getMockStorage(0)
     const webStoragePersister = createWebStoragePersister({
       throttleTime: 0,
       storage,
+      retry: removeOldestQuery,
     })
 
-    mutationCache.build(
-      queryClient,
-      {
-        mutationKey: ['MUTATIONS'],
-        mutationFn: () => Promise.resolve('M'.repeat(N)),
-      },
-      {
-        error: null,
-        context: '',
-        failureCount: 1,
-        isPaused: true,
-        status: 'success',
-        variables: '',
-        data: 'M'.repeat(N),
-      }
-    )
-    await sleep(1)
     await queryClient.prefetchQuery(['A'], () => Promise.resolve('A'.repeat(N)))
     await sleep(1)
-    await queryClient.prefetchQuery(['B'], () => Promise.resolve('B'.repeat(N)))
-    await queryClient.prefetchQuery(['C'], () => Promise.resolve('C'.repeat(N)))
-    await sleep(1)
-    await queryClient.prefetchQuery(['D'], () => Promise.resolve('D'.repeat(N)))
 
     const persistClient = {
-      buster: 'test-limit-mutations',
+      buster: 'test-limit',
       timestamp: Date.now(),
       clientState: dehydrate(queryClient),
     }
     webStoragePersister.persistClient(persistClient)
     await sleep(10)
     const restoredClient = await webStoragePersister.restoreClient()
-    expect(restoredClient?.clientState.mutations.length).toEqual(1)
-    expect(restoredClient?.clientState.queries.length).toEqual(3)
-    expect(
-      restoredClient?.clientState.queries.find(q => q.queryKey === ['A'])
-    ).toBeUndefined()
+    expect(restoredClient).toEqual(undefined)
   })
 })

src/persistQueryClient/index.ts

@@ -1,2 +1,3 @@
 export * from './persist'
 export * from './PersistQueryClientProvider'
+export * from './retryStrategies'

src/persistQueryClient/persist.ts

@@ -6,7 +6,8 @@ import {
   HydrateOptions,
   hydrate,
 } from '../core'
-import { Promisable } from 'type-fest'
+
+export type Promisable<T> = T | PromiseLike<T>
 
 export interface Persister {
   persistClient(persistClient: PersistedClient): Promisable<void>