gcs: mitigate transient errors causing corruption

For long backups, there appears to be a relatively high chance
of random issue like a 503. When these issues, it appears that
the clickhouse-backup retries are not sufficient, as we are seeing
missing table parts get reported.

This is a mitigation because in theory, the wrapping retries would
correctly retry these issues, however it does not appear to be the
case. More understanding is needed to fix that particular issue.

Author: devoxel

pkg/storage/gcs.go

@@ -279,6 +279,8 @@ func (gcs *GCS) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser
 	}
 	pClient := pClientObj.(*clientObject).Client
 	obj := pClient.Bucket(gcs.Config.Bucket).Object(key)
+	// always retry transient errors to mitigate retry logic bugs.
+	obj = obj.Retryer(storage.WithPolicy(storage.RetryAlways))
 	writer := obj.NewWriter(ctx)
 	writer.ChunkSize = gcs.Config.ChunkSize
 	writer.StorageClass = gcs.Config.StorageClass
@@ -365,6 +367,8 @@ func (gcs *GCS) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey
 	pClient := pClientObj.(*clientObject).Client
 	src := pClient.Bucket(srcBucket).Object(srcKey)
 	dst := pClient.Bucket(gcs.Config.Bucket).Object(dstKey)
+	// always retry transient errors to mitigate retry logic bugs.
+	dst = dst.Retryer(storage.WithPolicy(storage.RetryAlways))
 	attrs, err := src.Attrs(ctx)
 	if err != nil {
 		if pErr := gcs.clientPool.InvalidateObject(ctx, pClientObj); pErr != nil {