core, graph: Make maximum backoff for file data sources configurable

Author: lutter

core/src/polling_monitor/mod.rs

@@ -10,6 +10,7 @@ use std::task::Poll;
 use std::time::Duration;
 
 use graph::cheap_clone::CheapClone;
+use graph::env::ENV_VARS;
 use graph::futures03::future::BoxFuture;
 use graph::futures03::stream::StreamExt;
 use graph::futures03::{stream, Future, FutureExt, TryFutureExt};
@@ -29,8 +30,6 @@ pub use ipfs_service::{ipfs_service, IpfsService};
 
 const MIN_BACKOFF: Duration = Duration::from_secs(5);
 
-const MAX_BACKOFF: Duration = Duration::from_secs(600);
-
 struct Backoffs<ID> {
     backoff_maker: ExponentialBackoffMaker,
     backoffs: HashMap<ID, ExponentialBackoff>,
@@ -42,7 +41,7 @@ impl<ID: Eq + Hash> Backoffs<ID> {
         Self {
             backoff_maker: ExponentialBackoffMaker::new(
                 MIN_BACKOFF,
-                MAX_BACKOFF,
+                ENV_VARS.mappings.fds_max_backoff,
                 1.0,
                 HasherRng::new(),
             )

graph/src/env/mappings.rs

@@ -78,6 +78,10 @@ pub struct EnvVarsMapping {
     /// measure and can be removed after 2025-07-01, once we are sure the
     /// new behavior works as intended.
     pub store_errors_are_nondeterministic: bool,
+
+    /// Maximum backoff time for FDS requests. Set by
+    /// `GRAPH_FDS_MAX_BACKOFF` in seconds, defaults to 600.
+    pub fds_max_backoff: Duration,
 }
 
 // This does not print any values avoid accidentally leaking any sensitive env vars
@@ -116,6 +120,7 @@ impl TryFrom<InnerMappingHandlers> for EnvVarsMapping {
             allow_non_deterministic_ipfs: x.allow_non_deterministic_ipfs.0,
             disable_declared_calls: x.disable_declared_calls.0,
             store_errors_are_nondeterministic: x.store_errors_are_nondeterministic.0,
+            fds_max_backoff: Duration::from_secs(x.fds_max_backoff),
         };
         Ok(vars)
     }
@@ -157,6 +162,8 @@ pub struct InnerMappingHandlers {
     disable_declared_calls: EnvVarBoolean,
     #[envconfig(from = "GRAPH_STORE_ERRORS_ARE_NON_DETERMINISTIC", default = "false")]
     store_errors_are_nondeterministic: EnvVarBoolean,
+    #[envconfig(from = "GRAPH_FDS_MAX_BACKOFF", default = "600")]
+    fds_max_backoff: u64,
 }
 
 fn validate_ipfs_cache_location(path: PathBuf) -> Result<PathBuf, anyhow::Error> {