Startup Retry & Configurable Startup Time-out & Error handling

src/AzureAppConfigurationImpl.ts

@@ -7,6 +7,7 @@ import { AzureAppConfiguration, ConfigurationObjectConstructionOptions } from ".
 import { AzureAppConfigurationOptions } from "./AzureAppConfigurationOptions.js";
 import { IKeyValueAdapter } from "./IKeyValueAdapter.js";
 import { JsonKeyValueAdapter } from "./JsonKeyValueAdapter.js";
+import { DEFAULT_STARTUP_TIMEOUT_IN_MS } from "./StartupOptions.js";
 import { DEFAULT_REFRESH_INTERVAL_IN_MS, MIN_REFRESH_INTERVAL_IN_MS } from "./RefreshOptions.js";
 import { Disposable } from "./common/disposable.js";
 import { base64Helper, jsonSorter } from "./common/utils.js";
@@ -40,6 +41,8 @@ import { RequestTracingOptions, getConfigurationSettingWithTrace, listConfigurat
 import { FeatureFlagTracingOptions } from "./requestTracing/FeatureFlagTracingOptions.js";
 import { KeyFilter, LabelFilter, SettingSelector } from "./types.js";
 import { ConfigurationClientManager } from "./ConfigurationClientManager.js";
+import { getFixedBackoffDuration, calculateDynamicBackoffDuration } from "./failover.js";
+import { FailoverError, OperationError, isFailoverableError, isRetriableError } from "./error.js";
 
 type PagedSettingSelector = SettingSelector & {
     /**
@@ -123,10 +126,10 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
             } else {
                 for (const setting of watchedSettings) {
                     if (setting.key.includes("*") || setting.key.includes(",")) {
-                        throw new Error("The characters '*' and ',' are not supported in key of watched settings.");
+                        throw new RangeError("The characters '*' and ',' are not supported in key of watched settings.");
                     }
                     if (setting.label?.includes("*") || setting.label?.includes(",")) {
-                        throw new Error("The characters '*' and ',' are not supported in label of watched settings.");
+                        throw new RangeError("The characters '*' and ',' are not supported in label of watched settings.");
                     }
                     this.#sentinels.push(setting);
                 }
@@ -135,7 +138,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
             // custom refresh interval
             if (refreshIntervalInMs !== undefined) {
                 if (refreshIntervalInMs < MIN_REFRESH_INTERVAL_IN_MS) {
-                    throw new Error(`The refresh interval cannot be less than ${MIN_REFRESH_INTERVAL_IN_MS} milliseconds.`);
+                    throw new RangeError(`The refresh interval cannot be less than ${MIN_REFRESH_INTERVAL_IN_MS} milliseconds.`);
                 } else {
                     this.#kvRefreshInterval = refreshIntervalInMs;
                 }
@@ -153,7 +156,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
                 // custom refresh interval
                 if (refreshIntervalInMs !== undefined) {
                     if (refreshIntervalInMs < MIN_REFRESH_INTERVAL_IN_MS) {
-                        throw new Error(`The feature flag refresh interval cannot be less than ${MIN_REFRESH_INTERVAL_IN_MS} milliseconds.`);
+                        throw new RangeError(`The feature flag refresh interval cannot be less than ${MIN_REFRESH_INTERVAL_IN_MS} milliseconds.`);
                     } else {
                         this.#ffRefreshInterval = refreshIntervalInMs;
                     }
@@ -229,13 +232,30 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
      * Loads the configuration store for the first time.
      */
     async load() {
-        await this.#inspectFmPackage();
-        await this.#loadSelectedAndWatchedKeyValues();
-        if (this.#featureFlagEnabled) {
-            await this.#loadFeatureFlags();
+        const startupTimeout: number = this.#options?.startupOptions?.timeoutInMs ?? DEFAULT_STARTUP_TIMEOUT_IN_MS;
+        const abortController = new AbortController();
+        const abortSignal = abortController.signal;
+        let timeoutId;
+        try {
+            // Promise.race will be settled when the first promise in the list is settled
+            // It will not cancel the remaining promises in the list.
+            // To avoid memory leaks, we need to cancel other promises when one promise is settled.
+            await Promise.race([
+                this.#initializeWithRetryPolicy(abortSignal),
+                // this promise will be rejected after timeout
+                new Promise((_, reject) => {
+                    timeoutId = setTimeout(() => {
+                        abortController.abort(); // abort the initialization promise
+                        reject(new Error("Load operation timed out."));
+                    },
+                    startupTimeout);
+                })
+            ]);
+        } catch (error) {
+            throw new Error(`Failed to load: ${error.message}`);
+        } finally {
+            clearTimeout(timeoutId); // cancel the timeout promise
         }
-        // Mark all settings have loaded at startup.
-        this.#isInitialLoadCompleted = true;
     }
 
     /**
@@ -245,7 +265,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
         const separator = options?.separator ?? ".";
         const validSeparators = [".", ",", ";", "-", "_", "__", "/", ":"];
         if (!validSeparators.includes(separator)) {
-            throw new Error(`Invalid separator '${separator}'. Supported values: ${validSeparators.map(s => `'${s}'`).join(", ")}.`);
+            throw new RangeError(`Invalid separator '${separator}'. Supported values: ${validSeparators.map(s => `'${s}'`).join(", ")}.`);
         }
 
         // construct hierarchical data object from map
@@ -258,22 +278,22 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
                 const segment = segments[i];
                 // undefined or empty string
                 if (!segment) {
-                    throw new Error(`invalid key: ${key}`);
+                    throw new OperationError(`Failed to construct configuration object: Invalid key: ${key}`);
                 }
                 // create path if not exist
                 if (current[segment] === undefined) {
                     current[segment] = {};
                 }
                 // The path has been occupied by a non-object value, causing ambiguity.
                 if (typeof current[segment] !== "object") {
-                    throw new Error(`Ambiguity occurs when constructing configuration object from key '${key}', value '${value}'. The path '${segments.slice(0, i + 1).join(separator)}' has been occupied.`);
+                    throw new OperationError(`Ambiguity occurs when constructing configuration object from key '${key}', value '${value}'. The path '${segments.slice(0, i + 1).join(separator)}' has been occupied.`);
                 }
                 current = current[segment];
             }
 
             const lastSegment = segments[segments.length - 1];
             if (current[lastSegment] !== undefined) {
-                throw new Error(`Ambiguity occurs when constructing configuration object from key '${key}', value '${value}'. The key should not be part of another key.`);
+                throw new OperationError(`Ambiguity occurs when constructing configuration object from key '${key}', value '${value}'. The key should not be part of another key.`);
             }
             // set value to the last segment
             current[lastSegment] = value;
@@ -286,7 +306,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
      */
     async refresh(): Promise<void> {
         if (!this.#refreshEnabled && !this.#featureFlagRefreshEnabled) {
-            throw new Error("Refresh is not enabled for key-values or feature flags.");
+            throw new OperationError("Refresh is not enabled for key-values or feature flags.");
         }
 
         if (this.#refreshInProgress) {
@@ -305,7 +325,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
      */
     onRefresh(listener: () => any, thisArg?: any): Disposable {
         if (!this.#refreshEnabled && !this.#featureFlagRefreshEnabled) {
-            throw new Error("Refresh is not enabled for key-values or feature flags.");
+            throw new OperationError("Refresh is not enabled for key-values or feature flags.");
         }
 
         const boundedListener = listener.bind(thisArg);
@@ -320,6 +340,44 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
         return new Disposable(remove);
     }
 
+    /**
+     * Initializes the configuration provider.
+     */
+    async #initializeWithRetryPolicy(abortSignal: AbortSignal): Promise<void> {
+        if (!this.#isInitialLoadCompleted) {
+            await this.#inspectFmPackage();
+            const startTimestamp = Date.now();
+            do { // at least try to load once
+                try {
+                    await this.#loadSelectedAndWatchedKeyValues();
+                    if (this.#featureFlagEnabled) {
+                        await this.#loadFeatureFlags();
+                    }
+                    // Mark all settings have loaded at startup.
+                    this.#isInitialLoadCompleted = true;
+                    break;
+                } catch (error) {
+
+                    if (!isRetriableError(error)) {
+                        throw error;
+                    }
+                    if (abortSignal.aborted) {
+                        return;
+                    }
+                    const timeElapsed = Date.now() - startTimestamp;
+                    let postAttempts = 0;
+                    let backoffDuration = getFixedBackoffDuration(timeElapsed);
+                    if (backoffDuration === undefined) {
+                        postAttempts += 1;
+                        backoffDuration = calculateDynamicBackoffDuration(postAttempts);
+                    }
+                    await new Promise(resolve => setTimeout(resolve, backoffDuration));
+                    console.warn("Failed to load configuration settings at startup. Retrying...");
+                }
+            } while (!abortSignal.aborted);
+        }
+    }
+
     /**
      * Inspects the feature management package version.
      */
@@ -639,7 +697,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
         }
 
         this.#clientManager.refreshClients();
-        throw new Error("All clients failed to get configuration settings.");
+        throw new FailoverError("All fallback clients failed to get configuration settings.");
     }
 
     async #processKeyValues(setting: ConfigurationSetting<string>): Promise<[string, unknown]> {
@@ -671,7 +729,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
     async #parseFeatureFlag(setting: ConfigurationSetting<string>): Promise<any> {
         const rawFlag = setting.value;
         if (rawFlag === undefined) {
-            throw new Error("The value of configuration setting cannot be undefined.");
+            throw new RangeError("The value of configuration setting cannot be undefined.");
         }
         const featureFlag = JSON.parse(rawFlag);
 
@@ -895,13 +953,13 @@ function getValidSelectors(selectors: SettingSelector[]): SettingSelector[] {
     return uniqueSelectors.map(selectorCandidate => {
         const selector = { ...selectorCandidate };
         if (!selector.keyFilter) {
-            throw new Error("Key filter cannot be null or empty.");
+            throw new RangeError("Key filter cannot be null or empty.");
         }
         if (!selector.labelFilter) {
             selector.labelFilter = LabelFilter.Null;
         }
         if (selector.labelFilter.includes("*") || selector.labelFilter.includes(",")) {
-            throw new Error("The characters '*' and ',' are not supported in label filters.");
+            throw new RangeError("The characters '*' and ',' are not supported in label filters.");
         }
         return selector;
     });
@@ -925,9 +983,3 @@ function getValidFeatureFlagSelectors(selectors?: SettingSelector[]): SettingSel
     });
     return getValidSelectors(selectors);
 }
-
-function isFailoverableError(error: any): boolean {
-    // ENOTFOUND: DNS lookup failed, ENOENT: no such file or directory
-    return isRestError(error) && (error.code === "ENOTFOUND" || error.code === "ENOENT" ||
-        (error.statusCode !== undefined && (error.statusCode === 401 || error.statusCode === 403 || error.statusCode === 408 || error.statusCode === 429 || error.statusCode >= 500)));
-}

src/AzureAppConfigurationOptions.ts

@@ -6,9 +6,7 @@ import { KeyVaultOptions } from "./keyvault/KeyVaultOptions.js";
 import { RefreshOptions } from "./RefreshOptions.js";
 import { SettingSelector } from "./types.js";
 import { FeatureFlagOptions } from "./featureManagement/FeatureFlagOptions.js";
-
-export const MaxRetries = 2;
-export const MaxRetryDelayInMs = 60000;
+import { StartupOptions } from "./StartupOptions.js";
 
 export interface AzureAppConfigurationOptions {
     /**
@@ -48,6 +46,11 @@ export interface AzureAppConfigurationOptions {
      */
     featureFlagOptions?: FeatureFlagOptions;
 
+    /**
+     * Specifies options used to configure provider startup.
+     */
+    startupOptions?: StartupOptions;
+
     /**
      * Specifies whether to enable replica discovery or not.
      *

src/ConfigurationClientManager.ts

@@ -4,10 +4,15 @@
 import { AppConfigurationClient, AppConfigurationClientOptions } from "@azure/app-configuration";
 import { ConfigurationClientWrapper } from "./ConfigurationClientWrapper.js";
 import { TokenCredential } from "@azure/identity";
-import { AzureAppConfigurationOptions, MaxRetries, MaxRetryDelayInMs } from "./AzureAppConfigurationOptions.js";
+import { AzureAppConfigurationOptions } from "./AzureAppConfigurationOptions.js";
 import { isBrowser, isWebWorker } from "./requestTracing/utils.js";
 import * as RequestTracing from "./requestTracing/constants.js";
 import { shuffleList } from "./common/utils.js";
+import { OperationError } from "./error.js";
+
+// Configuration client retry options
+const CLIENT_MAX_RETRIES = 2;
+const CLIENT_MAX_RETRY_DELAY = 60_000; // 1 minute in milliseconds
 
 const TCP_ORIGIN_KEY_NAME = "_origin._tcp";
 const ALT_KEY_NAME = "_alt";
@@ -25,7 +30,6 @@ const MAX_ALTNATIVE_SRV_COUNT = 10;
 export class ConfigurationClientManager {
     #isFailoverable: boolean;
     #dns: any;
-    endpoint: URL;
     #secret : string;
     #id : string;
     #credential: TokenCredential;
@@ -38,6 +42,9 @@ export class ConfigurationClientManager {
     #lastFallbackClientUpdateTime: number = 0; // enforce to discover fallback client when it is expired
     #lastFallbackClientRefreshAttempt: number = 0; // avoid refreshing clients before the minimal refresh interval
 
+    // This property is public to allow recording the last successful endpoint for failover.
+    endpoint: URL;
+
     constructor (
         connectionStringOrEndpoint?: string | URL,
         credentialOrOptions?: TokenCredential | AzureAppConfigurationOptions,
@@ -58,7 +65,7 @@ export class ConfigurationClientManager {
                 this.#id = regexMatch[2];
                 this.#secret = regexMatch[3];
             } else {
-                throw new Error(`Invalid connection string. Valid connection strings should match the regex '${ConnectionStringRegex.source}'.`);
+                throw new RangeError(`Invalid connection string. Valid connection strings should match the regex '${ConnectionStringRegex.source}'.`);
             }
             staticClient = new AppConfigurationClient(connectionString, this.#clientOptions);
         } else if ((connectionStringOrEndpoint instanceof URL || typeof connectionStringOrEndpoint === "string") && credentialPassed) {
@@ -75,7 +82,7 @@ export class ConfigurationClientManager {
             this.#credential = credential;
             staticClient = new AppConfigurationClient(this.endpoint.origin, this.#credential, this.#clientOptions);
         } else {
-            throw new Error("A connection string or an endpoint with credential must be specified to create a client.");
+            throw new OperationError("A connection string or an endpoint with credential must be specified to create a client.");
         }
 
         this.#staticClients = [new ConfigurationClientWrapper(this.endpoint.origin, staticClient)];
@@ -260,8 +267,8 @@ function getClientOptions(options?: AzureAppConfigurationOptions): AppConfigurat
 
     // retry options
     const defaultRetryOptions = {
-        maxRetries: MaxRetries,
-        maxRetryDelayInMs: MaxRetryDelayInMs,
+        maxRetries: CLIENT_MAX_RETRIES,
+        maxRetryDelayInMs: CLIENT_MAX_RETRY_DELAY,
     };
     const retryOptions = Object.assign({}, defaultRetryOptions, options?.clientOptions?.retryOptions);
 
@@ -278,7 +285,7 @@ function getValidUrl(endpoint: string): URL {
         return new URL(endpoint);
     } catch (error) {
         if (error.code === "ERR_INVALID_URL") {
-            throw new Error("Invalid endpoint URL.", { cause: error });
+            throw new RangeError("Invalid endpoint URL.", { cause: error });
         } else {
             throw error;
         }

src/ConfigurationClientWrapper.ts

@@ -2,11 +2,7 @@
 // Licensed under the MIT license.
 
 import { AppConfigurationClient } from "@azure/app-configuration";
-
-const MaxBackoffDuration = 10 * 60 * 1000; // 10 minutes in milliseconds
-const MinBackoffDuration = 30 * 1000; // 30 seconds in milliseconds
-const MAX_SAFE_EXPONENTIAL = 30; // Used to avoid overflow. bitwise operations in JavaScript are limited to 32 bits. It overflows at 2^31 - 1.
-const JITTER_RATIO = 0.25;
+import { calculateDynamicBackoffDuration } from "./failover.js";
 
 export class ConfigurationClientWrapper {
     endpoint: string;
@@ -25,25 +21,7 @@ export class ConfigurationClientWrapper {
             this.backoffEndTime = Date.now();
         } else {
             this.#failedAttempts += 1;
-            this.backoffEndTime = Date.now() + calculateBackoffDuration(this.#failedAttempts);
+            this.backoffEndTime = Date.now() + calculateDynamicBackoffDuration(this.#failedAttempts);
         }
     }
 }
-
-export function calculateBackoffDuration(failedAttempts: number) {
-    if (failedAttempts <= 1) {
-        return MinBackoffDuration;
-    }
-
-    // exponential: minBackoff * 2 ^ (failedAttempts - 1)
-    const exponential = Math.min(failedAttempts - 1, MAX_SAFE_EXPONENTIAL);
-    let calculatedBackoffDuration = MinBackoffDuration * (1 << exponential);
-    if (calculatedBackoffDuration > MaxBackoffDuration) {
-        calculatedBackoffDuration = MaxBackoffDuration;
-    }
-
-    // jitter: random value between [-1, 1) * jitterRatio * calculatedBackoffMs
-    const jitter = JITTER_RATIO * (Math.random() * 2 - 1);
-
-    return calculatedBackoffDuration * (1 + jitter);
-}

src/StartupOptions.ts

@@ -0,0 +1,14 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+export const DEFAULT_STARTUP_TIMEOUT_IN_MS = 100 * 1000; // 100 seconds in milliseconds
+
+export interface StartupOptions {
+    /**
+     * The amount of time allowed to load data from Azure App Configuration on startup.
+     *
+     * @remarks
+     * If not specified, the default value is 100 seconds. Must be greater than 1 second.
+     */
+    timeoutInMs?: number;
+}

src/common/utils.ts

@@ -24,9 +24,9 @@ export function jsonSorter(key, value) {
 }
 
 export function shuffleList<T>(array: T[]): T[] {
-  for (let i = array.length - 1; i > 0; i--) {
-    const j = Math.floor(Math.random() * (i + 1));
-    [array[i], array[j]] = [array[j], array[i]];
-  }
-  return array;
+    for (let i = array.length - 1; i > 0; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        [array[i], array[j]] = [array[j], array[i]];
+    }
+    return array;
 }