Startup Retry & Configurable Startup Time-out & Error handling (#166)

* support startup retry and timeout

* update

* update

* update

* add testcase

* clarify error type

* update

* update

* update

* fix lint

* handle keyvault error

* update

* update

* update

* update

* update

* update

* handle keyvault reference error

* update

* fix lint

* update

* update

* add boot loop protection

* update

* update

* update testcase

* update

* update testcase

* update

* update

* update

* move error.ts to common folder

* handle transient network error

* update

* update

* keep error stack when fail to load

* update testcase

Author: zhiyuanliang-ms

rollup.config.mjs

@@ -4,7 +4,15 @@ import dts from "rollup-plugin-dts";
 
 export default [
   {
-    external: ["@azure/app-configuration", "@azure/keyvault-secrets", "@azure/core-rest-pipeline", "crypto", "dns/promises", "@microsoft/feature-management"],
+    external: [
+      "@azure/app-configuration",
+      "@azure/keyvault-secrets",
+      "@azure/core-rest-pipeline",
+      "@azure/identity",
+      "crypto",
+      "dns/promises",
+      "@microsoft/feature-management"
+    ],
     input: "src/index.ts",
     output: [
       {

src/AzureAppConfiguration.ts

@@ -3,6 +3,9 @@
 
 import { Disposable } from "./common/disposable.js";
 
+/**
+ * Azure App Configuration provider.
+ */
 export type AzureAppConfiguration = {
     /**
      * API to trigger refresh operation.

src/AzureAppConfigurationImpl.ts

@@ -7,7 +7,8 @@ import { AzureAppConfiguration, ConfigurationObjectConstructionOptions } from ".
 import { AzureAppConfigurationOptions } from "./AzureAppConfigurationOptions.js";
 import { IKeyValueAdapter } from "./IKeyValueAdapter.js";
 import { JsonKeyValueAdapter } from "./JsonKeyValueAdapter.js";
-import { DEFAULT_REFRESH_INTERVAL_IN_MS, MIN_REFRESH_INTERVAL_IN_MS } from "./RefreshOptions.js";
+import { DEFAULT_STARTUP_TIMEOUT_IN_MS } from "./StartupOptions.js";
+import { DEFAULT_REFRESH_INTERVAL_IN_MS, MIN_REFRESH_INTERVAL_IN_MS } from "./refresh/refreshOptions.js";
 import { Disposable } from "./common/disposable.js";
 import {
     FEATURE_FLAGS_KEY_NAME,
@@ -33,6 +34,10 @@ import { FeatureFlagTracingOptions } from "./requestTracing/FeatureFlagTracingOp
 import { AIConfigurationTracingOptions } from "./requestTracing/AIConfigurationTracingOptions.js";
 import { KeyFilter, LabelFilter, SettingSelector } from "./types.js";
 import { ConfigurationClientManager } from "./ConfigurationClientManager.js";
+import { getFixedBackoffDuration, getExponentialBackoffDuration } from "./common/backoffUtils.js";
+import { InvalidOperationError, ArgumentError, isFailoverableError, isInputError } from "./common/error.js";
+
+const MIN_DELAY_FOR_UNHANDLED_FAILURE = 5_000; // 5 seconds
 
 type PagedSettingSelector = SettingSelector & {
     /**
@@ -118,10 +123,10 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
             } else {
                 for (const setting of watchedSettings) {
                     if (setting.key.includes("*") || setting.key.includes(",")) {
-                        throw new Error("The characters '*' and ',' are not supported in key of watched settings.");
+                        throw new ArgumentError("The characters '*' and ',' are not supported in key of watched settings.");
                     }
                     if (setting.label?.includes("*") || setting.label?.includes(",")) {
-                        throw new Error("The characters '*' and ',' are not supported in label of watched settings.");
+                        throw new ArgumentError("The characters '*' and ',' are not supported in label of watched settings.");
                     }
                     this.#sentinels.push(setting);
                 }
@@ -130,7 +135,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
             // custom refresh interval
             if (refreshIntervalInMs !== undefined) {
                 if (refreshIntervalInMs < MIN_REFRESH_INTERVAL_IN_MS) {
-                    throw new Error(`The refresh interval cannot be less than ${MIN_REFRESH_INTERVAL_IN_MS} milliseconds.`);
+                    throw new RangeError(`The refresh interval cannot be less than ${MIN_REFRESH_INTERVAL_IN_MS} milliseconds.`);
                 } else {
                     this.#kvRefreshInterval = refreshIntervalInMs;
                 }
@@ -148,7 +153,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
                 // custom refresh interval
                 if (refreshIntervalInMs !== undefined) {
                     if (refreshIntervalInMs < MIN_REFRESH_INTERVAL_IN_MS) {
-                        throw new Error(`The feature flag refresh interval cannot be less than ${MIN_REFRESH_INTERVAL_IN_MS} milliseconds.`);
+                        throw new RangeError(`The feature flag refresh interval cannot be less than ${MIN_REFRESH_INTERVAL_IN_MS} milliseconds.`);
                     } else {
                         this.#ffRefreshInterval = refreshIntervalInMs;
                     }
@@ -225,13 +230,40 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
      * Loads the configuration store for the first time.
      */
     async load() {
-        await this.#inspectFmPackage();
-        await this.#loadSelectedAndWatchedKeyValues();
-        if (this.#featureFlagEnabled) {
-            await this.#loadFeatureFlags();
+        const startTimestamp = Date.now();
+        const startupTimeout: number = this.#options?.startupOptions?.timeoutInMs ?? DEFAULT_STARTUP_TIMEOUT_IN_MS;
+        const abortController = new AbortController();
+        const abortSignal = abortController.signal;
+        let timeoutId;
+        try {
+            // Promise.race will be settled when the first promise in the list is settled.
+            // It will not cancel the remaining promises in the list.
+            // To avoid memory leaks, we must ensure other promises will be eventually terminated.
+            await Promise.race([
+                this.#initializeWithRetryPolicy(abortSignal),
+                // this promise will be rejected after timeout
+                new Promise((_, reject) => {
+                    timeoutId = setTimeout(() => {
+                        abortController.abort(); // abort the initialization promise
+                        reject(new Error("Load operation timed out."));
+                    },
+                    startupTimeout);
+                })
+            ]);
+        } catch (error) {
+            if (!isInputError(error)) {
+                const timeElapsed = Date.now() - startTimestamp;
+                if (timeElapsed < MIN_DELAY_FOR_UNHANDLED_FAILURE) {
+                    // load() method is called in the application's startup code path.
+                    // Unhandled exceptions cause application crash which can result in crash loops as orchestrators attempt to restart the application.
+                    // Knowing the intended usage of the provider in startup code path, we mitigate back-to-back crash loops from overloading the server with requests by waiting a minimum time to propagate fatal errors.
+                    await new Promise(resolve => setTimeout(resolve, MIN_DELAY_FOR_UNHANDLED_FAILURE - timeElapsed));
+                }
+            }
+            throw new Error("Failed to load.", { cause: error });
+        } finally {
+            clearTimeout(timeoutId); // cancel the timeout promise
         }
-        // Mark all settings have loaded at startup.
-        this.#isInitialLoadCompleted = true;
     }
 
     /**
@@ -241,7 +273,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
         const separator = options?.separator ?? ".";
         const validSeparators = [".", ",", ";", "-", "_", "__", "/", ":"];
         if (!validSeparators.includes(separator)) {
-            throw new Error(`Invalid separator '${separator}'. Supported values: ${validSeparators.map(s => `'${s}'`).join(", ")}.`);
+            throw new ArgumentError(`Invalid separator '${separator}'. Supported values: ${validSeparators.map(s => `'${s}'`).join(", ")}.`);
         }
 
         // construct hierarchical data object from map
@@ -254,22 +286,22 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
                 const segment = segments[i];
                 // undefined or empty string
                 if (!segment) {
-                    throw new Error(`invalid key: ${key}`);
+                    throw new InvalidOperationError(`Failed to construct configuration object: Invalid key: ${key}`);
                 }
                 // create path if not exist
                 if (current[segment] === undefined) {
                     current[segment] = {};
                 }
                 // The path has been occupied by a non-object value, causing ambiguity.
                 if (typeof current[segment] !== "object") {
-                    throw new Error(`Ambiguity occurs when constructing configuration object from key '${key}', value '${value}'. The path '${segments.slice(0, i + 1).join(separator)}' has been occupied.`);
+                    throw new InvalidOperationError(`Ambiguity occurs when constructing configuration object from key '${key}', value '${value}'. The path '${segments.slice(0, i + 1).join(separator)}' has been occupied.`);
                 }
                 current = current[segment];
             }
 
             const lastSegment = segments[segments.length - 1];
             if (current[lastSegment] !== undefined) {
-                throw new Error(`Ambiguity occurs when constructing configuration object from key '${key}', value '${value}'. The key should not be part of another key.`);
+                throw new InvalidOperationError(`Ambiguity occurs when constructing configuration object from key '${key}', value '${value}'. The key should not be part of another key.`);
             }
             // set value to the last segment
             current[lastSegment] = value;
@@ -282,7 +314,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
      */
     async refresh(): Promise<void> {
         if (!this.#refreshEnabled && !this.#featureFlagRefreshEnabled) {
-            throw new Error("Refresh is not enabled for key-values or feature flags.");
+            throw new InvalidOperationError("Refresh is not enabled for key-values or feature flags.");
         }
 
         if (this.#refreshInProgress) {
@@ -301,7 +333,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
      */
     onRefresh(listener: () => any, thisArg?: any): Disposable {
         if (!this.#refreshEnabled && !this.#featureFlagRefreshEnabled) {
-            throw new Error("Refresh is not enabled for key-values or feature flags.");
+            throw new InvalidOperationError("Refresh is not enabled for key-values or feature flags.");
         }
 
         const boundedListener = listener.bind(thisArg);
@@ -316,6 +348,42 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
         return new Disposable(remove);
     }
 
+    /**
+     * Initializes the configuration provider.
+     */
+    async #initializeWithRetryPolicy(abortSignal: AbortSignal): Promise<void> {
+        if (!this.#isInitialLoadCompleted) {
+            await this.#inspectFmPackage();
+            const startTimestamp = Date.now();
+            let postAttempts = 0;
+            do { // at least try to load once
+                try {
+                    await this.#loadSelectedAndWatchedKeyValues();
+                    if (this.#featureFlagEnabled) {
+                        await this.#loadFeatureFlags();
+                    }
+                    this.#isInitialLoadCompleted = true;
+                    break;
+                } catch (error) {
+                    if (isInputError(error)) {
+                        throw error;
+                    }
+                    if (abortSignal.aborted) {
+                        return;
+                    }
+                    const timeElapsed = Date.now() - startTimestamp;
+                    let backoffDuration = getFixedBackoffDuration(timeElapsed);
+                    if (backoffDuration === undefined) {
+                        postAttempts += 1;
+                        backoffDuration = getExponentialBackoffDuration(postAttempts);
+                    }
+                    console.warn(`Failed to load. Error message: ${error.message}. Retrying in ${backoffDuration} ms.`);
+                    await new Promise(resolve => setTimeout(resolve, backoffDuration));
+                }
+            } while (!abortSignal.aborted);
+        }
+    }
+
     /**
      * Inspects the feature management package version.
      */
@@ -426,7 +494,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
             this.#aiConfigurationTracing.reset();
         }
 
-        // process key-values, watched settings have higher priority
+        // adapt configuration settings to key-values
         for (const setting of loadedSettings) {
             const [key, value] = await this.#processKeyValue(setting);
             keyValues.push([key, value]);
@@ -606,6 +674,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
         return response;
     }
 
+    // Only operations related to Azure App Configuration should be executed with failover policy.
     async #executeWithFailoverPolicy(funcToExecute: (client: AppConfigurationClient) => Promise<any>): Promise<any> {
         let clientWrappers = await this.#clientManager.getClients();
         if (this.#options?.loadBalancingEnabled && this.#lastSuccessfulEndpoint !== "" && clientWrappers.length > 1) {
@@ -645,7 +714,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
         }
 
         this.#clientManager.refreshClients();
-        throw new Error("All clients failed to get configuration settings.");
+        throw new Error("All fallback clients failed to get configuration settings.");
     }
 
     async #processKeyValue(setting: ConfigurationSetting<string>): Promise<[string, unknown]> {
@@ -700,7 +769,7 @@ export class AzureAppConfigurationImpl implements AzureAppConfiguration {
     async #parseFeatureFlag(setting: ConfigurationSetting<string>): Promise<any> {
         const rawFlag = setting.value;
         if (rawFlag === undefined) {
-            throw new Error("The value of configuration setting cannot be undefined.");
+            throw new ArgumentError("The value of configuration setting cannot be undefined.");
         }
         const featureFlag = JSON.parse(rawFlag);
 
@@ -762,13 +831,13 @@ function getValidSelectors(selectors: SettingSelector[]): SettingSelector[] {
     return uniqueSelectors.map(selectorCandidate => {
         const selector = { ...selectorCandidate };
         if (!selector.keyFilter) {
-            throw new Error("Key filter cannot be null or empty.");
+            throw new ArgumentError("Key filter cannot be null or empty.");
         }
         if (!selector.labelFilter) {
             selector.labelFilter = LabelFilter.Null;
         }
         if (selector.labelFilter.includes("*") || selector.labelFilter.includes(",")) {
-            throw new Error("The characters '*' and ',' are not supported in label filters.");
+            throw new ArgumentError("The characters '*' and ',' are not supported in label filters.");
         }
         return selector;
     });
@@ -792,9 +861,3 @@ function getValidFeatureFlagSelectors(selectors?: SettingSelector[]): SettingSel
     });
     return getValidSelectors(selectors);
 }
-
-function isFailoverableError(error: any): boolean {
-    // ENOTFOUND: DNS lookup failed, ENOENT: no such file or directory
-    return isRestError(error) && (error.code === "ENOTFOUND" || error.code === "ENOENT" ||
-        (error.statusCode !== undefined && (error.statusCode === 401 || error.statusCode === 403 || error.statusCode === 408 || error.statusCode === 429 || error.statusCode >= 500)));
-}

src/AzureAppConfigurationOptions.ts

@@ -3,12 +3,10 @@
 
 import { AppConfigurationClientOptions } from "@azure/app-configuration";
 import { KeyVaultOptions } from "./keyvault/KeyVaultOptions.js";
-import { RefreshOptions } from "./RefreshOptions.js";
+import { RefreshOptions } from "./refresh/refreshOptions.js";
 import { SettingSelector } from "./types.js";
 import { FeatureFlagOptions } from "./featureManagement/FeatureFlagOptions.js";
-
-export const MaxRetries = 2;
-export const MaxRetryDelayInMs = 60000;
+import { StartupOptions } from "./StartupOptions.js";
 
 export interface AzureAppConfigurationOptions {
     /**
@@ -48,6 +46,11 @@ export interface AzureAppConfigurationOptions {
      */
     featureFlagOptions?: FeatureFlagOptions;
 
+    /**
+     * Specifies options used to configure provider startup.
+     */
+    startupOptions?: StartupOptions;
+
     /**
      * Specifies whether to enable replica discovery or not.
      *