Merge pull request #21 from launchdarkly/eb/ch108408/big-segments

implement big segment store + replace default export with named exports

Author: eli-darkly

.eslintrc.js

@@ -3,6 +3,9 @@ module.exports = {
         "node": true,
         "es6": true
     },
+    "parserOptions": {
+        "ecmaVersion": 2017,
+    },
     "extends": "eslint:recommended",
     "rules": {
         "indent": [

README.md

@@ -4,7 +4,7 @@
 
 This library provides a DynamoDB-backed persistence mechanism (feature store) for the [LaunchDarkly Node.js SDK](https://github.com/launchdarkly/node-server-sdk), replacing the default in-memory feature store. It uses the AWS SDK for Node.js.
 
-The minimum version of the LaunchDarkly Node.js SDK for use with this library is 6.0.0.
+The minimum version of the LaunchDarkly Node.js SDK for use with this library is 6.2.0.
 
 For more information, see the [SDK features guide](https://docs.launchdarkly.com/sdk/features/database-integrations).
 
@@ -28,37 +28,37 @@ This assumes that you have already installed the LaunchDarkly Node.js SDK.
 
 4. Require the package:
 
-        var DynamoDBFeatureStore = require('launchdarkly-node-server-sdk-dynamodb');
+        const { DynamoDBFeatureStore } = require('launchdarkly-node-server-sdk-dynamodb');
 
 5. When configuring your SDK client, add the DynamoDB feature store:
 
-        var store = DynamoDBFeatureStore('YOUR TABLE NAME');
-        var config = { featureStore: store };
-        var client = LaunchDarkly.init('YOUR SDK KEY', config);
+        const store = DynamoDBFeatureStore('YOUR TABLE NAME');
+        const config = { featureStore: store };
+        const client = LaunchDarkly.init('YOUR SDK KEY', config);
 
     By default, the DynamoDB client will try to get your AWS credentials and region name from environment variables and/or local configuration files, as described in the AWS SDK documentation. You can also specify any valid [DynamoDB client options](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/DynamoDB.html#constructor-property) like this:
 
-        var dynamoDBOptions = { accessKeyId: 'YOUR KEY', secretAccessKey: 'YOUR SECRET' };
-        var store = DynamoDBFeatureStore('YOUR TABLE NAME', { clientOptions: dynamoDBOptions });
+        const dynamoDBOptions = { accessKeyId: 'YOUR KEY', secretAccessKey: 'YOUR SECRET' };
+        const store = DynamoDBFeatureStore('YOUR TABLE NAME', { clientOptions: dynamoDBOptions });
 
     Alternatively, if you already have a fully configured DynamoDB client object, you can tell LaunchDarkly to use that:
 
-        var store = DynamoDBFeatureStore('YOUR TABLE NAME', { dynamoDBClient: myDynamoDBClientInstance });
+        const store = DynamoDBFeatureStore('YOUR TABLE NAME', { dynamoDBClient: myDynamoDBClientInstance });
 
 6. If you are running a [LaunchDarkly Relay Proxy](https://github.com/launchdarkly/ld-relay) instance, or any other process that will prepopulate the DynamoDB table with feature flags from LaunchDarkly, you can use [daemon mode](https://github.com/launchdarkly/ld-relay#daemon-mode), so that the SDK retrieves flag data only from DynamoDB and does not communicate directly with LaunchDarkly. This is controlled by the SDK's `useLdd` option:
 
-        var config = { featureStore: store, useLdd: true };
-        var client = LaunchDarkly.init('YOUR SDK KEY', config);
+        const config = { featureStore: store, useLdd: true };
+        const client = LaunchDarkly.init('YOUR SDK KEY', config);
 
 7. If the same DynamoDB table is being shared by SDK clients for different LaunchDarkly environments, set the `prefix` option to a different short string for each one to keep the keys from colliding:
 
-        var store = DynamoDBFeatureStore('YOUR TABLE NAME', { prefix: 'env1' });
+        const store = DynamoDBFeatureStore('YOUR TABLE NAME', { prefix: 'env1' });
 
 ## Caching behavior
 
 To reduce traffic to DynamoDB, there is an optional in-memory cache that retains the last known data for a configurable amount of time. This is on by default; to turn it off (and guarantee that the latest feature flag data will always be retrieved from DynamoDB for every flag evaluation), configure the store as follows:
 
-        var store = DynamoDBFeatureStore('YOUR TABLE NAME', { cacheTTL: 0 });
+        const store = DynamoDBFeatureStore('YOUR TABLE NAME', { cacheTTL: 0 });
 
 ## About LaunchDarkly
 

dynamodb_big_segment_store.js

@@ -0,0 +1,93 @@
+const { initState } = require('./dynamodb_helpers');
+const { promisify } = require('util');
+
+const keyMetadata = 'big_segments_metadata';
+const keyUserData = 'big_segments_user';
+const attrSyncTime = 'synchronizedOn';
+const attrIncluded = 'included';
+const attrExcluded = 'excluded';
+
+// Note that the format of parameters in this implementation is a bit different than in the
+// LD DynamoDB integrations for some other platforms, because we are using the
+// AWS.DynamoDB.DocumentClient class, which represents values as simple types like
+// string or number, rather than in the { S: stringValue } or { N: numericStringValue }
+// format used by the basic AWS DynamoDB API.
+
+function DynamoDBBigSegmentStore(tableName, maybeOptions) {
+  const options = maybeOptions || {};
+  return () => // config parameter is currently unused because we don't need to do any logging
+    dynamoDBBigSegmentStoreImpl(tableName, options);
+}
+
+function dynamoDBBigSegmentStoreImpl(tableName, options) {
+  const state = initState(options);
+  const dynamoDBClient = state.client;
+  const prefix = state.prefix;
+
+  const store = {};
+
+  // Pre-promisify for efficiency. Note that we have to add .bind(client) to each method when
+  // when using promisify, because the AWS client methods don't work without a "this" context.
+  const clientGet = promisify(dynamoDBClient.get.bind(dynamoDBClient));
+
+  store.getMetadata = async () => {
+    const key = prefix + keyMetadata;
+    const data = await clientGet({
+      TableName: tableName,
+      Key: { namespace: key, key: key },
+    });
+    if (data.Item) {
+      const attr = data.Item[attrSyncTime];
+      if (attr) {
+        return { lastUpToDate: attr };
+      }
+    }
+    return { lastUpToDate: undefined };
+  };
+  
+  store.getUserMembership = async userHashKey => {
+    const data = await clientGet({
+      TableName: tableName,
+      Key: {
+        namespace: prefix + keyUserData,
+        key: userHashKey,
+      },
+    });
+    const item = data.Item;
+    if (item) {
+      const membership = {};
+      const excludedRefs = item[attrExcluded];
+      const includedRefs = item[attrIncluded];
+      // The actual type of these values in DynamoDB is a string set. The DocumentClient
+      // returns string set values as a special type where the actual list of strings is
+      // in a "values" property.
+      if (excludedRefs && excludedRefs.values) {
+        for (const ref of excludedRefs.values) {
+          membership[ref] = false;
+        }
+      }
+      if (includedRefs && includedRefs.values) {
+        for (const ref of includedRefs.values) {
+          membership[ref] = true;
+        }
+      }
+      return membership;
+    }
+    return null;
+  };
+
+  store.close = function() {
+    // The Node DynamoDB client is stateless, so close isn't a meaningful operation.
+  };
+
+  return store;
+}
+
+module.exports = {
+  DynamoDBBigSegmentStore,
+  keyMetadata,
+  keyUserData,
+  attrSyncTime,
+  attrIncluded,
+  attrExcluded,
+};

dynamodb_feature_store.js

@@ -1,15 +1,19 @@
-var AWS = require('aws-sdk');
-
-var helpers = require('./dynamodb_helpers');
-var CachingStoreWrapper = require('launchdarkly-node-server-sdk/caching_store_wrapper');
-
-var defaultCacheTTLSeconds = 15;
-
-function DynamoDBFeatureStore(tableName, options) {
-  var ttl = options && options.cacheTTL;
-  if (ttl === null || ttl === undefined) {
-    ttl = defaultCacheTTLSeconds;
-  }
+const { initState, batchWrite, queryHelper } = require('./dynamodb_helpers');
+const CachingStoreWrapper = require('launchdarkly-node-server-sdk/caching_store_wrapper');
+
+const defaultCacheTTLSeconds = 15;
+
+// Note that the format of parameters in this implementation is a bit different than in the
+// LD DynamoDB integrations for some other platforms, because we are using the
+// AWS.DynamoDB.DocumentClient class, which represents values as simple types like
+// string or number, rather than in the { S: stringValue } or { N: numericStringValue }
+// format used by the basic AWS DynamoDB API.
+
+function DynamoDBFeatureStore(tableName, maybeOptions) {
+  const options = maybeOptions || {};
+  const ttl = options.cacheTTL !== null && options.cacheTTL !== undefined
+    ? options.cacheTTL
+    : defaultCacheTTLSeconds;
   return config =>
     new CachingStoreWrapper(
       dynamoDBFeatureStoreInternal(tableName, options, config.logger),
@@ -18,13 +22,12 @@ function DynamoDBFeatureStore(tableName, options) {
     );
 }
 
-function dynamoDBFeatureStoreInternal(tableName, options, sdkLogger) {
-  options = options || {};
-  var logger = options.logger || sdkLogger;
-  var dynamoDBClient = options.dynamoDBClient || new AWS.DynamoDB.DocumentClient(options.clientOptions);
-  var prefix = options.prefix || '';
+function dynamoDBFeatureStoreInternal(tableName, options, logger) {
+  const state = initState(options);
+  const dynamoDBClient = state.client;
+  const prefix = state.prefix;
 
-  var store = {};
+  const store = {};
 
   store.getInternal = function(kind, key, cb) {
     dynamoDBClient.get({
@@ -47,7 +50,7 @@ function dynamoDBFeatureStoreInternal(tableName, options, sdkLogger) {
 
   store.getAllInternal = function(kind, cb) {
     var params = queryParamsForNamespace(kind.namespace);
-    helpers.queryHelper(dynamoDBClient, params).then(function (items) {
+    queryHelper(dynamoDBClient, params).then(function (items) {
       var results = {};
       for (var i = 0; i < items.length; i++) {
         var item = unmarshalItem(items[i]);
@@ -90,7 +93,7 @@ function dynamoDBFeatureStoreInternal(tableName, options, sdkLogger) {
         // Always write the initialized token when we initialize.
         ops.push({ PutRequest: { Item: initializedToken() } });
 
-        var writePromises = helpers.batchWrite(dynamoDBClient, tableName, ops);
+        var writePromises = batchWrite(dynamoDBClient, tableName, ops);
 
         return Promise.all(writePromises);
       })
@@ -149,7 +152,7 @@ function dynamoDBFeatureStoreInternal(tableName, options, sdkLogger) {
       TableName: tableName,
       KeyConditionExpression: 'namespace = :namespace',
       FilterExpression: 'attribute_not_exists(deleted) OR deleted = :deleted',
-      ExpressionAttributeValues: { ':namespace': prefixedNamespace(namespace), ':deleted': false }
+      ExpressionAttributeValues: { ':namespace': prefix + namespace, ':deleted': false }
     };
   }
 
@@ -159,24 +162,20 @@ function dynamoDBFeatureStoreInternal(tableName, options, sdkLogger) {
       var namespace = collection.kind.namespace;
       p = p.then(function(previousItems) {
         var params = queryParamsForNamespace(namespace);
-        return helpers.queryHelper(dynamoDBClient, params).then(function (items) {
+        return queryHelper(dynamoDBClient, params).then(function (items) {
           return previousItems.concat(items);
         });
       });
     });
     return p;
   }
 
-  function prefixedNamespace(baseNamespace) {
-    return prefix ? (prefix + ':' + baseNamespace) : baseNamespace;
-  }
-
   function namespaceForKind(kind) {
-    return prefixedNamespace(kind.namespace);
+    return prefix + kind.namespace;
   }
 
   function initializedToken() {
-    var value = prefixedNamespace('$inited');
+    var value = prefix + '$inited';
     return { namespace: value, key: value };
   }
 

dynamodb_helpers.js

@@ -1,3 +1,27 @@
+const AWS = require('aws-sdk');
+
+function optionalPrefix(prefix) {
+  // Unlike some other database integrations where the key prefix is mandatory and has
+  // a default value, in DynamoDB it is fine to not have a prefix. If there is one, we
+  // prepend it to keys with a ':' separator.
+  return prefix ? prefix + ':' : '';
+}
+
+function initState(options) {
+  const state = {
+    prefix: optionalPrefix(options.prefix)
+  };
+
+  if (options.dynamoDBClient) {
+    state.client = options.dynamoDBClient;
+  } else {
+    state.client = new AWS.DynamoDB.DocumentClient(options.clientOptions);
+  }
+  // Unlike some other database integrations, we don't need to keep track of whether we
+  // created our own client so as to shut it down later; the AWS client is stateless.
+  
+  return state;
+}
 
 function paginationHelper(params, executeFn, startKey) {
   return new Promise(function(resolve, reject) {
@@ -43,7 +67,9 @@ function batchWrite(client, tableName, ops) {
 }
 
 module.exports = {
-  batchWrite: batchWrite,
-  paginationHelper: paginationHelper,
-  queryHelper: queryHelper
+  initState,
+  optionalPrefix,
+  batchWrite,
+  paginationHelper,
+  queryHelper
 };

index.d.ts

@@ -8,25 +8,38 @@
 
 declare module 'launchdarkly-node-server-sdk-dynamodb' {
   import { LDFeatureStore, LDLogger, LDOptions } from 'launchdarkly-node-server-sdk';
+  import { BigSegmentStore } from 'launchdarkly-node-server-sdk/interfaces';
   import { DynamoDB } from 'aws-sdk';
 
   /**
    * Create a feature flag store backed by DynamoDB.
+   * 
+   * @param tableName The table name in DynamoDB (required). The table must already exist.
+   *   See: https://docs.launchdarkly.com/sdk/features/storing-data/dynamodb
+   * @param options Additional options for configuring the DynamoDB store's behavior.
    */
-  export default function DynamoDBFeatureStore(
-    /**
-     * The table name in DynamoDB. This table must already exist (see readme).
-     */
+  export function DynamoDBFeatureStore(
     tableName: string,
-
-    /**
-     * Options for configuring the feature store.
-     */
     options?: LDDynamoDBOptions
   ): (config: LDOptions) => LDFeatureStore;
 
   /**
-   * Options for configuring a DynamoDBFeatureStore.
+   * Configures a big segment store backed by a Redis instance.
+   * 
+   * "Big segments" are a specific type of user segments. For more information, read the
+   * LaunchDarkly documentation about user segments: https://docs.launchdarkly.com/home/users
+   *
+   * @param tableName The table name in DynamoDB (required). The table must already exist.
+   *   See: https://docs.launchdarkly.com/sdk/features/storing-data/dynamodb
+   * @param options Additional options for configuring the DynamoDB store's behavior.
+   */
+  export function DynamoDBBigSegmentStore(
+    tableName: string,
+    options?: LDDynamoDBOptions
+  ): (config: LDOptions) => BigSegmentStore;
+
+  /**
+   * Options for configuring [[DynamoDBFeatureStore]] or [[DynamoDBBigSegmentStore]].
    */
   export interface LDDynamoDBOptions {
     /**
@@ -49,8 +62,14 @@ declare module 'launchdarkly-node-server-sdk-dynamodb' {
     prefix?: string;
 
     /**
-     * The expiration time for local caching, in seconds. To disable local caching, set this to zero.
-     * If not specified, the default is 15 seconds.
+     * The amount of time, in seconds, that recently read or updated items should remain in an
+     * in-memory cache. If it is zero, there will be no in-memory caching.
+     * 
+     * This parameter applies only to [[DynamoDBFeatureStore]]. It is ignored for [[DynamoDBBigSegmentStore]].
+     * Caching for [[DynamoDBBigSegmentStore]] is configured separately, in the SDK's
+     * `LDBigSegmentsOptions` type, since it is independent of what database implementation is used.
+     * 
+     * If omitted, the default value is 15 seconds.
      */
     cacheTTL?: number;
 

index.js

@@ -0,0 +1,7 @@
+const DynamoDBFeatureStore = require('./dynamodb_feature_store');
+const DynamoDBBigSegmentStore = require('./dynamodb_big_segment_store');
+
+module.exports = {
+  DynamoDBFeatureStore,
+  DynamoDBBigSegmentStore,
+};