feat(inbox): Add local test function - working locally!

m-szaf · m-szaf · commit 1a1fbddd8f4a · 2023-12-24T09:34:27.000-05:00
diff --git a/src/construct.handler.ts b/src/construct.handler.ts
@@ -1,16 +1,13 @@
 import * as awsLambda from 'aws-lambda';
 
-
 import { getSecrets, getRedisConnection, getQueueDepth, publishMetrics } from './methods';
 
-
 export interface Result {
   region: string;
   vpcId: string;
   count: number;
 }
 
-
 export const handler = async () => {
   try {
     const host = process.env.REDIS_ADDR;
@@ -27,12 +24,12 @@ export const handler = async () => {
       throw new Error('REDIS_PORT environment variable does not parseInt');
     }
 
-    // parse queues
-    const queuesBlob = process.env.QUEUES;
+    // parse queue names
+    const queuesBlob = process.env.QUEUE_NAMES;
     if (!queuesBlob) {
-      throw new Error('QUEUES environment variable not set');
+      throw new Error('QUEUE_NAMES environment variable not set');
     }
-    const queues: string[] = JSON.parse(queuesBlob);
+    const queueNames: string[] = JSON.parse(queuesBlob);
 
     // get username/password
     const redisSecretArn = process.env.REDIS_SECRET_ARN;
@@ -50,7 +47,11 @@ export const handler = async () => {
       throw new Error('REDIS_SECRET_USERNAME_PATH environment variable not set');
     }
 
-    const { username, password } = await getSecrets({redisSecretArn, redisSecretPasswordPath, redisSecretUsernamePath });
+    const { username, password } = await getSecrets({
+      redisSecretArn,
+      redisSecretPasswordPath,
+      redisSecretUsernamePath,
+    });
 
     // connect to redis instance, note this waits until the connection is 'ready'.
     const redisConnection = await getRedisConnection({
@@ -63,23 +64,23 @@ export const handler = async () => {
     });
 
     // capture the values for the queues
-    const results = await Promise.all(queues.map(async(queueName) => {
-      let depth = await getQueueDepth({redisConnection, queueName});
-      if(!depth) {
-        console.error(`Error fetching xlen for ${queueName}`);
-        depth = -1;
-      }
-      return {queueName, depth};
-    }));
-
-    if(results === undefined) {
+    const results = await Promise.all(
+      queueNames.map(async (queueName) => {
+        let depth = await getQueueDepth({ redisConnection, queueName });
+        if (!depth) {
+          console.error(`Error fetching queue depth for ${queueName}`);
+          depth = -1;
+        }
+        return { queueName, depth };
+      }),
+    );
+
+    if (results === undefined) {
       throw new Error('Error fetching ALL queue depths');
     }
 
     // publish the metrics
     publishMetrics(results);
-
-
   } catch (error) {
     console.error('Error publishing metric data', error);
     throw error;
diff --git a/src/construct.ts b/src/construct.ts
@@ -25,9 +25,9 @@ export interface RedisQueueDepthMetricPublisherProps {
    */
   readonly publishFrequency?: Duration;
   /**
-   * List of queues to measure depth for.
+   * List of queue names to measure depth for.
    */
-  readonly queues: string[];
+  readonly queueNames: string[];
   /**
    * Where is Redis?
    */
@@ -115,7 +115,7 @@ export class RedisQueueDepthMetricPublisher extends Construct {
     [
       new PolicyStatement({
         sid: 'fetchRedisSecret',
-        actions: ['WRITEME'],
+        actions: ['secretsmanager:GetSecretValue'],
         resources: [props.redisSecretArn],
       }),
       new PolicyStatement({
@@ -141,12 +141,12 @@ export class RedisQueueDepthMetricPublisher extends Construct {
     }
 
     this.handler
-      .addEnvironment('QUEUES', JSON.stringify(props.queues))
+      .addEnvironment('QUEUE_NAMES', JSON.stringify(props.queueNames))
       .addEnvironment('REDIS_ADDR', props.redisAddr)
       .addEnvironment('REDIS_PORT', props.redisPort ?? '6379')
       .addEnvironment('REDIS_SECRET_ARN', props.redisSecretArn)
       .addEnvironment('REDIS_SECRET_PASSWORD_PATH', props.redisSecretPasswordPath ?? 'password')
-      .addEnvironment('REDIS_SECRET_USERNAME_PATH', props.redisSecretUsernamePath ?? 'username')
+      .addEnvironment('REDIS_SECRET_USERNAME_PATH', props.redisSecretUsernamePath ?? 'username');
 
     this.rule = new Rule(this, 'rule', {
       schedule: Schedule.rate(this.publishFrequency),
diff --git a/src/methods.ts b/src/methods.ts
@@ -23,16 +23,11 @@ export async function getSecrets(props: GetSecretsProps): Promise<GetSecretsResp
   }
 
   return {
-    username: secret[props.redisSecretUsernamePath],
-    password: secret[props.redisSecretPasswordPath],
+    username: secret[props.redisSecretUsernamePath as keyof typeof secret],
+    password: secret[props.redisSecretPasswordPath as keyof typeof secret],
   };
 }
 
-function getCurrentTimeMs(): number {
-  const date = new Date();
-  return date.getTime();
-}
-
 export interface GetRedisConnectionProps extends RedisOptions {
   /**
    * How long to wait for the redis connection to be ready.
@@ -49,51 +44,73 @@ export interface GetRedisConnectionProps extends RedisOptions {
  *
  * @example
  * const redis = await getRedisConnection({
- *   redisAddr: 'redis.example.com',
- *   redisPort: '6379',
+ *   host: 'redis.example.com',
+ *   port: 6379,
  *   username: 'XXXXXXXX',
  *   password: 'XXXXXXXX',
  * }
-*/
+ */
 export async function getRedisConnection(props: GetRedisConnectionProps): Promise<Redis> {
-  const r = await new Redis(props);
+  const redisConnection = new Redis(props);
 
   // Wait until it actually connects, or throw.
-  const timeout = getCurrentTimeMs() + (props.maxWaitForReadyMs ?? 1000);
-  while(r.status !== 'ready') {
-    if (getCurrentTimeMs() > timeout) {
-      throw new Error(`Timeout: Redis connection is ${r.status}, not 'ready'`);
+
+  const timeout = Date.now() + (props.maxWaitForReadyMs ?? 5000);
+  while (redisConnection.status !== 'ready') {
+    if (Date.now() > timeout) {
+      throw new Error(`Timeout: Redis connection is ${redisConnection.status}, not 'ready'`);
     }
-    console.debug(`Redis connection is ${r.status}, waiting...`);
-    await new Promise(resolve => setTimeout(resolve, 100));
+    console.debug(`Redis connection is ${redisConnection.status}, waiting...`);
+    await new Promise((resolve) => setTimeout(resolve, 100));
   }
 
-  return r;
+  return redisConnection;
 }
 
 export interface GetQueueDepthProps {
   redisConnection: Redis;
   queueName: string;
 }
+
 export async function getQueueDepth(props: GetQueueDepthProps): Promise<number> {
-  // TODO: validate this is what we're actually doing.
-  return await props.redisConnection.xlen(props.queueName);
-}
+  let count = 0;
+  let cursor = '0';
+
+  do {
+    const result = await props.redisConnection.scan(cursor, 'MATCH', `${props.queueName}*`, 'COUNT', '1000');
+    cursor = result[0];
+    count += result[1].length;
+  } while (cursor !== '0');
 
+  return count;
+}
 
 export interface PublishMetricsProps {}
 export interface PublishMetricsPayload {
   queueName: string;
   depth: number;
 }
-export async function publishMetrics(payload: PublishMetricsPayload[], options?: MetricsOptions,): Promise<void> {
+export async function publishMetrics(payload: PublishMetricsPayload[], options?: MetricsOptions): Promise<void> {
   const metrics = new Metrics(options);
   payload.forEach((p) => {
     if (p.depth >= 0) {
-     metrics.addMetric(p.queueName, MetricUnits.Count, p.depth)
+      metrics.addMetric(p.queueName, MetricUnits.Count, p.depth);
     }
   });
-  await metrics.publishStoredMetrics();
+  metrics.publishStoredMetrics();
   return;
 }
 
+async function localTest() {
+  const conn = await getRedisConnection({
+    host: 'localhost',
+  });
+  const depth = await getQueueDepth({
+    redisConnection: conn,
+    queueName: 'NOTIFQUEUE:NOTIFQUEUE',
+  });
+
+  console.log({ depth });
+}
+
+// localTest();
diff --git a/test/construct.handler.test.ts b/test/construct.handler.test.ts
@@ -1,4 +1,4 @@
-import { monitor } from '../src/construct.handler';
+import { handler } from '../src/construct.handler';
 
 // Silence log output
 (['log', 'error'] as jest.FunctionPropertyNames<Required<Console>>[]).forEach((func) =>
@@ -19,12 +19,12 @@ interface EnvVarTestCase {
 }
 
 const defaultEnvVars: Record<string, any> = {
-  'REDIS_ADDR': 'fakeAddr',
-  'REDIS_PORT': '12345',
-  'QUEUES': JSON.stringify(['fakeQueue']),
-  'REDIS_SECRET_ARN': 'fakeArn',
-  'REDIS_SECRET_PASSWORD_PATH': 'fakePasswordPath',
-  'REDIS_SECRET_USERNAME_PATH': 'fakeUsernamePath',
+  REDIS_ADDR: 'fakeAddr',
+  REDIS_PORT: '12345',
+  QUEUES: JSON.stringify(['fakeQueue']),
+  REDIS_SECRET_ARN: 'fakeArn',
+  REDIS_SECRET_PASSWORD_PATH: 'fakePasswordPath',
+  REDIS_SECRET_USERNAME_PATH: 'fakeUsernamePath',
 };
 
 describe('environment variable testing', () => {
@@ -35,17 +35,21 @@ describe('environment variable testing', () => {
     Object.keys(defaultEnvVars).forEach((key) => delete env[key]);
   }
 
-  beforeAll(()=> {env = process.env});
+  beforeAll(() => {
+    env = process.env;
+  });
   beforeEach(cleanEnv);
   afterAll(() => {
     cleanEnv();
     process.env = env; // restore original env vars
   });
 
-  it.each<EnvVarTestCase>([{
+  it.each<EnvVarTestCase>([
+    {
       name: 'REDIS_ADDR',
       expectedError: 'REDIS_ADDR environment variable not set',
-  }])('monitor errors when required environment variable', ({name, expectedError}) => {
+    },
+  ])('handler errors when required environment variable', ({ name, expectedError }) => {
     // Mock in ALL the defaultEnvVars EXCEPT the one being tested
     Object.keys(defaultEnvVars).forEach((key) => {
       if (key === name) {
@@ -54,7 +58,7 @@ describe('environment variable testing', () => {
       process.env[key] = defaultEnvVars[key];
     });
 
-    expect(monitor()).rejects.toThrow(expectedError);
+    expect(handler()).rejects.toThrow(expectedError);
   });
 });
 
diff --git a/test/construct.test.ts b/test/construct.test.ts
@@ -17,7 +17,7 @@ const defaultRedisQueueDepthMetricPublisherProps: RedisQueueDepthMetricPublisher
   cwNamespace: 'example',
   cloudwatchLogsRetention: RetentionDays.TWO_WEEKS,
   publishFrequency: Duration.seconds(5),
-  queues: ['fakeQueue', 'fakeQueue2'],
+  queueNames: ['fakeQueue', 'fakeQueue2'],
   redisAddr: 'fakeRedisAddr',
   redisPort: '123456',
   redisSecretArn: 'fakeRedisSecretArn',
@@ -39,14 +39,12 @@ function createRedisQueueDepthMetricPublisher(id: string, props?: RedisQueueDept
 describe.skip('RedisQueueDepthMetricPublisher', () => {
   it('creates resources', () => {
     createRedisQueueDepthMetricPublisher('defaultProps', defaultRedisQueueDepthMetricPublisherProps);
-    template.resourceCountIs('AWS::Lambda::Function', 2); // One 
+    template.resourceCountIs('AWS::Lambda::Function', 2); // One
     expect(redisQueueDepthMetricPublisher.publishFrequency).toEqual(1);
     expect(redisQueueDepthMetricPublisher.regions).toEqual(['us-east-1']);
   });
   it('creates resources with default props', () => {
-    createRedisQueueDepthMetricPublisher('noProps', {
-      
-    });
+    createRedisQueueDepthMetricPublisher('noProps');
     template.resourceCountIs('AWS::Lambda::Function', 2);
     expect(redisQueueDepthMetricPublisher.publishFrequency).toEqual(1);
     expect(redisQueueDepthMetricPublisher.regions).toEqual(['us-east-1']);
