Merge pull request #156 from cloudgraphdev/feature/EP-3205-support-missing-ecs-services

feat(aws): support missing ecs services

Author: m-pizarro

src/enums/relations.ts

@@ -7,6 +7,7 @@ export default {
   apiGatewayRestApi: ['apiGatewayResource', 'apiGatewayStage'],
   route53HostedZone: ['route53Record'],
   emrCluster: ['emrInstance', 'emrStep'],
+  ecsCluster: ['ecsService'],
   ecsService: ['ecsTaskSet', 'ecsTaskDefinition'],
   iamInstanceProfile: ['ec2Instance'],
   apiGatewayDomainName: ['apiGatewayRestApi'],

src/services/ecsCluster/data.ts

@@ -1,121 +1,97 @@
+import CloudGraph from '@cloudgraph/sdk'
 import { Config } from 'aws-sdk'
-import { AWSError } from 'aws-sdk/lib/error'
 import ECS, {
   Cluster,
   ListClustersRequest,
   ListClustersResponse,
 } from 'aws-sdk/clients/ecs'
-import CloudGraph from '@cloudgraph/sdk'
+import { AWSError } from 'aws-sdk/lib/error'
 import groupBy from 'lodash/groupBy'
 import isEmpty from 'lodash/isEmpty'
 import awsLoggerText from '../../properties/logger'
 import { AwsTag, TagMap } from '../../types'
-import { convertAwsTagsToTagMap } from '../../utils/format'
-import AwsErrorLog from '../../utils/errorLog'
 import { initTestEndpoint } from '../../utils'
+import AwsErrorLog from '../../utils/errorLog'
+import { convertAwsTagsToTagMap } from '../../utils/format'
 
 const lt = { ...awsLoggerText }
 const { logger } = CloudGraph
 const serviceName = 'ECS cluster'
 const errorLog = new AwsErrorLog(serviceName)
 const endpoint = initTestEndpoint(serviceName)
-
+const MAX_ITEMS = 100
 export interface RawAwsEcsCluster extends Omit<Cluster, 'Tags'> {
   region: string
   Tags?: TagMap
 }
 
-export default async ({
-  regions,
-  config,
-}: {
-  regions: string
-  config: Config
-}): Promise<{ [region: string]: RawAwsEcsCluster[] }> =>
-  new Promise(async resolve => {
-    /**
-     * Get the arns of all the ECS Clusters
-     */
-    const ecsClusterData: RawAwsEcsCluster[] = []
-    const ecsClusterArns: Array<{ region: string; clusterArns: string[] }> = []
-    const regionPromises = []
-
-    const listClusterArns = async ({
-      ecs,
-      region,
-      token: nextToken = '',
-      resolveRegion,
-    }: {
-      ecs: ECS
-      region: string
-      token?: string
-      resolveRegion: Function
-    }) => {
-      let args: ListClustersRequest = {}
-
-      if (nextToken) {
-        args = { ...args, nextToken }
+export const listClusterArnsForRegion = async (ecs: ECS): Promise<string[]> =>
+  new Promise<string[]>(resolve => {
+    const clusterArnList: string[] = []
+    const args: ListClustersRequest = {}
+    const listAllClusterArns = (token?: string): void => {
+      args.maxResults = MAX_ITEMS
+      if (token) {
+        args.nextToken = token
       }
-
-      return ecs.listClusters(
-        args,
-        (err: AWSError, data: ListClustersResponse) => {
+      try {
+        ecs.listClusters(args, (err: AWSError, data: ListClustersResponse) => {
           if (err) {
             errorLog.generateAwsErrorLog({
               functionName: 'ecs:listClusters',
               err,
             })
           }
 
-          /**
-           * No Cluster data for this region
-           */
           if (isEmpty(data)) {
-            return resolveRegion()
+            return resolve([])
           }
 
-          const { clusterArns, nextToken: token } = data
-
-          logger.debug(lt.fetchedEcsClusters(clusterArns.length))
+          const { clusterArns = [], nextToken } = data || {}
 
-          /**
-           * No Clusters Found
-           */
+          clusterArnList.push(...clusterArns)
 
-          if (isEmpty(clusterArns)) {
-            return resolveRegion()
+          if (nextToken) {
+            listAllClusterArns(nextToken)
+          } else {
+            resolve(clusterArnList)
           }
+        })
+      } catch (error) {
+        resolve([])
+      }
+    }
+    listAllClusterArns()
+  })
 
-          /**
-           * Check to see if there are more
-           */
-
-          if (token) {
-            listClusterArns({ region, token, ecs, resolveRegion })
-          }
-
-          /**
-           * Add the found Clusters to the ecsClusterArns
-           */
-
-          ecsClusterArns.push({ region, clusterArns })
-
-          /**
-           * If this is the last page of data then return
-           */
+export default async ({
+  regions,
+  config,
+}: {
+  regions: string
+  config: Config
+}): Promise<{ [region: string]: RawAwsEcsCluster[] }> =>
+  new Promise(async resolve => {
+    /**
+     * Get all ECS Clusters Arns
+     */
+    const ecsClusterData: RawAwsEcsCluster[] = []
+    const ecsClusterArns: Array<{ region: string; clusterArns: string[] }> = []
+    const regionPromises = []
 
-          if (!token) {
-            resolveRegion()
-          }
+    regions.split(',').forEach(region => {
+      const ecs = new ECS({ ...config, region, endpoint })
+      const regionPromise = new Promise<void>(async resolveRegion => {
+        const clusterArnsList = await listClusterArnsForRegion(ecs)
+        if (!isEmpty(clusterArnsList)) {
+          ecsClusterArns.push({
+            clusterArns: clusterArnsList,
+            region,
+          })
         }
-      )
-    }
+        resolveRegion()
+      })
 
-    regions.split(',').map(region => {
-      const ecs = new ECS({ ...config, region, endpoint })
-      const regionPromise = new Promise<void>(resolveRegion =>
-        listClusterArns({ ecs, region, resolveRegion })
-      )
       regionPromises.push(regionPromise)
     })
 
@@ -126,10 +102,10 @@ export default async ({
      */
 
     const clusterPromises = ecsClusterArns.map(
-      async ({ region, clusterArns: clusters }) =>
+      async ({ region, clusterArns }) =>
         new Promise<void>(resolveEcsData =>
           new ECS({ ...config, region, endpoint }).describeClusters(
-            { clusters },
+            { clusters: clusterArns },
             (err, data) => {
               if (err) {
                 errorLog.generateAwsErrorLog({

src/services/ecsService/data.ts

@@ -1,52 +1,132 @@
-import { Config } from 'aws-sdk'
-import ECS, { Service } from 'aws-sdk/clients/ecs'
 import CloudGraph from '@cloudgraph/sdk'
-import flatMap from 'lodash/flatMap'
+import { Config } from 'aws-sdk'
+import ECS, { ListServicesRequest, Service } from 'aws-sdk/clients/ecs'
 import groupBy from 'lodash/groupBy'
 import isEmpty from 'lodash/isEmpty'
+import services from '../../enums/services'
 import awsLoggerText from '../../properties/logger'
 import { AwsTag, TagMap } from '../../types'
-import { convertAwsTagsToTagMap } from '../../utils/format'
-import AwsErrorLog from '../../utils/errorLog'
 import { initTestEndpoint } from '../../utils'
-import EcsClusterClass from '../ecsCluster'
-import { RawAwsEcsCluster } from '../ecsCluster/data'
+import AwsErrorLog from '../../utils/errorLog'
+import { convertAwsTagsToTagMap } from '../../utils/format'
+import { listClusterArnsForRegion, RawAwsEcsCluster } from '../ecsCluster/data'
 
 const lt = { ...awsLoggerText }
 const { logger } = CloudGraph
 const serviceName = 'ECS service'
 const errorLog = new AwsErrorLog(serviceName)
 const endpoint = initTestEndpoint(serviceName)
-
+const MAX_ITEMS = 100
 export interface RawAwsEcsService extends Service {
   region: string
   Tags?: TagMap
+  ClusterArn: string
 }
 
+export const listServicesArnForRegion = async (
+  ecs: ECS,
+  cluster: string
+): Promise<string[]> =>
+  new Promise<string[]>(resolve => {
+    const serviceArnList: string[] = []
+    const getRestApisOpts: ListServicesRequest = {
+      cluster,
+      maxResults: MAX_ITEMS,
+    }
+    const listAllServiceArns = (token?: string): void => {
+      if (token) {
+        getRestApisOpts.nextToken = token
+      }
+      try {
+        ecs.listServices({ cluster }, (err, data) => {
+          if (err) {
+            errorLog.generateAwsErrorLog({
+              functionName: 'ecs:listServices',
+              err,
+            })
+          }
+
+          if (isEmpty(data)) {
+            return resolve([])
+          }
+
+          const { serviceArns = [], nextToken } = data
+
+          serviceArnList.push(...serviceArns)
+
+          if (nextToken) {
+            listAllServiceArns(nextToken)
+          } else {
+            resolve(serviceArnList)
+          }
+        })
+      } catch (error) {
+        resolve([])
+      }
+    }
+    listAllServiceArns()
+  })
+
 export default async ({
   regions,
   config,
+  rawData,
 }: {
   regions: string
   config: Config
+  rawData: any
 }): Promise<{
   [region: string]: RawAwsEcsService[]
 }> =>
   new Promise(async resolve => {
     const ecsServices: RawAwsEcsService[] = []
-    const ecsClusterClass = new EcsClusterClass({ logger: CloudGraph.logger })
-    const clusterResult = await ecsClusterClass.getData({
-      ...config,
-      regions,
-    })
-    const ecsClusters: RawAwsEcsCluster[] = flatMap(clusterResult)
+    const regionPromises = []
+    const ecsClusterData: Array<{ region: string; clusterArn: string }> = []
+
+    const existingData: { [property: string]: RawAwsEcsCluster[] } =
+      rawData?.find(({ name }) => name === services.ecsCluster)?.data || {}
+
+    if (isEmpty(existingData)) {
+      // Refresh data
+      regions.split(',').map(region => {
+        const ecsClient = new ECS({ ...config, region, endpoint })
+        const regionPromise = new Promise<void>(async resolveRegion => {
+          const clusterArnList = await listClusterArnsForRegion(ecsClient)
+          if (!isEmpty(clusterArnList)) {
+            ecsClusterData.push(
+              ...clusterArnList.map(arn => ({
+                clusterArn: arn,
+                region,
+              }))
+            )
+          }
+          resolveRegion()
+        })
+        regionPromises.push(regionPromise)
+        return null
+      })
+      await Promise.all(regionPromises)
+    } else {
+      // Uses existing data
+      regions.split(',').map(region => {
+        if (!isEmpty(existingData[region])) {
+          ecsClusterData.push(
+            ...existingData[region].map(ecsCluster => ({
+              clusterArn: ecsCluster.clusterArn,
+              region,
+            }))
+          )
+        }
+        return null
+      })
+    }
 
     /**
      * Get the arns of all the services
      */
     let ecsServiceArns: any = await Promise.all(
-      ecsClusters.map(
-        async ({ clusterName: cluster, region }) =>
+      ecsClusterData.map(
+        async ({ clusterArn: cluster, region }) =>
           new Promise(resolveEcsData =>
             new ECS({ ...config, region, endpoint }).listServices(
               { cluster },
@@ -81,10 +161,10 @@ export default async ({
      * Get all the details for each service
      */
     const ecsServicePromises = ecsServiceArns.map(
-      async ({ region, serviceArns: services, cluster }) =>
+      async ({ region, serviceArns, cluster }) =>
         new Promise<void>(resolveEcsData =>
           new ECS({ ...config, region, endpoint }).describeServices(
-            { services, cluster },
+            { services: serviceArns, cluster },
             (err, data) => {
               if (err) {
                 errorLog.generateAwsErrorLog({
@@ -97,14 +177,15 @@ export default async ({
                 return resolveEcsData()
               }
 
-              const { services = [] } = data
+              const { services: servicesList = [] } = data
 
-              logger.debug(lt.fetchedEcsServices(services.length))
+              logger.debug(lt.fetchedEcsServices(servicesList.length))
 
               ecsServices.push(
-                ...services.map(service => ({
+                ...servicesList.map(service => ({
                   region,
                   ...service,
+                  ClusterArn: cluster,
                   Tags: convertAwsTagsToTagMap(service.tags as AwsTag[]),
                 }))
               )