Merge branch 'fix/modify-efs-FSxL-recipes' into 'develop'

fix: allow efs and FSxL associate to optionally provided sg and make efs TLS statement optional

See merge request mwvaughn/aws-hpc-recipes!158

Author: Lenox Hsu

recipes/storage/efs_simple/assets/main.yaml

@@ -9,12 +9,14 @@ Metadata:
         Parameters:
           - AutomaticBackups
           - ThroughputMode
+          - EnforceTLS
       - Label:
           default: Networking and Access
         Parameters:
           - VpcId
           - SubnetIds
           - SubnetCount
+          - SecurityGroupName
 
 Parameters:
   VpcId:
@@ -45,6 +47,17 @@ Parameters:
     AllowedValues:
          - "elastic"
          - "bursting"
+  EnforceTLS:
+    Description: Enforce TLS for data in transit
+    Type: String
+    Default: "true"
+    AllowedValues:
+      - "true"
+      - "false"
+  SecurityGroupName:
+    Type: String
+    Description: (Optional) An existing security group to associate to the file system. If none is provided, a new security group will be created.
+    Default: ""
 
 Conditions:
   1AZCondition: !Or
@@ -54,6 +67,9 @@ Conditions:
     - !Equals [!Ref 'SubnetCount', '2']
     - !Condition '3AZCondition'
   3AZCondition: !Equals [!Ref 'SubnetCount', '3']
+  EnforceTLSCondition: !Equals ["true", !Ref EnforceTLS]
+  CreateSecurityGroup: !Equals ["", !Ref SecurityGroupName]
+  UseExistingSecurityGroup: !Not [!Equals ["", !Ref SecurityGroupName]]
 
 Resources:
 
@@ -80,14 +96,17 @@ Resources:
             Condition:
               Bool:
                 'elasticfilesystem:AccessedViaMountTarget': 'true'
-          - Sid: efs-enforce-tls
-            Effect: Deny
-            Principal:
-              AWS: '*'
-            Action: '*'
-            Condition:
-              Bool:
-                'aws:SecureTransport': 'false'
+          - !If
+            - EnforceTLSCondition
+            - Sid: efs-enforce-tls
+              Effect: Deny
+              Principal:
+                AWS: '*'
+              Action: '*'
+              Condition:
+                Bool:
+                  'aws:SecureTransport': 'false'
+            - !Ref 'AWS::NoValue'
       BackupPolicy: 
         Status: !Ref AutomaticBackups
       Encrypted: true
@@ -96,6 +115,7 @@ Resources:
   # Security group
   EfsSecurityGroup:
     Type: AWS::EC2::SecurityGroup
+    Condition: CreateSecurityGroup
     Properties: 
       GroupDescription: !Sub 
        - 'Allows traffic to EFS filesystem ${FileSystemId}'
@@ -120,7 +140,10 @@ Resources:
       Description: Allow incoming traffic to EFS from members of security group
       FromPort: 2049
       ToPort: 2049
-      GroupId: !Ref EfsSecurityGroup
+      GroupId: !If 
+          - CreateSecurityGroup
+          - !Ref EfsSecurityGroup
+          - !GetAtt SecurityGroupLookup.GroupId
       SourceSecurityGroupId: !Ref EfsClientSecurityGroup
 
   EfsClientSecurityGroupOutboundRule:
@@ -131,35 +154,112 @@ Resources:
       FromPort: 2049
       ToPort: 2049
       GroupId: !Ref EfsClientSecurityGroup
-      DestinationSecurityGroupId: !Ref EfsSecurityGroup
+      DestinationSecurityGroupId: !If 
+          - CreateSecurityGroup
+          - !Ref EfsSecurityGroup
+          - !GetAtt SecurityGroupLookup.GroupId
 
   EfsMountTarget1:
     Type: AWS::EFS::MountTarget
     Condition: 1AZCondition
     Properties:
          FileSystemId: !Ref EfsFilesystem
-         SecurityGroups:
-              - !Ref EfsSecurityGroup
+         SecurityGroups: !If 
+                - CreateSecurityGroup
+                - [!Ref EfsSecurityGroup]
+                - [!GetAtt SecurityGroupLookup.GroupId]
          SubnetId: !Select [ 0, !Ref SubnetIds ]
 
   EfsMountTarget2:
     Type: AWS::EFS::MountTarget
     Condition: 2AZCondition
     Properties:
          FileSystemId: !Ref EfsFilesystem
-         SecurityGroups:
-              - !Ref EfsSecurityGroup
+         SecurityGroups: !If 
+                - CreateSecurityGroup
+                - [!Ref EfsSecurityGroup]
+                - [!GetAtt SecurityGroupLookup.GroupId]
          SubnetId: !Select [ 1, !Ref SubnetIds ]
 
   EfsMountTarget3:
     Type: AWS::EFS::MountTarget
     Condition: 3AZCondition
     Properties:
          FileSystemId: !Ref EfsFilesystem
-         SecurityGroups:
-              - !Ref EfsSecurityGroup
+         SecurityGroups: !If 
+                - CreateSecurityGroup
+                - [!Ref EfsSecurityGroup]
+                - [!GetAtt SecurityGroupLookup.GroupId]
          SubnetId: !Select [ 2, !Ref SubnetIds ]
 
+  SecurityGroupLookup:
+    Type: Custom::SecurityGroupLookup
+    Condition: UseExistingSecurityGroup
+    Properties:
+      ServiceToken: !GetAtt SecurityGroupLookupFunction.Arn
+      VpcId: !Ref VpcId
+      GroupName: !Ref SecurityGroupName
+
+  SecurityGroupLookupRole:
+    Type: AWS::IAM::Role
+    Condition: UseExistingSecurityGroup
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: lambda.amazonaws.com
+            Action: sts:AssumeRole
+      ManagedPolicyArns:
+        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
+      Policies:
+        - PolicyName: DescribeSecurityGroups
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action: ec2:DescribeSecurityGroups
+                Resource: '*'
+
+  SecurityGroupLookupFunction:
+    Type: AWS::Lambda::Function
+    Condition: UseExistingSecurityGroup
+    Properties:
+      Runtime: python3.9
+      Handler: index.handler
+      Role: !GetAtt SecurityGroupLookupRole.Arn
+      Code:
+        ZipFile: |
+          import boto3
+          import cfnresponse
+          
+          def handler(event, context):
+            try:
+              if event['RequestType'] in ['Create', 'Update']:
+                ec2 = boto3.client('ec2')
+                vpc_id = event['ResourceProperties']['VpcId']
+                group_name = event['ResourceProperties']['GroupName']
+                
+                response = ec2.describe_security_groups(
+                  Filters=[
+                    {'Name': 'vpc-id', 'Values': [vpc_id]},
+                    {'Name': 'group-name', 'Values': [group_name]}
+                  ]
+                )
+                
+                if len(response['SecurityGroups']) == 0:
+                  raise Exception(f"Security group {group_name} not found in VPC {vpc_id}")
+                
+                group_id = response['SecurityGroups'][0]['GroupId']
+                cfnresponse.send(event, context, cfnresponse.SUCCESS, 
+                               {'GroupId': group_id})
+              else:
+                cfnresponse.send(event, context, cfnresponse.SUCCESS, {})
+            except Exception as e:
+              cfnresponse.send(event, context, cfnresponse.FAILED, 
+                             {'Error': str(e)})
+
 Outputs:
   EFSFilesystemId:
     Description: The ID of the EFS filesystem that has been created

recipes/storage/fsx_lustre/assets/scratch.yaml

@@ -16,6 +16,7 @@ Metadata:
         Parameters:
           - VpcId
           - SubnetId
+          - SecurityGroupName
 
 Parameters:
   VpcId:
@@ -42,11 +43,20 @@ Parameters:
       - "2.15"
       - "2.12"
     Default: "2.15"
+  SecurityGroupName:
+    Type: String
+    Description: (Optional) An existing security group to associate to the file system. If none is provided, a new security group will be created.
+    Default: ""
+
+Conditions:
+  CreateSecurityGroup: !Equals ["", !Ref SecurityGroupName]
+  UseExistingSecurityGroup: !Not [!Equals ["", !Ref SecurityGroupName]]
 
 Resources:
 
   LustreServersSG:
       Type: AWS::EC2::SecurityGroup
+      Condition: CreateSecurityGroup
       Properties:
         GroupDescription: 'Allows traffic to FSx for Lustre filesystem'
         GroupName: !Sub '${AWS::StackName}-fsxl-security-group'
@@ -77,7 +87,10 @@ Resources:
       FromPort: 988
       ToPort: 988
       GroupId: !Ref LustreClientsSG
-      SourceSecurityGroupId: !Ref LustreServersSG
+      SourceSecurityGroupId: !If 
+          - CreateSecurityGroup
+          - !Ref LustreServersSG
+          - !GetAtt SecurityGroupLookup.GroupId
 
   LustreClientsSGfromLustreClients1021:
     Type: AWS::EC2::SecurityGroupIngress
@@ -97,7 +110,10 @@ Resources:
       FromPort: 1021
       ToPort: 1023
       GroupId: !Ref LustreClientsSG
-      SourceSecurityGroupId: !Ref LustreServersSG
+      SourceSecurityGroupId: !If 
+          - CreateSecurityGroup
+          - !Ref LustreServersSG
+          - !GetAtt SecurityGroupLookup.GroupId
 
   LustreClientsSGtoLustreClients988:
     Type: AWS::EC2::SecurityGroupEgress
@@ -117,7 +133,10 @@ Resources:
       FromPort: 988
       ToPort: 988
       GroupId: !Ref LustreClientsSG
-      DestinationSecurityGroupId: !Ref LustreServersSG
+      DestinationSecurityGroupId: !If 
+          - CreateSecurityGroup
+          - !Ref LustreServersSG
+          - !GetAtt SecurityGroupLookup.GroupId
 
   LustreClientsSGtoLustreClients1021:
     Type: AWS::EC2::SecurityGroupEgress
@@ -137,7 +156,10 @@ Resources:
       FromPort: 1021
       ToPort: 1023
       GroupId: !Ref LustreClientsSG
-      DestinationSecurityGroupId: !Ref LustreServersSG
+      DestinationSecurityGroupId: !If 
+          - CreateSecurityGroup
+          - !Ref LustreServersSG
+          - !GetAtt SecurityGroupLookup.GroupId
 
   FSxLFilesystem:
     Type: AWS::FSx::FileSystem
@@ -149,7 +171,10 @@ Resources:
       FileSystemTypeVersion: !Ref LustreVersion
       StorageCapacity: !Ref Capacity
       SecurityGroupIds:
-        - !Ref LustreServersSG
+        - !If 
+          - CreateSecurityGroup
+          - !Ref LustreServersSG
+          - !GetAtt SecurityGroupLookup.GroupId
         - !Ref LustreClientsSG
       SubnetIds:
         - !Ref SubnetId
@@ -159,6 +184,74 @@ Resources:
       Tags:
         - Key: HPCRecipes
           Value: "true"
+  
+  SecurityGroupLookup:
+    Type: Custom::SecurityGroupLookup
+    Condition: UseExistingSecurityGroup
+    Properties:
+      ServiceToken: !GetAtt SecurityGroupLookupFunction.Arn
+      VpcId: !Ref VpcId
+      GroupName: !Ref SecurityGroupName
+
+  SecurityGroupLookupRole:
+    Type: AWS::IAM::Role
+    Condition: UseExistingSecurityGroup
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: lambda.amazonaws.com
+            Action: sts:AssumeRole
+      ManagedPolicyArns:
+        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
+      Policies:
+        - PolicyName: DescribeSecurityGroups
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action: ec2:DescribeSecurityGroups
+                Resource: '*'
+
+  SecurityGroupLookupFunction:
+    Type: AWS::Lambda::Function
+    Condition: UseExistingSecurityGroup
+    Properties:
+      Runtime: python3.9
+      Handler: index.handler
+      Role: !GetAtt SecurityGroupLookupRole.Arn
+      Code:
+        ZipFile: |
+          import boto3
+          import cfnresponse
+          
+          def handler(event, context):
+            try:
+              if event['RequestType'] in ['Create', 'Update']:
+                ec2 = boto3.client('ec2')
+                vpc_id = event['ResourceProperties']['VpcId']
+                group_name = event['ResourceProperties']['GroupName']
+                
+                response = ec2.describe_security_groups(
+                  Filters=[
+                    {'Name': 'vpc-id', 'Values': [vpc_id]},
+                    {'Name': 'group-name', 'Values': [group_name]}
+                  ]
+                )
+                
+                if len(response['SecurityGroups']) == 0:
+                  raise Exception(f"Security group {group_name} not found in VPC {vpc_id}")
+                
+                group_id = response['SecurityGroups'][0]['GroupId']
+                cfnresponse.send(event, context, cfnresponse.SUCCESS, 
+                               {'GroupId': group_id})
+              else:
+                cfnresponse.send(event, context, cfnresponse.SUCCESS, {})
+            except Exception as e:
+              cfnresponse.send(event, context, cfnresponse.FAILED, 
+                             {'Error': str(e)})
 
 Outputs:
   FSxLustreFilesystemId: