Drop the VPC field from the AWSMachine spec, look up the VPC from the subnet instead

Author: cnmcavoy

api/v1beta1/awsmachine_conversion.go

@@ -39,7 +39,6 @@ func (src *AWSMachine) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.InstanceMetadataOptions = restored.Spec.InstanceMetadataOptions
 	dst.Spec.PlacementGroupName = restored.Spec.PlacementGroupName
 	dst.Spec.PrivateDNSName = restored.Spec.PrivateDNSName
-	dst.Spec.VPC = restored.Spec.VPC
 	dst.Spec.SecurityGroupOverrides = restored.Spec.SecurityGroupOverrides
 
 	return nil
@@ -89,7 +88,6 @@ func (r *AWSMachineTemplate) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.Template.Spec.InstanceMetadataOptions = restored.Spec.Template.Spec.InstanceMetadataOptions
 	dst.Spec.Template.Spec.PlacementGroupName = restored.Spec.Template.Spec.PlacementGroupName
 	dst.Spec.Template.Spec.PrivateDNSName = restored.Spec.Template.Spec.PrivateDNSName
-	dst.Spec.Template.Spec.VPC = restored.Spec.Template.Spec.VPC
 	dst.Spec.Template.Spec.SecurityGroupOverrides = restored.Spec.Template.Spec.SecurityGroupOverrides
 
 	return nil

api/v1beta1/zz_generated.conversion.go

@@ -109,11 +109,6 @@ type AWSMachineSpec struct {
 	// +optional
 	AdditionalSecurityGroups []AWSResourceReference `json:"additionalSecurityGroups,omitempty"`
 
-	// VPC is a reference to the VPC to use when picking a subnet to use for this
-	// instance. Only valid if the subnet (id or filters) is also specified.
-	// +optional
-	VPC *AWSResourceReference `json:"vpc,omitempty"`
-
 	// Subnet is a reference to the subnet to use for this instance. If not specified,
 	// the cluster subnet will be used.
 	// +optional

api/v1beta2/awsmachine_types.go

@@ -912,36 +912,6 @@ spec:
                   built-in support for gzip-compressed user data user data stored
                   in aws secret manager is always gzip-compressed.
                 type: boolean
-              vpc:
-                description: VPC is a reference to the VPC to use when picking a subnet
-                  to use for this instance. Only valid if the subnet (id or filters)
-                  is also specified.
-                properties:
-                  filters:
-                    description: 'Filters is a set of key/value pairs used to identify
-                      a resource They are applied according to the rules defined by
-                      the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html'
-                    items:
-                      description: Filter is a filter used to identify an AWS resource.
-                      properties:
-                        name:
-                          description: Name of the filter. Filter names are case-sensitive.
-                          type: string
-                        values:
-                          description: Values includes one or more filter values.
-                            Filter values are case-sensitive.
-                          items:
-                            type: string
-                          type: array
-                      required:
-                      - name
-                      - values
-                      type: object
-                    type: array
-                  id:
-                    description: ID of resource
-                    type: string
-                type: object
             required:
             - instanceType
             type: object

api/v1beta2/zz_generated.deepcopy.go

@@ -876,38 +876,6 @@ spec:
                           cloud-init has built-in support for gzip-compressed user
                           data user data stored in aws secret manager is always gzip-compressed.
                         type: boolean
-                      vpc:
-                        description: VPC is a reference to the VPC to use when picking
-                          a subnet to use for this instance. Only valid if the subnet
-                          (id or filters) is also specified.
-                        properties:
-                          filters:
-                            description: 'Filters is a set of key/value pairs used
-                              to identify a resource They are applied according to
-                              the rules defined by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html'
-                            items:
-                              description: Filter is a filter used to identify an
-                                AWS resource.
-                              properties:
-                                name:
-                                  description: Name of the filter. Filter names are
-                                    case-sensitive.
-                                  type: string
-                                values:
-                                  description: Values includes one or more filter
-                                    values. Filter values are case-sensitive.
-                                  items:
-                                    type: string
-                                  type: array
-                              required:
-                              - name
-                              - values
-                              type: object
-                            type: array
-                          id:
-                            description: ID of resource
-                            type: string
-                        type: object
                     required:
                     - instanceType
                     type: object

config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml

@@ -530,10 +530,6 @@ func mockedCreateSecretCall(s *mock_services.MockSecretInterfaceMockRecorder) {
 func mockedCreateInstanceCalls(m *mocks.MockEC2APIMockRecorder) {
 	m.DescribeInstancesWithContext(context.TODO(), gomock.Eq(&ec2.DescribeInstancesInput{
 		Filters: []*ec2.Filter{
-			{
-				Name:   aws.String("vpc-id"),
-				Values: aws.StringSlice([]string{""}),
-			},
 			{
 				Name:   aws.String("tag:sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"),
 				Values: aws.StringSlice([]string{"owned"}),
@@ -642,10 +638,6 @@ func mockedCreateInstanceCalls(m *mocks.MockEC2APIMockRecorder) {
 			Name:   aws.String("state"),
 			Values: aws.StringSlice([]string{"pending", "available"}),
 		},
-		{
-			Name:   aws.String("vpc-id"),
-			Values: aws.StringSlice([]string{""}),
-		},
 		{
 			Name:   aws.String("subnet-id"),
 			Values: aws.StringSlice([]string{"subnet-1"}),

config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml

@@ -120,6 +120,30 @@ spec:
 
 Users may either specify `failureDomain` on the Machine or MachineDeployment objects, _or_ users may explicitly specify subnet IDs on the AWSMachine or AWSMachineTemplate objects. If both are specified, the subnet ID is used and the `failureDomain` is ignored.
 
+### Placing EC2 Instances in Specific External VPCs
+
+CAPA clusters are deployed within a single VPC, but it's possible to place machines that live in external VPCs. For this kind of configuration, we assume that all the VPCs have the ability to communicate, either through external peering, a transit gateway, or some other mechanism already established outside of CAPA. CAPA will not create a tunnel or manage the network configuration for any secondary VPCs.
+
+The AWSMachineTemplate `subnet` field allows specifying filters or specific subnet ids for worker machine placement. If the filters or subnet id is specified in a secondary VPC, CAPA will place the machine in that VPC and subnet.
+
+```yaml
+spec:
+  template:
+    spec:
+      subnet:
+        filters:
+          name: "vpc-id"
+          values:
+            - "secondary-vpc-id"
+      securityGroupOverrides:
+        node: sg-04e870a3507a5ad2c5c8c2
+        node-eks-additional: sg-04e870a3507a5ad2c5c8c1
+```
+
+#### Caveats/Notes
+
+CAPA helpfully creates security groups for various roles in the cluster and automatically attaches them to workers. However, security groups are tied to a specific VPC, so workers placed in a VPC outside of the cluster will need to have these security groups created by some external process first and set in the `securityGroupOverrides` field, otherwise the ec2 creation will fail.
+
 ### Security Groups
 
 To use existing security groups for instances for a cluster, add this to the AWSCluster specification:
@@ -147,6 +171,15 @@ spec:
     - ...
 ```
 
+It's also possible to override the cluster security groups for an individual AWSMachine or AWSMachineTemplate:
+
+```yaml
+spec:
+  SecurityGroupOverrides:
+    node: sg-04e870a3507a5ad2c5c8c2
+    node-eks-additional: sg-04e870a3507a5ad2c5c8c1
+```
+
 ### Control Plane Load Balancer
 
 The cluster control plane is accessed through a Classic ELB. By default, Cluster API creates the Classic ELB. To use an existing Classic ELB, add its name to the AWSCluster specification:

controllers/awsmachine_controller_test.go

@@ -43,13 +43,8 @@ import (
 func (s *Service) GetRunningInstanceByTags(scope *scope.MachineScope) (*infrav1.Instance, error) {
 	s.scope.Debug("Looking for existing machine instance by tags")
 
-	vpcID, err := s.getVPCID(scope)
-	if err != nil {
-		return nil, err
-	}
 	input := &ec2.DescribeInstancesInput{
 		Filters: []*ec2.Filter{
-			filter.EC2.VPC(vpcID),
 			filter.EC2.ClusterOwned(s.scope.Name()),
 			filter.EC2.Name(scope.Name()),
 			filter.EC2.InstanceStates(ec2.InstanceStateNamePending, ec2.InstanceStateNameRunning),
@@ -312,13 +307,6 @@ func (s *Service) findSubnet(scope *scope.MachineScope) (string, error) {
 		criteria := []*ec2.Filter{
 			filter.EC2.SubnetStates(ec2.SubnetStatePending, ec2.SubnetStateAvailable),
 		}
-		if !scope.IsExternallyManaged() {
-			vpcID, err := s.getVPCID(scope)
-			if err != nil {
-				return "", fmt.Errorf("failed to lookup vpc for subnets: %w", err)
-			}
-			criteria = append(criteria, filter.EC2.VPC(vpcID))
-		}
 		if scope.AWSMachine.Spec.Subnet.ID != nil {
 			criteria = append(criteria, &ec2.Filter{Name: aws.String("subnet-id"), Values: aws.StringSlice([]string{*scope.AWSMachine.Spec.Subnet.ID})})
 		}
@@ -353,6 +341,11 @@ func (s *Service) findSubnet(scope *scope.MachineScope) (string, error) {
 			}
 			filtered = append(filtered, subnet)
 		}
+		// prefer a subnet in the cluster VPC if multiple match
+		clusterVPC := s.scope.VPC().ID
+		sort.SliceStable(filtered, func(i, j int) bool {
+			return strings.Compare(*filtered[i].VpcId, clusterVPC) > strings.Compare(*filtered[j].VpcId, clusterVPC)
+		})
 		if len(filtered) == 0 {
 			errMessage = fmt.Sprintf("failed to run machine %q, found %d subnets matching criteria but post-filtering failed.",
 				scope.Name(), len(subnets)) + errMessage
@@ -403,30 +396,6 @@ func (s *Service) findSubnet(scope *scope.MachineScope) (string, error) {
 	}
 }
 
-func (s *Service) getVPCID(scope *scope.MachineScope) (string, error) {
-	if scope.AWSMachine.Spec.VPC == nil {
-		return s.scope.VPC().ID, nil
-	}
-	if scope.AWSMachine.Spec.VPC.ID != nil {
-		return *scope.AWSMachine.Spec.VPC.ID, nil
-	}
-	filters := make([]*ec2.Filter, 0, len(scope.AWSMachine.Spec.VPC.Filters))
-	for _, filter := range scope.AWSMachine.Spec.VPC.Filters {
-		filters = append(filters, &ec2.Filter{
-			Name:   aws.String(filter.Name),
-			Values: aws.StringSlice(filter.Values),
-		})
-	}
-	out, err := s.EC2Client.DescribeVpcsWithContext(context.TODO(), &ec2.DescribeVpcsInput{Filters: filters})
-	if err != nil {
-		return "", err
-	}
-	if out != nil && len(out.Vpcs) == 1 {
-		return *out.Vpcs[0].VpcId, nil
-	}
-	return "", fmt.Errorf("ambigious VPC filters, only one VPC expected, got %v", out)
-}
-
 // getFilteredSubnets fetches subnets filtered based on the criteria passed.
 func (s *Service) getFilteredSubnets(criteria ...*ec2.Filter) ([]*ec2.Subnet, error) {
 	out, err := s.EC2Client.DescribeSubnetsWithContext(context.TODO(), &ec2.DescribeSubnetsInput{Filters: criteria})