Don't assign IPs if the network's routing mode is dynamic

Author: vishesh92

.github/workflows/go-coverage.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: 1.22
+          go-version: 1.23
       - name: Run go test with coverage
         run: COVER_PROFILE=coverage.txt make test
       - name: Codecov upload

api/v1beta3/cloudstackfailuredomain_types.go

@@ -72,6 +72,10 @@ type Network struct {
 	// Cloudstack VPC the network belongs to.
 	// +optional
 	VPC *VPC `json:"vpc,omitempty"`
+
+	// Cloudstack Network's routing mode.
+	// +optional
+	NetworkMode string `json:"networkMode,omitempty"`
 }
 
 type VPC struct {

api/v1beta3/cloudstackisolatednetwork_types.go

@@ -68,19 +68,23 @@ type CloudStackIsolatedNetworkStatus struct {
 	// The ID of the lb rule used to assign VMs to the lb.
 	LBRuleID string `json:"loadBalancerRuleID,omitempty"`
 
+	// Network mode of the network.
+	NetworkMode string `json:"networkMode,omitempty"`
+
 	// Ready indicates the readiness of this provider resource.
 	Ready bool `json:"ready"`
 }
 
 func (n *CloudStackIsolatedNetwork) Network() *Network {
 	return &Network{
-		Name:     n.Spec.Name,
-		Type:     "IsolatedNetwork",
-		ID:       n.Spec.ID,
-		Gateway:  n.Spec.Gateway,
-		Netmask:  n.Spec.Netmask,
-		VPC:      n.Spec.VPC,
-		Offering: n.Spec.Offering,
+		Name:        n.Spec.Name,
+		Type:        "IsolatedNetwork",
+		ID:          n.Spec.ID,
+		Gateway:     n.Spec.Gateway,
+		Netmask:     n.Spec.Netmask,
+		VPC:         n.Spec.VPC,
+		Offering:    n.Spec.Offering,
+		NetworkMode: n.Status.NetworkMode,
 	}
 }
 

config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml

@@ -425,6 +425,9 @@ spec:
                               description: Cloudstack Network Netmask the cluster
                                 is built in.
                               type: string
+                            networkMode:
+                              description: Cloudstack Network's routing mode.
+                              type: string
                             offering:
                               description: |-
                                 Cloudstack Network Offering the cluster is built in.

config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackfailuredomains.yaml

@@ -193,6 +193,9 @@ spec:
                         description: Cloudstack Network Netmask the cluster is built
                           in.
                         type: string
+                      networkMode:
+                        description: Cloudstack Network's routing mode.
+                        type: string
                       offering:
                         description: |-
                           Cloudstack Network Offering the cluster is built in.

config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackisolatednetworks.yaml

@@ -254,6 +254,9 @@ spec:
               loadBalancerRuleID:
                 description: The ID of the lb rule used to assign VMs to the lb.
                 type: string
+              networkMode:
+                description: Network mode of the network.
+                type: string
               publicIPID:
                 description: The CS public IP ID to use for the k8s endpoint.
                 type: string

controllers/cloudstackmachine_controller.go

@@ -295,13 +295,17 @@ func (r *CloudStackMachineReconciliationRunner) RequeueIfInstanceNotRunning() (r
 // AddToLBIfNeeded adds instance to load balancer if it is a control plane in an isolated network.
 func (r *CloudStackMachineReconciliationRunner) AddToLBIfNeeded() (retRes ctrl.Result, reterr error) {
 	if util.IsControlPlaneMachine(r.CAPIMachine) && r.FailureDomain.Spec.Zone.Network.Type == cloud.NetworkTypeIsolated {
-		r.Log.Info("Assigning VM to load balancer rule.")
 		if r.IsoNet.Spec.Name == "" {
 			return r.RequeueWithMessage("Could not get required Isolated Network for VM, requeueing.")
 		}
-		err := r.CSUser.AssignVMToLoadBalancerRule(r.IsoNet, *r.ReconciliationSubject.Spec.InstanceID)
-		if err != nil {
-			return ctrl.Result{}, err
+
+		if r.IsoNet.Status.NetworkMode == "" {
+			// For non-routed networks, use load balancer
+			r.Log.Info("Assigning VM to load balancer rule.")
+			err := r.CSUser.AssignVMToLoadBalancerRule(r.IsoNet, *r.ReconciliationSubject.Spec.InstanceID)
+			if err != nil {
+				return ctrl.Result{}, err
+			}
 		}
 	}
 	return ctrl.Result{}, nil

pkg/cloud/isolated_network.go

@@ -148,6 +148,7 @@ func (c *client) CreateIsolatedNetwork(fd *infrav1.CloudStackFailureDomain, isoN
 	isoNet.Spec.ID = resp.Id
 	isoNet.Spec.Gateway = resp.Gateway
 	isoNet.Spec.Netmask = resp.Netmask
+	isoNet.Status.NetworkMode = resp.Ip4routing
 	return c.AddCreatedByCAPCTag(ResourceTypeNetwork, isoNet.Spec.ID)
 }
 
@@ -171,17 +172,28 @@ func (c *client) OpenFirewallRules(isoNet *infrav1.CloudStackIsolatedNetwork) (r
 
 	protocols := []string{NetworkProtocolTCP, NetworkProtocolUDP, NetworkProtocolICMP}
 	for _, proto := range protocols {
-		p := c.cs.Firewall.NewCreateEgressFirewallRuleParams(isoNet.Spec.ID, proto)
+		var err error
+		if isoNet.Status.NetworkMode != "" {
+			p := c.cs.Firewall.NewCreateRoutingFirewallRuleParams(isoNet.Spec.ID, proto)
+			if proto == "icmp" {
+				p.SetIcmptype(-1)
+				p.SetIcmpcode(-1)
+			}
+			_, err = c.cs.Firewall.CreateRoutingFirewallRule(p)
+		} else {
+			p := c.cs.Firewall.NewCreateEgressFirewallRuleParams(isoNet.Spec.ID, proto)
 
-		if proto == "icmp" {
-			p.SetIcmptype(-1)
-			p.SetIcmpcode(-1)
-		}
+			if proto == "icmp" {
+				p.SetIcmptype(-1)
+				p.SetIcmpcode(-1)
+			}
 
-		_, err := c.cs.Firewall.CreateEgressFirewallRule(p)
+			_, err = c.cs.Firewall.CreateEgressFirewallRule(p)
+		}
 		if err != nil &&
-			// Ignore errors regarding already existing fw rules for TCP/UDP
+			// Ignore errors regarding already existing fw rules for TCP/UDP for non-dynamic routing mode
 			!strings.Contains(strings.ToLower(err.Error()), "there is already") &&
+			!strings.Contains(strings.ToLower(err.Error()), "conflicts with rule") &&
 			// Ignore errors regarding already existing fw rule for ICMP
 			!strings.Contains(strings.ToLower(err.Error()), "new rule conflicts with existing rule") {
 			retErr = errors.Wrapf(
@@ -298,6 +310,7 @@ func (c *client) GetOrCreateIsolatedNetwork(
 		isoNet.Spec.ID = net.ID
 		isoNet.Spec.Gateway = net.Gateway
 		isoNet.Spec.Netmask = net.Netmask
+		isoNet.Status.NetworkMode = net.NetworkMode
 		if net.VPC != nil && net.VPC.ID != "" {
 			isoNet.Spec.VPC = net.VPC
 		}
@@ -316,14 +329,17 @@ func (c *client) GetOrCreateIsolatedNetwork(
 		}
 	}
 
-	// Associate Public IP with CloudStackIsolatedNetwork
-	if err := c.AssociatePublicIPAddress(fd, isoNet, csCluster); err != nil {
-		return errors.Wrapf(err, "associating public IP address to csCluster")
-	}
+	// Handle control plane endpoint based on network type
+	if isoNet.Status.NetworkMode == "" {
+		// For non-routed networks, use public IP and load balancer
+		if err := c.AssociatePublicIPAddress(fd, isoNet, csCluster); err != nil {
+			return errors.Wrapf(err, "associating public IP address to csCluster")
+		}
 
-	// Setup a load balancing rule to map VMs to Public IP.
-	if err := c.GetOrCreateLoadBalancerRule(isoNet, csCluster); err != nil {
-		return errors.Wrap(err, "getting or creating load balancing rule")
+		// Setup a load balancing rule to map VMs to Public IP.
+		if err := c.GetOrCreateLoadBalancerRule(isoNet, csCluster); err != nil {
+			return errors.Wrap(err, "getting or creating load balancing rule")
+		}
 	}
 
 	//  Open the Isolated Network on endopint port.

pkg/cloud/network.go

@@ -66,6 +66,7 @@ func (c *client) ResolveNetwork(net *infrav1.Network) (retErr error) {
 		net.Gateway = netDetails.Gateway
 		net.Netmask = netDetails.Netmask
 		net.Offering = netDetails.Networkofferingname
+		net.NetworkMode = netDetails.Ip4routing
 		if netDetails.Vpcid != "" {
 			if net.VPC == nil {
 				net.VPC = &infrav1.VPC{}
@@ -90,6 +91,7 @@ func (c *client) ResolveNetwork(net *infrav1.Network) (retErr error) {
 	net.Gateway = netDetails.Gateway
 	net.Netmask = netDetails.Netmask
 	net.Offering = netDetails.Networkofferingname
+	net.NetworkMode = netDetails.Ip4routing
 	if netDetails.Vpcid != "" {
 		if net.VPC == nil {
 			net.VPC = &infrav1.VPC{}

templates/cluster-template-with-kube-vip.yaml

@@ -68,23 +68,29 @@ spec:
           spec:
             containers:
             - args:
-              - start
+              - manager
               env:
               - name: vip_arp
                 value: "true"
               - name: vip_leaderelection
                 value: "true"
-              - name: vip_address
-                value: ${CLUSTER_ENDPOINT_IP}
               - name: vip_interface
                 value: ens3
+              - name: address
+                value: ${CLUSTER_ENDPOINT_IP}
+              - name: vip_cidr
+                value: "32"
+              - name: cp_enable
+                value: "true"
+              - name: cp_namespace
+                value: kube-system
               - name: vip_leaseduration
                 value: "15"
               - name: vip_renewdeadline
                 value: "10"
               - name: vip_retryperiod
                 value: "2"
-              image: public.ecr.aws/i3w0y7q3/plunder-app/kube-vip:v0.3.7-eks-a-v0.0.0-dev-build.0
+              image: ghcr.io/kube-vip/kube-vip:v0.4.0
               imagePullPolicy: IfNotPresent
               name: kube-vip
               resources: {}
@@ -96,10 +102,14 @@ spec:
               volumeMounts:
               - mountPath: /etc/kubernetes/admin.conf
                 name: kubeconfig
+            hostAliases:
+            - hostnames:
+              - kubernetes
+              ip: 127.0.0.1
             hostNetwork: true
             volumes:
             - hostPath:
-                path: /etc/kubernetes/admin.conf
+                path: /etc/kubernetes/super-admin.conf
                 type: FileOrCreate
               name: kubeconfig
           status: {}