From 3c521068fcd44aa1390eb036ec52937b6d133a14 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Jul 2025 20:06:57 +0000 Subject: [PATCH 1/4] Initial plan From b771da849c6054869fe12072afdd35ffd48975df Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Jul 2025 20:17:31 +0000 Subject: [PATCH 2/4] Add BYO CNI cluster automation to hack/aks/Makefile Co-authored-by: vipul-21 <8600441+vipul-21@users.noreply.github.com> --- hack/aks/Makefile | 105 +++++++++++++++++++++++++++++++++++++++++++++ hack/aks/README.md | 61 ++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index d0ce352528..c828885b66 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -33,6 +33,17 @@ CLUSTER ?= $(USER)-$(REGION) GROUP ?= $(CLUSTER) VNET ?= $(CLUSTER) +# BYO CNI cluster configuration +CNI_TYPE ?= cilium +CNS_VERSION ?= v1.5.38 +AZURE_IPAM_VERSION ?= v0.3.0 +CNS_IMAGE_REPO ?= MCR +CILIUM_DIR ?= 1.14 +CILIUM_VERSION_TAG ?= v1.14.8 +CILIUM_IMAGE_REGISTRY ?= acnpublic.azurecr.io +IPV6_HP_BPF_VERSION ?= v0.0.3 +REPO_ROOT ?= $(shell git rev-parse --show-toplevel) + # Long Term Support (LTS) ifeq ($(LTS),true) LTS_ARGS=--k8s-support-plan AKSLongTermSupport --tier premium @@ -109,6 +120,15 @@ vars: ## Show the input vars configured for the cluster commands @echo K8S_VER=$(K8S_VER) @echo LTS_ARGS=$(if $(LTS_ARGS),$(LTS_ARGS),$(LTS)) @echo COMMON_AKS_FIELDS=$(COMMON_AKS_FIELDS) + @echo CNI_TYPE=$(CNI_TYPE) + @echo CNS_VERSION=$(CNS_VERSION) + @echo AZURE_IPAM_VERSION=$(AZURE_IPAM_VERSION) + @echo CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) + @echo CILIUM_DIR=$(CILIUM_DIR) + @echo CILIUM_VERSION_TAG=$(CILIUM_VERSION_TAG) + @echo CILIUM_IMAGE_REGISTRY=$(CILIUM_IMAGE_REGISTRY) + @echo IPV6_HP_BPF_VERSION=$(IPV6_HP_BPF_VERSION) + @echo REPO_ROOT=$(REPO_ROOT) ##@ SWIFT Infra @@ -413,3 +433,88 @@ restart-vmss: ## Restarts the nodes in the cluster scale-nodes: ## Scales the nodes in the cluster $(AZCLI) aks nodepool scale --resource-group $(GROUP) --cluster-name $(CLUSTER) --name $(NODEPOOL) --node-count $(NODE_COUNT) + +##@ BYO CNI Automation + +byocni-cluster-up: ## Create complete BYO CNI cluster with CNS and CNI (default: Cilium) + @echo "Creating BYO CNI cluster with CNS and $(CNI_TYPE)..." + @echo "Variables: CLUSTER=$(CLUSTER), CNS_VERSION=$(CNS_VERSION), CNI_TYPE=$(CNI_TYPE)" + @$(MAKE) validate-cni-type + @$(MAKE) overlay-byocni-nokubeproxy-up + @echo "Cluster created successfully. Deploying CNS..." + @$(MAKE) deploy-cns + @echo "CNS deployed successfully. Deploying $(CNI_TYPE)..." +ifeq ($(CNI_TYPE),cilium) + @$(MAKE) deploy-cilium +else + @echo "Warning: CNI_TYPE=$(CNI_TYPE) not supported yet. Only cilium is currently supported." + @echo "Available CNI types: cilium" + @exit 1 +endif + @echo "BYO CNI cluster setup completed successfully!" + +validate-cni-type: ## Validate the CNI type +ifeq ($(CNI_TYPE),cilium) + @echo "✓ CNI type validation passed: $(CNI_TYPE)" +else + @echo "✗ Error: CNI_TYPE=$(CNI_TYPE) is not supported." + @echo "Available CNI types: cilium" + @echo "Example: make byocni-cluster-up CNI_TYPE=cilium" + @exit 1 +endif + +deploy-cns: ## Deploy CNS to the cluster + @echo "Deploying CNS with version $(CNS_VERSION)..." + cd $(REPO_ROOT) && sudo -E env "PATH=$$PATH" make test-load \ + CNS_ONLY=true \ + CNS_VERSION=$(CNS_VERSION) \ + AZURE_IPAM_VERSION=$(AZURE_IPAM_VERSION) \ + INSTALL_CNS=true \ + INSTALL_OVERLAY=true \ + CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) + +deploy-cilium: ## Deploy Cilium to the cluster + @echo "Deploying Cilium $(CILIUM_VERSION_TAG) from directory v$(CILIUM_DIR)..." + @if [ ! -d "$(REPO_ROOT)/test/integration/manifests/cilium/v$(CILIUM_DIR)" ]; then \ + echo "Error: Cilium directory v$(CILIUM_DIR) not found."; \ + echo "Available versions: $$(ls $(REPO_ROOT)/test/integration/manifests/cilium/ | grep '^v' | tr '\n' ' ')"; \ + exit 1; \ + fi + kubectl apply -f $(REPO_ROOT)/test/integration/manifests/cilium/v$(CILIUM_DIR)/cilium-config/cilium-config.yaml + kubectl apply -f $(REPO_ROOT)/test/integration/manifests/cilium/v$(CILIUM_DIR)/cilium-operator/files + kubectl apply -f $(REPO_ROOT)/test/integration/manifests/cilium/v$(CILIUM_DIR)/cilium-agent/files + @export CILIUM_VERSION_TAG=$(CILIUM_VERSION_TAG) && \ + export CILIUM_IMAGE_REGISTRY=$(CILIUM_IMAGE_REGISTRY) && \ + export IPV6_HP_BPF_VERSION=$(IPV6_HP_BPF_VERSION) && \ + envsubst '$${CILIUM_VERSION_TAG},$${CILIUM_IMAGE_REGISTRY},$${IPV6_HP_BPF_VERSION}' < $(REPO_ROOT)/test/integration/manifests/cilium/v$(CILIUM_DIR)/cilium-operator/templates/deployment.yaml | kubectl apply -f - && \ + envsubst '$${CILIUM_VERSION_TAG},$${CILIUM_IMAGE_REGISTRY},$${IPV6_HP_BPF_VERSION}' < $(REPO_ROOT)/test/integration/manifests/cilium/v$(CILIUM_DIR)/cilium-agent/templates/daemonset.yaml | kubectl apply -f - + +byocni-cluster-vars: ## Show variables for BYO CNI cluster setup + @echo "=== BYO CNI Cluster Configuration ===" + @echo "Basic cluster settings:" + @echo " CLUSTER=$(CLUSTER)" + @echo " GROUP=$(GROUP)" + @echo " REGION=$(REGION)" + @echo " SUB=$(SUB)" + @echo " VNET=$(VNET)" + @echo "" + @echo "CNI configuration:" + @echo " CNI_TYPE=$(CNI_TYPE)" + @echo "" + @echo "CNS configuration:" + @echo " CNS_VERSION=$(CNS_VERSION)" + @echo " AZURE_IPAM_VERSION=$(AZURE_IPAM_VERSION)" + @echo " CNS_IMAGE_REPO=$(CNS_IMAGE_REPO)" + @echo "" + @echo "Cilium configuration:" + @echo " CILIUM_DIR=$(CILIUM_DIR)" + @echo " CILIUM_VERSION_TAG=$(CILIUM_VERSION_TAG)" + @echo " CILIUM_IMAGE_REGISTRY=$(CILIUM_IMAGE_REGISTRY)" + @echo " IPV6_HP_BPF_VERSION=$(IPV6_HP_BPF_VERSION)" + @echo "" + @echo "Repository root:" + @echo " REPO_ROOT=$(REPO_ROOT)" + @echo "" + @echo "Example usage:" + @echo " make byocni-cluster-up CLUSTER=my-cluster SUB=" + @echo " make byocni-cluster-up CLUSTER=my-cluster SUB= CNS_VERSION=v1.6.0 CILIUM_DIR=1.16 CILIUM_VERSION_TAG=v1.16.5" diff --git a/hack/aks/README.md b/hack/aks/README.md index 7e3e89ffa8..f8d4dd7575 100644 --- a/hack/aks/README.md +++ b/hack/aks/README.md @@ -45,4 +45,65 @@ AKS Clusters windows-nodepool-up Add windows node pool down Delete the cluster vmss-restart Restart the nodes of the cluster + +BYO CNI Automation + byocni-cluster-up Create complete BYO CNI cluster with CNS and CNI (default: Cilium) + deploy-cns Deploy CNS to the cluster + deploy-cilium Deploy Cilium to the cluster + byocni-cluster-vars Show variables for BYO CNI cluster setup + validate-cni-type Validate the CNI type +``` + +## BYO CNI Cluster Automation + +The Makefile now includes automated setup for complete BYO CNI clusters with CNS and CNI deployment. + +### Quick Start + +Create a BYO CNI cluster with Cilium (default): +```bash +make byocni-cluster-up CLUSTER=my-cluster SUB= +``` + +### Customization + +All parameters are configurable: +```bash +make byocni-cluster-up \ + CLUSTER=my-cilium-cluster \ + SUB= \ + CNS_VERSION=v1.6.0 \ + CILIUM_DIR=1.16 \ + CILIUM_VERSION_TAG=v1.16.5 \ + CILIUM_IMAGE_REGISTRY=mcr.microsoft.com/containernetworking +``` + +### Available Configuration + +- `CNI_TYPE`: cilium (default) - Future CNI types can be added +- `CNS_VERSION`: CNS version to deploy (default: v1.5.38) +- `AZURE_IPAM_VERSION`: Azure IPAM version (default: v0.3.0) +- `CNS_IMAGE_REPO`: CNS image repository - MCR or ACR (default: MCR) +- `CILIUM_DIR`: Cilium version directory - 1.12, 1.13, 1.14, 1.16, 1.17 (default: 1.14) +- `CILIUM_VERSION_TAG`: Cilium image tag (default: v1.14.8) +- `CILIUM_IMAGE_REGISTRY`: Cilium image registry (default: acnpublic.azurecr.io) +- `IPV6_HP_BPF_VERSION`: IPv6 HP BPF version for dual stack (default: v0.0.3) + +View all configuration variables: +```bash +make byocni-cluster-vars +``` + +### Workflow + +The `byocni-cluster-up` target orchestrates three main steps: + +1. **Cluster Creation**: Uses `overlay-byocni-nokubeproxy-up` to create AKS cluster +2. **CNS Deployment**: Uses root makefile `test-load` target with CNS-specific parameters +3. **CNI Deployment**: Deploys Cilium using manifests from `test/integration/manifests/cilium/` + +Individual steps can also be run separately: +```bash +make deploy-cns +make deploy-cilium ``` From 7a08a48017e3ef45d544519cb4acca79101c0c02 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Jul 2025 20:21:51 +0000 Subject: [PATCH 3/4] Add dual-stack support and comprehensive testing for BYO CNI automation Co-authored-by: vipul-21 <8600441+vipul-21@users.noreply.github.com> --- hack/aks/Makefile | 8 ++++++++ hack/aks/README.md | 1 + 2 files changed, 9 insertions(+) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index c828885b66..0a7d00b834 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -42,6 +42,7 @@ CILIUM_DIR ?= 1.14 CILIUM_VERSION_TAG ?= v1.14.8 CILIUM_IMAGE_REGISTRY ?= acnpublic.azurecr.io IPV6_HP_BPF_VERSION ?= v0.0.3 +DUALSTACK ?= false REPO_ROOT ?= $(shell git rev-parse --show-toplevel) # Long Term Support (LTS) @@ -128,6 +129,7 @@ vars: ## Show the input vars configured for the cluster commands @echo CILIUM_VERSION_TAG=$(CILIUM_VERSION_TAG) @echo CILIUM_IMAGE_REGISTRY=$(CILIUM_IMAGE_REGISTRY) @echo IPV6_HP_BPF_VERSION=$(IPV6_HP_BPF_VERSION) + @echo DUALSTACK=$(DUALSTACK) @echo REPO_ROOT=$(REPO_ROOT) @@ -480,7 +482,12 @@ deploy-cilium: ## Deploy Cilium to the cluster echo "Available versions: $$(ls $(REPO_ROOT)/test/integration/manifests/cilium/ | grep '^v' | tr '\n' ' ')"; \ exit 1; \ fi +ifeq ($(DUALSTACK),true) + @echo "Deploying Cilium with dual-stack configuration..." + kubectl apply -f $(REPO_ROOT)/test/integration/manifests/cilium/v$(CILIUM_DIR)/cilium-config/cilium-config-dualstack.yaml +else kubectl apply -f $(REPO_ROOT)/test/integration/manifests/cilium/v$(CILIUM_DIR)/cilium-config/cilium-config.yaml +endif kubectl apply -f $(REPO_ROOT)/test/integration/manifests/cilium/v$(CILIUM_DIR)/cilium-operator/files kubectl apply -f $(REPO_ROOT)/test/integration/manifests/cilium/v$(CILIUM_DIR)/cilium-agent/files @export CILIUM_VERSION_TAG=$(CILIUM_VERSION_TAG) && \ @@ -511,6 +518,7 @@ byocni-cluster-vars: ## Show variables for BYO CNI cluster setup @echo " CILIUM_VERSION_TAG=$(CILIUM_VERSION_TAG)" @echo " CILIUM_IMAGE_REGISTRY=$(CILIUM_IMAGE_REGISTRY)" @echo " IPV6_HP_BPF_VERSION=$(IPV6_HP_BPF_VERSION)" + @echo " DUALSTACK=$(DUALSTACK)" @echo "" @echo "Repository root:" @echo " REPO_ROOT=$(REPO_ROOT)" diff --git a/hack/aks/README.md b/hack/aks/README.md index f8d4dd7575..cc0d9cbb23 100644 --- a/hack/aks/README.md +++ b/hack/aks/README.md @@ -88,6 +88,7 @@ make byocni-cluster-up \ - `CILIUM_VERSION_TAG`: Cilium image tag (default: v1.14.8) - `CILIUM_IMAGE_REGISTRY`: Cilium image registry (default: acnpublic.azurecr.io) - `IPV6_HP_BPF_VERSION`: IPv6 HP BPF version for dual stack (default: v0.0.3) +- `DUALSTACK`: Enable dual-stack configuration (default: false) View all configuration variables: ```bash From 726916847fe14ebb98bac453de556872a126ebb8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Jul 2025 22:50:29 +0000 Subject: [PATCH 4/4] Add VM SKU parameter, Azure CNI support, and improved image registry documentation Co-authored-by: tamilmani1989 <1103826+tamilmani1989@users.noreply.github.com> --- hack/aks/Makefile | 24 ++++++++++++++++++++---- hack/aks/README.md | 40 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 0a7d00b834..7c4d8c0d18 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -442,15 +442,21 @@ byocni-cluster-up: ## Create complete BYO CNI cluster with CNS and CNI (default: @echo "Creating BYO CNI cluster with CNS and $(CNI_TYPE)..." @echo "Variables: CLUSTER=$(CLUSTER), CNS_VERSION=$(CNS_VERSION), CNI_TYPE=$(CNI_TYPE)" @$(MAKE) validate-cni-type +ifeq ($(CNI_TYPE),azurecni) + @$(MAKE) overlay-up +else @$(MAKE) overlay-byocni-nokubeproxy-up +endif @echo "Cluster created successfully. Deploying CNS..." @$(MAKE) deploy-cns @echo "CNS deployed successfully. Deploying $(CNI_TYPE)..." ifeq ($(CNI_TYPE),cilium) @$(MAKE) deploy-cilium +else ifeq ($(CNI_TYPE),azurecni) + @echo "Azure CNI is already configured in the cluster. No additional CNI deployment needed." else - @echo "Warning: CNI_TYPE=$(CNI_TYPE) not supported yet. Only cilium is currently supported." - @echo "Available CNI types: cilium" + @echo "Warning: CNI_TYPE=$(CNI_TYPE) not supported yet." + @echo "Available CNI types: cilium, azurecni" @exit 1 endif @echo "BYO CNI cluster setup completed successfully!" @@ -458,10 +464,13 @@ endif validate-cni-type: ## Validate the CNI type ifeq ($(CNI_TYPE),cilium) @echo "✓ CNI type validation passed: $(CNI_TYPE)" +else ifeq ($(CNI_TYPE),azurecni) + @echo "✓ CNI type validation passed: $(CNI_TYPE)" else @echo "✗ Error: CNI_TYPE=$(CNI_TYPE) is not supported." - @echo "Available CNI types: cilium" + @echo "Available CNI types: cilium, azurecni" @echo "Example: make byocni-cluster-up CNI_TYPE=cilium" + @echo "Example: make byocni-cluster-up CNI_TYPE=azurecni" @exit 1 endif @@ -504,6 +513,7 @@ byocni-cluster-vars: ## Show variables for BYO CNI cluster setup @echo " REGION=$(REGION)" @echo " SUB=$(SUB)" @echo " VNET=$(VNET)" + @echo " VM_SIZE=$(VM_SIZE)" @echo "" @echo "CNI configuration:" @echo " CNI_TYPE=$(CNI_TYPE)" @@ -511,7 +521,7 @@ byocni-cluster-vars: ## Show variables for BYO CNI cluster setup @echo "CNS configuration:" @echo " CNS_VERSION=$(CNS_VERSION)" @echo " AZURE_IPAM_VERSION=$(AZURE_IPAM_VERSION)" - @echo " CNS_IMAGE_REPO=$(CNS_IMAGE_REPO)" + @echo " CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) (MCR/ACR - affects CNS image paths)" @echo "" @echo "Cilium configuration:" @echo " CILIUM_DIR=$(CILIUM_DIR)" @@ -520,9 +530,15 @@ byocni-cluster-vars: ## Show variables for BYO CNI cluster setup @echo " IPV6_HP_BPF_VERSION=$(IPV6_HP_BPF_VERSION)" @echo " DUALSTACK=$(DUALSTACK)" @echo "" + @echo "Image registry options:" + @echo " - MCR: mcr.microsoft.com/containernetworking" + @echo " - ACR: acnpublic.azurecr.io (default for Cilium)" + @echo " - Custom: your-registry.azurecr.io/path" + @echo "" @echo "Repository root:" @echo " REPO_ROOT=$(REPO_ROOT)" @echo "" @echo "Example usage:" @echo " make byocni-cluster-up CLUSTER=my-cluster SUB=" + @echo " make byocni-cluster-up CLUSTER=my-cluster SUB= CNI_TYPE=azurecni" @echo " make byocni-cluster-up CLUSTER=my-cluster SUB= CNS_VERSION=v1.6.0 CILIUM_DIR=1.16 CILIUM_VERSION_TAG=v1.16.5" diff --git a/hack/aks/README.md b/hack/aks/README.md index cc0d9cbb23..b0fb717182 100644 --- a/hack/aks/README.md +++ b/hack/aks/README.md @@ -65,6 +65,11 @@ Create a BYO CNI cluster with Cilium (default): make byocni-cluster-up CLUSTER=my-cluster SUB= ``` +Create a BYO CNI cluster with Azure CNI: +```bash +make byocni-cluster-up CLUSTER=my-cluster SUB= CNI_TYPE=azurecni +``` + ### Customization All parameters are configurable: @@ -72,21 +77,44 @@ All parameters are configurable: make byocni-cluster-up \ CLUSTER=my-cilium-cluster \ SUB= \ + VM_SIZE=Standard_D2s_v3 \ CNS_VERSION=v1.6.0 \ CILIUM_DIR=1.16 \ CILIUM_VERSION_TAG=v1.16.5 \ CILIUM_IMAGE_REGISTRY=mcr.microsoft.com/containernetworking ``` +Using different image repositories: +```bash +# Using MCR for both CNS and Cilium +make byocni-cluster-up \ + CLUSTER=my-cluster \ + SUB= \ + CNS_IMAGE_REPO=MCR \ + CILIUM_IMAGE_REGISTRY=mcr.microsoft.com/containernetworking + +# Using ACR for CNS and custom registry for Cilium +make byocni-cluster-up \ + CLUSTER=my-cluster \ + SUB= \ + CNS_IMAGE_REPO=ACR \ + CILIUM_IMAGE_REGISTRY=my-registry.azurecr.io/cilium +``` + ### Available Configuration -- `CNI_TYPE`: cilium (default) - Future CNI types can be added +- `CNI_TYPE`: cilium, azurecni (default: cilium) +- `VM_SIZE`: Node VM size (default: Standard_B2s) - `CNS_VERSION`: CNS version to deploy (default: v1.5.38) - `AZURE_IPAM_VERSION`: Azure IPAM version (default: v0.3.0) - `CNS_IMAGE_REPO`: CNS image repository - MCR or ACR (default: MCR) + - MCR: Uses Microsoft Container Registry paths + - ACR: Uses Azure Container Registry paths - `CILIUM_DIR`: Cilium version directory - 1.12, 1.13, 1.14, 1.16, 1.17 (default: 1.14) - `CILIUM_VERSION_TAG`: Cilium image tag (default: v1.14.8) - `CILIUM_IMAGE_REGISTRY`: Cilium image registry (default: acnpublic.azurecr.io) + - Can be set to mcr.microsoft.com/containernetworking for MCR + - Or custom registry URL - `IPV6_HP_BPF_VERSION`: IPv6 HP BPF version for dual stack (default: v0.0.3) - `DUALSTACK`: Enable dual-stack configuration (default: false) @@ -97,12 +125,18 @@ make byocni-cluster-vars ### Workflow -The `byocni-cluster-up` target orchestrates three main steps: +The `byocni-cluster-up` target orchestrates the complete setup workflow: -1. **Cluster Creation**: Uses `overlay-byocni-nokubeproxy-up` to create AKS cluster +**For Cilium CNI (default):** +1. **Cluster Creation**: Uses `overlay-byocni-nokubeproxy-up` to create AKS cluster without CNI 2. **CNS Deployment**: Uses root makefile `test-load` target with CNS-specific parameters 3. **CNI Deployment**: Deploys Cilium using manifests from `test/integration/manifests/cilium/` +**For Azure CNI:** +1. **Cluster Creation**: Uses `overlay-up` to create AKS cluster with Azure CNI pre-configured +2. **CNS Deployment**: Uses root makefile `test-load` target with CNS-specific parameters +3. **CNI Configuration**: Azure CNI is already configured - no additional deployment needed + Individual steps can also be run separately: ```bash make deploy-cns