feat: Chained cni conflist with cilium

.pipelines/build/scripts/cni.sh

@@ -61,5 +61,6 @@ pushd "$REPO_ROOT"/cni
   cp azure-$OS-swift-overlay.conflist "$OUT_DIR"/files/azure-swift-overlay.conflist
   cp azure-$OS-swift-overlay-dualstack.conflist "$OUT_DIR"/files/azure-swift-overlay-dualstack.conflist
   cp azure-$OS-multitenancy.conflist "$OUT_DIR"/files/multitenancy.conflist
+  cp azure-chained-cilium.conflist "$OUT_DIR"/files/azure-chained-cilium.conflist
   cp "$REPO_ROOT"/telemetry/azure-vnet-telemetry.config "$OUT_DIR"/files/azure-vnet-telemetry.config
 popd

cni/Dockerfile

@@ -33,6 +33,7 @@ COPY --from=azure-vnet /azure-container-networking/cni/azure-linux-multitenancy-
 COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS-swift-overlay.conflist /payload/azure-swift-overlay.conflist
 COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS-swift-overlay-dualstack.conflist /payload/azure-swift-overlay-dualstack.conflist
 COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS-multitenancy.conflist /payload/azure-multitenancy.conflist
+COPY --from=azure-vnet /azure-container-networking/cni/azure-chained-cilium.conflist /payload/azure-chained-cilium.conflist
 COPY --from=azure-vnet /azure-container-networking/telemetry/azure-vnet-telemetry.config /payload/azure-vnet-telemetry.config
 RUN cd /payload && sha256sum * > sum.txt
 RUN gzip --verbose --best --recursive /payload && for f in /payload/*.gz; do mv -- "$f" "${f%%.gz}"; done

cni/azure-chained-cilium.conflist

@@ -0,0 +1,21 @@
+{
+	"cniVersion": "0.3.0",
+	"name": "azure",
+	"plugins": [
+		{
+			"type": "azure-vnet",
+			"mode": "transparent",
+			"ipsToRouteViaHost": [
+				"169.254.20.10"
+			],
+			"executionMode": "v4swift",
+			"ipam": {
+				"type": "azure-cns"
+			}
+		},
+		{
+			"name": "cilium",
+			"type": "cilium-cni"
+		}
+	]
+}

cns/cniconflist/generator.go

@@ -71,6 +71,10 @@ type SWIFTGenerator struct {
 	Writer io.WriteCloser
 }
 
+type AzureCNIChainedCiliumGenerator struct {
+	Writer io.WriteCloser
+}
+
 func (v *V4OverlayGenerator) Close() error {
 	if err := v.Writer.Close(); err != nil {
 		return errors.Wrap(err, "error closing generator")
@@ -110,3 +114,11 @@ func (v *SWIFTGenerator) Close() error {
 
 	return nil
 }
+
+func (v *AzureCNIChainedCiliumGenerator) Close() error {
+	if err := v.Writer.Close(); err != nil {
+		return errors.Wrap(err, "error closing generator")
+	}
+
+	return nil
+}

cns/cniconflist/generator_linux.go

@@ -161,3 +161,33 @@ func (v *SWIFTGenerator) Generate() error {
 
 	return nil
 }
+
+func (v *AzureCNIChainedCiliumGenerator) Generate() error {
+	conflist := cniConflist{
+		CNIVersion: azurecniVersion,
+		Name:       azureName,
+		Plugins: []any{
+			cni.NetworkConfig{
+				Type:              azureType,
+				Mode:              cninet.OpModeTransparent,
+				IPsToRouteViaHost: []string{nodeLocalDNSIP},
+				ExecutionMode:     string(util.V4Swift),
+				IPAM: cni.IPAM{
+					Type: network.AzureCNS,
+				},
+			},
+			cni.NetworkConfig{
+				Name: ciliumcniName,
+				Type: ciliumcniType,
+			},
+		},
+	}
+
+	enc := json.NewEncoder(v.Writer)
+	enc.SetIndent("", "\t")
+	if err := enc.Encode(conflist); err != nil {
+		return errors.Wrap(err, "error encoding conflist to json")
+	}
+
+	return nil
+}

cns/cniconflist/generator_linux_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 
 	"github.com/Azure/azure-container-networking/cns/cniconflist"
-	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 type bufferWriteCloser struct {
@@ -23,13 +23,13 @@ func TestGenerateV4OverlayConflist(t *testing.T) {
 	buffer := new(bytes.Buffer)
 	g := cniconflist.V4OverlayGenerator{Writer: &bufferWriteCloser{buffer}}
 	err := g.Generate()
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
 	fixtureBytes, err := os.ReadFile(fixture)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
 	// remove newlines and carriage returns in case these UTs are running on Windows
-	assert.Equal(t, removeNewLines(fixtureBytes), removeNewLines(buffer.Bytes()))
+	require.Equal(t, removeNewLines(fixtureBytes), removeNewLines(buffer.Bytes()))
 }
 
 func TestGenerateDualStackOverlayConflist(t *testing.T) {
@@ -38,13 +38,13 @@ func TestGenerateDualStackOverlayConflist(t *testing.T) {
 	buffer := new(bytes.Buffer)
 	g := cniconflist.DualStackOverlayGenerator{Writer: &bufferWriteCloser{buffer}}
 	err := g.Generate()
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
 	fixtureBytes, err := os.ReadFile(fixture)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
 	// remove newlines and carriage returns in case these UTs are running on Windows
-	assert.Equal(t, removeNewLines(fixtureBytes), removeNewLines(buffer.Bytes()))
+	require.Equal(t, removeNewLines(fixtureBytes), removeNewLines(buffer.Bytes()))
 }
 
 func TestGenerateOverlayConflist(t *testing.T) {
@@ -53,13 +53,13 @@ func TestGenerateOverlayConflist(t *testing.T) {
 	buffer := new(bytes.Buffer)
 	g := cniconflist.OverlayGenerator{Writer: &bufferWriteCloser{buffer}}
 	err := g.Generate()
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
 	fixtureBytes, err := os.ReadFile(fixture)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
 	// remove newlines and carriage returns in case these UTs are running on Windows
-	assert.Equal(t, removeNewLines(fixtureBytes), removeNewLines(buffer.Bytes()))
+	require.Equal(t, removeNewLines(fixtureBytes), removeNewLines(buffer.Bytes()))
 }
 
 func TestGenerateCiliumConflist(t *testing.T) {
@@ -68,13 +68,13 @@ func TestGenerateCiliumConflist(t *testing.T) {
 	buffer := new(bytes.Buffer)
 	g := cniconflist.CiliumGenerator{Writer: &bufferWriteCloser{buffer}}
 	err := g.Generate()
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
 	fixtureBytes, err := os.ReadFile(fixture)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
 	// remove newlines and carriage returns in case these UTs are running on Windows
-	assert.Equal(t, removeNewLines(fixtureBytes), removeNewLines(buffer.Bytes()))
+	require.Equal(t, removeNewLines(fixtureBytes), removeNewLines(buffer.Bytes()))
 }
 
 func TestGenerateSWIFTConflist(t *testing.T) {
@@ -83,13 +83,28 @@ func TestGenerateSWIFTConflist(t *testing.T) {
 	buffer := new(bytes.Buffer)
 	g := cniconflist.SWIFTGenerator{Writer: &bufferWriteCloser{buffer}}
 	err := g.Generate()
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
 	fixtureBytes, err := os.ReadFile(fixture)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
 	// remove newlines and carriage returns in case these UTs are running on Windows
-	assert.Equal(t, removeNewLines(fixtureBytes), removeNewLines(buffer.Bytes()))
+	require.Equal(t, removeNewLines(fixtureBytes), removeNewLines(buffer.Bytes()))
+}
+
+func TestGenerateAzurecniCiliumConflist(t *testing.T) {
+	fixture := "testdata/fixtures/azure-chained-cilium.conflist"
+
+	buffer := new(bytes.Buffer)
+	g := cniconflist.AzureCNIChainedCiliumGenerator{Writer: &bufferWriteCloser{buffer}}
+	err := g.Generate()
+	require.NoError(t, err)
+
+	fixtureBytes, err := os.ReadFile(fixture)
+	require.NoError(t, err)
+
+	// remove newlines and carriage returns in case these UTs are running on Windows
+	require.Equal(t, removeNewLines(fixtureBytes), removeNewLines(buffer.Bytes()))
 }
 
 // removeNewLines will remove the newlines and carriage returns from the byte slice

cns/cniconflist/generator_windows.go

@@ -25,3 +25,7 @@ func (v *CiliumGenerator) Generate() error {
 func (v *SWIFTGenerator) Generate() error {
 	return errNotImplemented
 }
+
+func (v *AzureCNIChainedCiliumGenerator) Generate() error {
+	return errNotImplemented
+}

cns/cniconflist/testdata/fixtures/azure-chained-cilium.conflist

@@ -0,0 +1,34 @@
+{
+	"cniVersion": "0.3.0",
+	"name": "azure",
+	"plugins": [
+		{
+			"type": "azure-vnet",
+			"mode": "transparent",
+			"ipsToRouteViaHost": [
+				"169.254.20.10"
+			],
+			"executionMode": "v4swift",
+			"ipam": {
+				"type": "azure-cns"
+			},
+			"dns": {},
+			"runtimeConfig": {
+				"dns": {}
+			},
+			"windowsSettings": {}
+		},
+		{
+			"name": "cilium",
+			"type": "cilium-cni",
+			"ipam": {
+				"type": ""
+			},
+			"dns": {},
+			"runtimeConfig": {
+				"dns": {}
+			},
+			"windowsSettings": {}
+		}
+	]
+}

cns/service/main.go

@@ -121,11 +121,12 @@ const (
 type cniConflistScenario string
 
 const (
-	scenarioV4Overlay        cniConflistScenario = "v4overlay"
-	scenarioDualStackOverlay cniConflistScenario = "dualStackOverlay"
-	scenarioOverlay          cniConflistScenario = "overlay"
-	scenarioCilium           cniConflistScenario = "cilium"
-	scenarioSWIFT            cniConflistScenario = "swift"
+	scenarioV4Overlay             cniConflistScenario = "v4overlay"
+	scenarioDualStackOverlay      cniConflistScenario = "dualStackOverlay"
+	scenarioOverlay               cniConflistScenario = "overlay"
+	scenarioCilium                cniConflistScenario = "cilium"
+	scenarioSWIFT                 cniConflistScenario = "swift"
+	scenarioAzurecniChainedCilium cniConflistScenario = "azurecni-chained-cilium"
 )
 
 var (
@@ -623,6 +624,8 @@ func main() {
 			conflistGenerator = &cniconflist.CiliumGenerator{Writer: writer}
 		case scenarioSWIFT:
 			conflistGenerator = &cniconflist.SWIFTGenerator{Writer: writer}
+		case scenarioAzurecniChainedCilium:
+			conflistGenerator = &cniconflist.AzureCNIChainedCiliumGenerator{Writer: writer}
 		default:
 			logger.Errorf("unable to generate cni conflist for unknown scenario: %s", scenario)
 			os.Exit(1)

docs/feature/swift-v2/setup-guide-azcni.md

@@ -0,0 +1,46 @@
+# Swiftv2 Cilium In-place Upgrade Guide
+
+## Steps
+### Clone repo + checkout branch for *.yamls
+```
+git clone https://github.com/Azure/azure-container-networking.git
+git checkout master
+```
+
+### Update Conflist
+
+```
+export CONFLIST=azure-chained-cilium.conflist
+export CONFLIST_PRIORITY=05
+export CNI_IMAGE=acnpublic.azurecr.io/public/containernetworking/azure-cni:v1.7.5-3
+envsubst '${CONFLIST},${CONFLIST_PRIORITY},${CNI_IMAGE}' < test/integration/manifests/cni/conflist-installer.yaml | kubectl apply -f -
+```
+
+
+### Apply Cilium config
+```
+export DIR=1.17
+export CILIUM_VERSION_TAG=v1.17.7-250927
+export CILIUM_IMAGE_REGISTRY=mcr.microsoft.com/containernetworking
+kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-config/cilium-chained-config.yaml
+```
+
+
+### Apply Cilium Agent + Operator + RBAC
+```
+kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-operator/files
+kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-agent/files
+envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-agent/templates/daemonset.yaml | kubectl apply -f -
+envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-operator/templates/deployment.yaml | kubectl apply -f -
+```
+
+
+### Quick Summary
+- Apply conflist installer to update conflist on all nodes
+- Apply Cilium Config
+- Apply Agent + Operator + RBAC
+
+
+## Quick Vaildation testing
+- Check Cilium Management with
+  - `kubectl get cep -A`

docs/feature/swift-v2/setup-guide-cil.md

@@ -0,0 +1,36 @@
+# Swiftv2 Managed Cilium Setup Guide
+
+## Steps
+### Clone repo + checkout branch for *.yamls
+```
+git clone https://github.com/Azure/azure-container-networking.git
+git checkout master
+```
+
+### Update Conflist
+
+```
+export CONFLIST=azure-chained-cilium.conflist
+export CONFLIST_PRIORITY=05
+export CNI_IMAGE=acnpublic.azurecr.io/public/containernetworking/azure-cni:v1.7.5-3
+envsubst '${CONFLIST},${CONFLIST_PRIORITY},${CNI_IMAGE}' < test/integration/manifests/cni/conflist-installer-byon.yaml | kubectl apply -f -
+```
+
+
+### Apply Watcher
+```
+kubectl apply -f test/integration/manifests/cilium/watcher/deployment.yaml
+```
+
+- Watcher obtains existing Cilium RBAC and Daemonset from managed node
+  - We overwrite Cilium Configmap values through the use of args on the `cilium-agent` container within the watcher deployment.
+
+
+
+### Quick Summary
+- Apply conflist installer to update conflist on BYON
+- Apply Watcher and Overwrite existing CM values through `cilium-agent` container
+
+## Quick Vaildation testing
+Check Cilium Management with
+- `kubectl get cep -A`

hack/manifests/kubectl.yaml

@@ -6,7 +6,7 @@ metadata:
 spec:
   containers:
   - name: kubectl
-    image: docker.io/bitnami/kubectl:latest
+    image: mcr.microsoft.com/oss/v2/kubernetes/kubectl
     command: ["/bin/bash", "-c", "--"]
     args: ["sleep 3600"]
     env:

test/integration/manifests/cilium/netpol/default-allow.yaml

@@ -0,0 +1,15 @@
+## Only allows traffic within the default namespace
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: allow-default
+spec:
+  endpointSelector: {}
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: default
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: default

test/integration/manifests/cilium/v1.17/cilium-agent/templates/daemonset.yaml

@@ -29,8 +29,6 @@ spec:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
-              - key: kubernetes.azure.com/cluster
-                operator: Exists
               - key: type
                 operator: NotIn
                 values:
@@ -418,7 +416,7 @@ spec:
           path: /proc/sys/kernel
           type: Directory
         name: host-proc-sys-kernel
-      - hostPath: 
+      - hostPath:
           path: /var/run/netns
           type: DirectoryOrCreate
         name: cilium-netns