Skip to content

Commit 2aa70b3

Browse files
sivakami-projectssivakamiCopilot
authored
Swiftv2 Long running cluster - test pipeline (#4099)
* init swiftv2 pipeline for persistent tests on aks clusters. * Set default params. * Update pipeline.yaml for Azure Pipelines * long running pipeline infra setup. * Set depedencies for pipeline jobs. * template for long running cluster. * set template. * set dependency for jobs. * Change job name. * Set job scripts. * set pipeline scripts with permissions. * set script path. * set template params. * Set pipeline template for long running clusters. * test change. * set params. * set params in pipeline scripts. * set cx vnet name. * Create clusters parallely * create NSG. * Change dependency for creating nsg. * Update .pipelines/swiftv2-long-running/scripts/create_peerings.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: sivakami-projects <126191544+sivakami-projects@users.noreply.github.com> * Update .pipelines/swiftv2-long-running/scripts/create_nsg.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: sivakami-projects <126191544+sivakami-projects@users.noreply.github.com> * Add success/error message for each resource creation. * Remove unused argument from template. * Rename subnets. Changed NSG rules to prevent network connectivity between vnet 1 subnet 1 and vnet 1 subnet2. * Private endpoints. * Change pipeline template. * Set output variables. * private endpoint. * update private endpoint. * create storage account. * disallow shared key access. * change pipeline template. * Removed unused param. * Link private endpoint dns to vnet a2 and vnet a3. * attach nsg rule to subnets. * Link nsg with subnet. * Private endpoint fix - long running pipeline. * Verify each resource creation - long running cluster test pipeline. * verify storage account creation. * use make tragets to create aks clusters. * misc. * set aks custom headers. * Use aks common field in swiftv2-podsubnet-cluster creation. --------- Signed-off-by: sivakami-projects <126191544+sivakami-projects@users.noreply.github.com> Co-authored-by: sivakami <sivakamis@microsoft.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent cc08804 commit 2aa70b3

File tree

9 files changed

+636
-0
lines changed

9 files changed

+636
-0
lines changed

‎.pipelines/swiftv2-long-running/pipeline.yaml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
trigger: none
2+
3+
parameters:
4+
- name: subscriptionId
5+
displayName: "Azure Subscription ID"
6+
type: string
7+
default: "37deca37-c375-4a14-b90a-043849bd2bf1"
8+
9+
- name: location
10+
displayName: "Deployment Region"
11+
type: string
12+
default: "centraluseuap"
13+
14+
- name: resourceGroupName
15+
displayName: "Resource Group Name"
16+
type: string
17+
default: "long-run-$(Build.BuildId)"
18+
19+
- name: vmSkuDefault
20+
displayName: "VM SKU for Default Node Pool"
21+
type: string
22+
default: "Standard_D2s_v3"
23+
24+
- name: vmSkuHighNIC
25+
displayName: "VM SKU for High NIC Node Pool"
26+
type: string
27+
default: "Standard_D16s_v3"
28+
29+
- name: serviceConnection
30+
displayName: "Azure Service Connection"
31+
type: string
32+
default: "Azure Container Networking - Standalone Test Service Connection"
33+
34+
extends:
35+
template: template/long-running-pipeline-template.yaml
36+
parameters:
37+
subscriptionId: ${{ parameters.subscriptionId }}
38+
location: ${{ parameters.location }}
39+
resourceGroupName: ${{ parameters.resourceGroupName }}
40+
vmSkuDefault: ${{ parameters.vmSkuDefault }}
41+
vmSkuHighNIC: ${{ parameters.vmSkuHighNIC }}
42+
serviceConnection: ${{ parameters.serviceConnection }}

‎.pipelines/swiftv2-long-running/scripts/create_aks.sh‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
trap 'echo "[ERROR] Failed during Resource group or AKS cluster creation." >&2' ERR
4+
SUBSCRIPTION_ID=$1
5+
LOCATION=$2
6+
RG=$3
7+
VM_SKU_DEFAULT=$4
8+
VM_SKU_HIGHNIC=$5
9+
10+
CLUSTER_COUNT=2
11+
CLUSTER_PREFIX="aks"
12+
DEFAULT_NODE_COUNT=1
13+
COMMON_TAGS="fastpathenabled=true RGOwner=LongRunningTestPipelines stampcreatorserviceinfo=true"
14+
15+
wait_for_provisioning() { # Helper for safe retry/wait for provisioning states (basic)
16+
local rg="$1" clusterName="$2"
17+
echo "Waiting for AKS '$clusterName' in RG '$rg' to reach Succeeded/Failed (polling)..."
18+
while :; do
19+
state=$(az aks show --resource-group "$rg" --name "$clusterName" --query provisioningState -o tsv 2>/dev/null || true)
20+
if [ -z "$state" ]; then
21+
sleep 3
22+
continue
23+
fi
24+
case "$state" in
25+
Succeeded|Succeeded*) echo "Provisioning state: $state"; break ;;
26+
Failed|Canceled|Rejected) echo "Provisioning finished with state: $state"; break ;;
27+
*) printf "."; sleep 6 ;;
28+
esac
29+
done
30+
}
31+
32+
33+
for i in $(seq 1 "$CLUSTER_COUNT"); do
34+
echo "=============================="
35+
echo " Working on cluster set #$i"
36+
echo "=============================="
37+
38+
CLUSTER_NAME="${CLUSTER_PREFIX}-${i}"
39+
echo "Creating AKS cluster '$CLUSTER_NAME' in RG '$RG'"
40+
41+
make -C ./hack/aks azcfg AZCLI=az REGION=$LOCATION
42+
43+
make -C ./hack/aks swiftv2-podsubnet-cluster-up \
44+
AZCLI=az REGION=$LOCATION \
45+
SUB=$SUBSCRIPTION_ID \
46+
GROUP=$RG \
47+
CLUSTER=$CLUSTER_NAME \
48+
NODE_COUNT=$DEFAULT_NODE_COUNT \
49+
VM_SIZE=$VM_SKU_DEFAULT \
50+
51+
echo " - waiting for AKS provisioning state..."
52+
wait_for_provisioning "$RG" "$CLUSTER_NAME"
53+
54+
echo "Adding multi-tenant nodepool ' to '$CLUSTER_NAME'"
55+
make -C ./hack/aks linux-swiftv2-nodepool-up \
56+
AZCLI=az REGION=$LOCATION \
57+
GROUP=$RG \
58+
VM_SIZE=$VM_SKU_HIGHNIC \
59+
CLUSTER=$CLUSTER_NAME \
60+
SUB=$SUBSCRIPTION_ID \
61+
62+
done
63+
echo "All done. Created $CLUSTER_COUNT cluster set(s)."

‎.pipelines/swiftv2-long-running/scripts/create_nsg.sh‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
trap 'echo "[ERROR] Failed during NSG creation or rule setup." >&2' ERR
4+
5+
SUBSCRIPTION_ID=$1
6+
RG=$2
7+
LOCATION=$3
8+
9+
VNET_A1="cx_vnet_a1"
10+
SUBNET1_PREFIX="10.10.1.0/24"
11+
SUBNET2_PREFIX="10.10.2.0/24"
12+
NSG_NAME="${VNET_A1}-nsg"
13+
14+
verify_nsg() {
15+
local rg="$1"; local name="$2"
16+
echo "==> Verifying NSG: $name"
17+
if az network nsg show -g "$rg" -n "$name" &>/dev/null; then
18+
echo "[OK] Verified NSG $name exists."
19+
else
20+
echo "[ERROR] NSG $name not found!" >&2
21+
exit 1
22+
fi
23+
}
24+
25+
verify_nsg_rule() {
26+
local rg="$1"; local nsg="$2"; local rule="$3"
27+
echo "==> Verifying NSG rule: $rule in $nsg"
28+
if az network nsg rule show -g "$rg" --nsg-name "$nsg" -n "$rule" &>/dev/null; then
29+
echo "[OK] Verified NSG rule $rule exists in $nsg."
30+
else
31+
echo "[ERROR] NSG rule $rule not found in $nsg!" >&2
32+
exit 1
33+
fi
34+
}
35+
36+
verify_subnet_nsg_association() {
37+
local rg="$1"; local vnet="$2"; local subnet="$3"; local nsg="$4"
38+
echo "==> Verifying NSG association on subnet $subnet..."
39+
local associated_nsg
40+
associated_nsg=$(az network vnet subnet show -g "$rg" --vnet-name "$vnet" -n "$subnet" --query "networkSecurityGroup.id" -o tsv 2>/dev/null || echo "")
41+
if [[ "$associated_nsg" == *"$nsg"* ]]; then
42+
echo "[OK] Verified subnet $subnet is associated with NSG $nsg."
43+
else
44+
echo "[ERROR] Subnet $subnet is NOT associated with NSG $nsg!" >&2
45+
exit 1
46+
fi
47+
}
48+
49+
# -------------------------------
50+
# 1. Create NSG
51+
# -------------------------------
52+
echo "==> Creating Network Security Group: $NSG_NAME"
53+
az network nsg create -g "$RG" -n "$NSG_NAME" -l "$LOCATION" --output none \
54+
&& echo "[OK] NSG '$NSG_NAME' created."
55+
verify_nsg "$RG" "$NSG_NAME"
56+
57+
# -------------------------------
58+
# 2. Create NSG Rules
59+
# -------------------------------
60+
echo "==> Creating NSG rule to DENY traffic from Subnet1 ($SUBNET1_PREFIX) to Subnet2 ($SUBNET2_PREFIX)"
61+
az network nsg rule create \
62+
--resource-group "$RG" \
63+
--nsg-name "$NSG_NAME" \
64+
--name deny-subnet1-to-subnet2 \
65+
--priority 100 \
66+
--source-address-prefixes "$SUBNET1_PREFIX" \
67+
--destination-address-prefixes "$SUBNET2_PREFIX" \
68+
--direction Inbound \
69+
--access Deny \
70+
--protocol "*" \
71+
--description "Deny all traffic from Subnet1 to Subnet2" \
72+
--output none \
73+
&& echo "[OK] Deny rule from Subnet1 → Subnet2 created."
74+
75+
verify_nsg_rule "$RG" "$NSG_NAME" "deny-subnet1-to-subnet2"
76+
77+
echo "==> Creating NSG rule to DENY traffic from Subnet2 ($SUBNET2_PREFIX) to Subnet1 ($SUBNET1_PREFIX)"
78+
az network nsg rule create \
79+
--resource-group "$RG" \
80+
--nsg-name "$NSG_NAME" \
81+
--name deny-subnet2-to-subnet1 \
82+
--priority 200 \
83+
--source-address-prefixes "$SUBNET2_PREFIX" \
84+
--destination-address-prefixes "$SUBNET1_PREFIX" \
85+
--direction Inbound \
86+
--access Deny \
87+
--protocol "*" \
88+
--description "Deny all traffic from Subnet2 to Subnet1" \
89+
--output none \
90+
&& echo "[OK] Deny rule from Subnet2 → Subnet1 created."
91+
92+
verify_nsg_rule "$RG" "$NSG_NAME" "deny-subnet2-to-subnet1"
93+
94+
# -------------------------------
95+
# 3. Associate NSG with Subnets
96+
# -------------------------------
97+
for SUBNET in s1 s2; do
98+
echo "==> Associating NSG $NSG_NAME with subnet $SUBNET"
99+
az network vnet subnet update \
100+
--name "$SUBNET" \
101+
--vnet-name "$VNET_A1" \
102+
--resource-group "$RG" \
103+
--network-security-group "$NSG_NAME" \
104+
--output none
105+
verify_subnet_nsg_association "$RG" "$VNET_A1" "$SUBNET" "$NSG_NAME"
106+
done
107+
108+
echo "NSG '$NSG_NAME' created successfully with bidirectional isolation between Subnet1 and Subnet2."
109+

‎.pipelines/swiftv2-long-running/scripts/create_pe.sh‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
trap 'echo "[ERROR] Failed during Private Endpoint or DNS setup." >&2' ERR
4+
5+
SUBSCRIPTION_ID=$1
6+
LOCATION=$2
7+
RG=$3
8+
SA1_NAME=$4 # Storage account 1
9+
10+
VNET_A1="cx_vnet_a1"
11+
VNET_A2="cx_vnet_a2"
12+
VNET_A3="cx_vnet_a3"
13+
SUBNET_PE_A1="pe"
14+
PE_NAME="${SA1_NAME}-pe"
15+
PRIVATE_DNS_ZONE="privatelink.blob.core.windows.net"
16+
17+
# -------------------------------
18+
# Function: Verify Resource Exists
19+
# -------------------------------
20+
verify_dns_zone() {
21+
local rg="$1"; local zone="$2"
22+
echo "==> Verifying Private DNS zone: $zone"
23+
if az network private-dns zone show -g "$rg" -n "$zone" &>/dev/null; then
24+
echo "[OK] Verified DNS zone $zone exists."
25+
else
26+
echo "[ERROR] DNS zone $zone not found!" >&2
27+
exit 1
28+
fi
29+
}
30+
31+
verify_dns_link() {
32+
local rg="$1"; local zone="$2"; local link="$3"
33+
echo "==> Verifying DNS link: $link for zone $zone"
34+
if az network private-dns link vnet show -g "$rg" --zone-name "$zone" -n "$link" &>/dev/null; then
35+
echo "[OK] Verified DNS link $link exists."
36+
else
37+
echo "[ERROR] DNS link $link not found!" >&2
38+
exit 1
39+
fi
40+
}
41+
42+
verify_private_endpoint() {
43+
local rg="$1"; local name="$2"
44+
echo "==> Verifying Private Endpoint: $name"
45+
if az network private-endpoint show -g "$rg" -n "$name" &>/dev/null; then
46+
echo "[OK] Verified Private Endpoint $name exists."
47+
else
48+
echo "[ERROR] Private Endpoint $name not found!" >&2
49+
exit 1
50+
fi
51+
}
52+
53+
# 1. Create Private DNS zone
54+
echo "==> Creating Private DNS zone: $PRIVATE_DNS_ZONE"
55+
az network private-dns zone create -g "$RG" -n "$PRIVATE_DNS_ZONE" --output none \
56+
&& echo "[OK] DNS zone $PRIVATE_DNS_ZONE created."
57+
58+
verify_dns_zone "$RG" "$PRIVATE_DNS_ZONE"
59+
60+
# 2. Link DNS zone to VNet
61+
for VNET in "$VNET_A1" "$VNET_A2" "$VNET_A3"; do
62+
LINK_NAME="${VNET}-link"
63+
echo "==> Linking DNS zone $PRIVATE_DNS_ZONE to VNet $VNET"
64+
az network private-dns link vnet create \
65+
-g "$RG" -n "$LINK_NAME" \
66+
--zone-name "$PRIVATE_DNS_ZONE" \
67+
--virtual-network "$VNET" \
68+
--registration-enabled false \
69+
--output none \
70+
&& echo "[OK] Linked DNS zone to $VNET."
71+
verify_dns_link "$RG" "$PRIVATE_DNS_ZONE" "$LINK_NAME"
72+
done
73+
74+
# 3. Create Private Endpoint
75+
echo "==> Creating Private Endpoint for Storage Account: $SA1_NAME"
76+
SA1_ID=$(az storage account show -g "$RG" -n "$SA1_NAME" --query id -o tsv)
77+
az network private-endpoint create \
78+
-g "$RG" -n "$PE_NAME" -l "$LOCATION" \
79+
--vnet-name "$VNET_A1" --subnet "$SUBNET_PE_A1" \
80+
--private-connection-resource-id "$SA1_ID" \
81+
--group-id blob \
82+
--connection-name "${PE_NAME}-conn" \
83+
--output none \
84+
&& echo "[OK] Private Endpoint $PE_NAME created for $SA1_NAME."
85+
verify_private_endpoint "$RG" "$PE_NAME"
86+
87+
echo "All Private DNS and Endpoint resources created and verified successfully."

‎.pipelines/swiftv2-long-running/scripts/create_peerings.sh‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
trap 'echo "[ERROR] Failed during VNet peering creation." >&2' ERR
4+
5+
RG=$1
6+
VNET_A1="cx_vnet_a1"
7+
VNET_A2="cx_vnet_a2"
8+
VNET_A3="cx_vnet_a3"
9+
VNET_B1="cx_vnet_b1"
10+
11+
verify_peering() {
12+
local rg="$1"; local vnet="$2"; local peering="$3"
13+
echo "==> Verifying peering $peering on $vnet..."
14+
if az network vnet peering show -g "$rg" --vnet-name "$vnet" -n "$peering" --query "peeringState" -o tsv | grep -q "Connected"; then
15+
echo "[OK] Peering $peering on $vnet is Connected."
16+
else
17+
echo "[ERROR] Peering $peering on $vnet not found or not Connected!" >&2
18+
exit 1
19+
fi
20+
}
21+
22+
peer_two_vnets() {
23+
local rg="$1"; local v1="$2"; local v2="$3"; local name12="$4"; local name21="$5"
24+
echo "==> Peering $v1 <-> $v2"
25+
az network vnet peering create -g "$rg" -n "$name12" --vnet-name "$v1" --remote-vnet "$v2" --allow-vnet-access --output none \
26+
&& echo "Created peering $name12"
27+
az network vnet peering create -g "$rg" -n "$name21" --vnet-name "$v2" --remote-vnet "$v1" --allow-vnet-access --output none \
28+
&& echo "Created peering $name21"
29+
30+
# Verify both peerings are active
31+
verify_peering "$rg" "$v1" "$name12"
32+
verify_peering "$rg" "$v2" "$name21"
33+
}
34+
35+
peer_two_vnets "$RG" "$VNET_A1" "$VNET_A2" "A1-to-A2" "A2-to-A1"
36+
peer_two_vnets "$RG" "$VNET_A2" "$VNET_A3" "A2-to-A3" "A3-to-A2"
37+
peer_two_vnets "$RG" "$VNET_A1" "$VNET_A3" "A1-to-A3" "A3-to-A1"
38+
echo "All VNet peerings created and verified successfully."

‎.pipelines/swiftv2-long-running/scripts/create_storage.sh‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
trap 'echo "[ERROR] Failed during Storage Account creation." >&2' ERR
4+
5+
SUBSCRIPTION_ID=$1
6+
LOCATION=$2
7+
RG=$3
8+
9+
RAND=$(openssl rand -hex 4)
10+
SA1="sa1${RAND}"
11+
SA2="sa2${RAND}"
12+
13+
# Set subscription context
14+
az account set --subscription "$SUBSCRIPTION_ID"
15+
16+
# Create storage accounts
17+
for SA in "$SA1" "$SA2"; do
18+
echo "==> Creating storage account $SA"
19+
az storage account create \
20+
--name "$SA" \
21+
--resource-group "$RG" \
22+
--location "$LOCATION" \
23+
--sku Standard_LRS \
24+
--kind StorageV2 \
25+
--allow-blob-public-access false \
26+
--allow-shared-key-access false \
27+
--https-only true \
28+
--min-tls-version TLS1_2 \
29+
--query "name" -o tsv \
30+
&& echo "Storage account $SA created successfully."
31+
# Verify creation success
32+
echo "==> Verifying storage account $SA exists..."
33+
if az storage account show --name "$SA" --resource-group "$RG" &>/dev/null; then
34+
echo "[OK] Storage account $SA verified successfully."
35+
else
36+
echo "[ERROR] Storage account $SA not found after creation!" >&2
37+
exit 1
38+
fi
39+
done
40+
41+
echo "All storage accounts created and verified successfully."
42+
43+
# Set pipeline output variables
44+
set +x
45+
echo "##vso[task.setvariable variable=StorageAccount1;isOutput=true]$SA1"
46+
echo "##vso[task.setvariable variable=StorageAccount2;isOutput=true]$SA2"
47+
set -x

0 commit comments

Comments
 (0)